[mlir] Change vector.transfer_read/write "masked" attribute to "in_bounds".

This is in preparation for adding a new "mask" operand. The existing "masked" attribute was used to specify dimensions that may be out-of-bounds. Such transfers can be lowered to masked load/stores. The new "in_bounds" attribute is used to specify dimensions that are guaranteed to be within bounds. (Semantics is inverted.)

Differential Revision: https://reviews.llvm.org/D99639

Author: matthias-springer

mlir/include/mlir/Dialect/Vector/VectorOps.h

@@ -121,18 +121,18 @@ enum class VectorTransposeLowering {
   /// intrinsics.
   Flat = 1,
 };
-/// Enum to control the splitting of `vector.transfer` operations into masked
-/// and unmasked variants.
+/// Enum to control the splitting of `vector.transfer` operations into
+/// in-bounds and out-of-bounds variants.
 enum class VectorTransferSplit {
   /// Do not split vector transfer operations.
   None = 0,
-  /// Split using masked + unmasked vector.transfer operations.
+  /// Split using in-bounds + out-of-bounds vector.transfer operations.
   VectorTransfer = 1,
-  /// Split using a unmasked vector.transfer + linalg.fill + linalg.copy
+  /// Split using an in-bounds vector.transfer + linalg.fill + linalg.copy
   /// operations.
   LinalgCopy = 2,
-  /// Do not split vector transfer operation but instead mark it as "unmasked".
-  ForceUnmasked = 3
+  /// Do not split vector transfer operation but instead mark it as "in-bounds".
+  ForceInBounds = 3
 };
 /// Structure to control the behavior of vector transform patterns.
 struct VectorTransformsOptions {

mlir/include/mlir/Dialect/Vector/VectorOps.td

@@ -1139,7 +1139,7 @@ def Vector_TransferReadOp :
     ]>,
     Arguments<(ins AnyShaped:$source, Variadic<Index>:$indices,
                AffineMapAttr:$permutation_map, AnyType:$padding,
-               OptionalAttr<BoolArrayAttr>:$masked)>,
+               OptionalAttr<BoolArrayAttr>:$in_bounds)>,
     Results<(outs AnyVector:$vector)> {
 
   let summary = "Reads a supervector from memory into an SSA vector value.";
@@ -1166,15 +1166,14 @@ def Vector_TransferReadOp :
     The size of the slice is specified by the size of the vector, given as the
     return type.
 
-    An `ssa-value` of the same elemental type as the MemRef/Tensor is provided
-    as the last operand to specify padding in the case of out-of-bounds
-    accesses.
+    An SSA value `padding` of the same elemental type as the MemRef/Tensor is
+    provided to specify a fallback value in the case of out-of-bounds accesses.
 
     An optional boolean array attribute is provided to specify which dimensions
-    of the transfer need masking. When a dimension is specified as not requiring
-    masking, the `vector.transfer_read` may be lowered to simple loads. The
-    absence of this `masked` attribute signifies that all dimensions of the
-    transfer need to be masked.
+    of the transfer are guaranteed to be within bounds. The absence of this
+    `in_bounds` attribute signifies that any dimension of the transfer may be
+    out-of-bounds. A `vector.transfer_read` can be lowered to a simple load if
+    all dimensions are specified to be within bounds.
 
     This operation is called 'read' by opposition to 'load' because the
     super-vector granularity is generally not representable with a single
@@ -1291,15 +1290,15 @@ def Vector_TransferReadOp :
     // Builder that sets padding to zero.
     OpBuilder<(ins "VectorType":$vector, "Value":$source,
       "ValueRange":$indices, "AffineMap":$permutationMap,
-      CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>,
+      CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
     // Builder that sets padding to 'getMinorIdentityMap'.
     OpBuilder<(ins "VectorType":$vector, "Value":$source,
       "ValueRange":$indices, "Value":$padding,
-      CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>,
+      CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
     // Builder that sets permutation map (resp. padding) to
     // 'getMinorIdentityMap' (resp. zero).
     OpBuilder<(ins "VectorType":$vector, "Value":$source,
-      "ValueRange":$indices, CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>
+      "ValueRange":$indices, CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
   ];
 
   let hasFolder = 1;
@@ -1314,7 +1313,7 @@ def Vector_TransferWriteOp :
     Arguments<(ins AnyVector:$vector, AnyShaped:$source,
                Variadic<Index>:$indices,
                AffineMapAttr:$permutation_map,
-               OptionalAttr<BoolArrayAttr>:$masked)>,
+               OptionalAttr<BoolArrayAttr>:$in_bounds)>,
     Results<(outs Optional<AnyRankedTensor>:$result)> {
 
   let summary = "The vector.transfer_write op writes a supervector to memory.";
@@ -1343,10 +1342,10 @@ def Vector_TransferWriteOp :
     The size of the slice is specified by the size of the vector.
 
     An optional boolean array attribute is provided to specify which dimensions
-    of the transfer need masking. When a dimension is specified as not requiring
-    masking, the `vector.transfer_write` may be lowered to simple stores. The
-    absence of this `mask` attribute signifies that all dimensions of the
-    transfer need to be masked.
+    of the transfer are guaranteed to be within bounds. The absence of this
+    `in_bounds` attribute signifies that any dimension of the transfer may be
+    out-of-bounds. A `vector.transfer_write` can be lowered to a simple store
+    if all dimensions are specified to be within bounds.
 
     This operation is called 'write' by opposition to 'store' because the
     super-vector granularity is generally not representable with a single
@@ -1387,13 +1386,13 @@ def Vector_TransferWriteOp :
   let builders = [
     // Builder that sets permutation map to 'getMinorIdentityMap'.
     OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
-      CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>,
+      CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
     OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
       "AffineMap":$permutationMap)>,
     OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
-      "AffineMapAttr":$permutationMap, "ArrayAttr":$masked)>,
+      "AffineMapAttr":$permutationMap, "ArrayAttr":$inBounds)>,
     OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
-      "AffineMap":$permutationMap, "ArrayAttr":$masked)>,
+      "AffineMap":$permutationMap, "ArrayAttr":$inBounds)>,
   ];
 
   let hasFolder = 1;

mlir/include/mlir/Dialect/Vector/VectorTransforms.h

@@ -166,7 +166,8 @@ struct UnrollVectorPattern : public RewritePattern {
   UnrollVectorOptions options;
 };
 
-/// Split a vector.transfer operation into an unmasked fastpath and a slowpath.
+/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds
+/// masking) fastpath and a slowpath.
 /// If `ifOp` is not null and the result is `success, the `ifOp` points to the
 /// newly created conditional upon function return.
 /// To accomodate for the fact that the original vector.transfer indexing may be
@@ -185,11 +186,11 @@ struct UnrollVectorPattern : public RewritePattern {
 ///      memref.cast %A: memref<A...> to compatibleMemRefType
 ///      scf.yield %view : compatibleMemRefType, index, index
 ///    } else {
-///      // slowpath, masked vector.transfer or linalg.copy.
+///      // slowpath, not in-bounds vector.transfer or linalg.copy.
 ///      memref.cast %alloc: memref<B...> to compatibleMemRefType
 ///      scf.yield %4 : compatibleMemRefType, index, index
 //     }
-///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
+///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]}
 /// ```
 /// where `alloc` is a top of the function alloca'ed buffer of one vector.
 ///

mlir/include/mlir/Interfaces/VectorInterfaces.td

@@ -53,12 +53,12 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
 
   let methods = [
     StaticInterfaceMethod<
-      /*desc=*/"Return the `masked` attribute name.",
+      /*desc=*/"Return the `in_bounds` attribute name.",
       /*retTy=*/"StringRef",
-      /*methodName=*/"getMaskedAttrName",
+      /*methodName=*/"getInBoundsAttrName",
       /*args=*/(ins),
       /*methodBody=*/"",
-      /*defaultImplementation=*/ [{ return "masked"; }]
+      /*defaultImplementation=*/ [{ return "in_bounds"; }]
     >,
     StaticInterfaceMethod<
       /*desc=*/"Return the `permutation_map` attribute name.",
@@ -70,16 +70,16 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
     >,
     InterfaceMethod<
       /*desc=*/[{
-      Return `false` when the `masked` attribute at dimension
-      `dim` is set to `false`. Return `true` otherwise.}],
+      Return `true` when the `in_bounds` attribute at dimension
+      `dim` is set to `true`. Return `false` otherwise.}],
       /*retTy=*/"bool",
-      /*methodName=*/"isMaskedDim",
+      /*methodName=*/"isDimInBounds",
       /*args=*/(ins "unsigned":$dim),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return !$_op.masked() ||
-          $_op.masked()->template cast<ArrayAttr>()[dim]
-                        .template cast<BoolAttr>().getValue();
+        return $_op.in_bounds() &&
+          $_op.in_bounds()->template cast<ArrayAttr>()[dim]
+                          .template cast<BoolAttr>().getValue();
       }]
     >,
     InterfaceMethod<
@@ -115,11 +115,11 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
       /*defaultImplementation=*/
     >,
     InterfaceMethod<
-      /*desc=*/"Return the `masked` boolean ArrayAttr.",
+      /*desc=*/"Return the `in_bounds` boolean ArrayAttr.",
       /*retTy=*/"Optional<ArrayAttr>",
-      /*methodName=*/"masked",
+      /*methodName=*/"in_bounds",
       /*args=*/(ins),
-      /*methodBody=*/"return $_op.masked();"
+      /*methodBody=*/"return $_op.in_bounds();"
       /*defaultImplementation=*/
     >,
     InterfaceMethod<
@@ -162,14 +162,15 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
         "return $_op.getShapedType().getRank() - $_op.getTransferRank();"
     >,
     InterfaceMethod<
-      /*desc=*/[{ Returns true if at least one of the dimensions is masked.}],
+      /*desc=*/[{ Returns true if at least one of the dimensions may be
+                  out-of-bounds.}],
       /*retTy=*/"bool",
-      /*methodName=*/"hasMaskedDim",
+      /*methodName=*/"hasOutOfBoundsDim",
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
         for (unsigned idx = 0, e = $_op.getTransferRank(); idx < e; ++idx)
-          if ($_op.isMaskedDim(idx))
+          if (!$_op.isDimInBounds(idx))
             return true;
         return false;
       }]

mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp

@@ -79,7 +79,7 @@ void LoadOpOfSubViewFolder<vector::TransferReadOp>::replaceOp(
     ArrayRef<Value> sourceIndices, PatternRewriter &rewriter) const {
   rewriter.replaceOpWithNewOp<vector::TransferReadOp>(
       loadOp, loadOp.getVectorType(), subViewOp.source(), sourceIndices,
-      loadOp.permutation_map(), loadOp.padding(), loadOp.maskedAttr());
+      loadOp.permutation_map(), loadOp.padding(), loadOp.in_boundsAttr());
 }
 
 template <>
@@ -92,12 +92,12 @@ void StoreOpOfSubViewFolder<memref::StoreOp>::replaceOp(
 
 template <>
 void StoreOpOfSubViewFolder<vector::TransferWriteOp>::replaceOp(
-    vector::TransferWriteOp tranferWriteOp, memref::SubViewOp subViewOp,
+    vector::TransferWriteOp transferWriteOp, memref::SubViewOp subViewOp,
     ArrayRef<Value> sourceIndices, PatternRewriter &rewriter) const {
   rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
-      tranferWriteOp, tranferWriteOp.vector(), subViewOp.source(),
-      sourceIndices, tranferWriteOp.permutation_map(),
-      tranferWriteOp.maskedAttr());
+      transferWriteOp, transferWriteOp.vector(), subViewOp.source(),
+      sourceIndices, transferWriteOp.permutation_map(),
+      transferWriteOp.in_boundsAttr());
 }
 } // namespace
 

mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp

@@ -1248,14 +1248,15 @@ class VectorTransferConversion : public ConvertOpToLLVMPattern<ConcreteOp> {
     Value vectorDataPtr =
         castDataPtr(rewriter, loc, dataPtr, memRefType, toLLVMTy(vtp));
 
-    if (!xferOp.isMaskedDim(0))
+    if (xferOp.isDimInBounds(0))
       return replaceTransferOpWithLoadOrStore(rewriter,
                                               *this->getTypeConverter(), loc,
                                               xferOp, operands, vectorDataPtr);
 
     // 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
     // 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
-    // 4. Let dim the memref dimension, compute the vector comparison mask:
+    // 4. Let dim the memref dimension, compute the vector comparison mask
+    //    (in-bounds mask):
     //   [ offset + 0 .. offset + vector_length - 1 ] < [ dim .. dim ]
     //
     // TODO: when the leaf transfer rank is k > 1, we need the last `k`

mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp

@@ -72,7 +72,7 @@ class VectorTransferConversion : public ConvertOpToLLVMPattern<ConcreteOp> {
       return failure();
 
     // Have it handled in vector->llvm conversion pass.
-    if (!xferOp.isMaskedDim(0))
+    if (xferOp.isDimInBounds(0))
       return failure();
 
     auto toLLVMTy = [&](Type t) {

mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp

@@ -230,7 +230,7 @@ emitInBoundsCondition(PatternRewriter &rewriter,
     Value iv = std::get<0>(it), off = std::get<1>(it), ub = std::get<2>(it);
     using namespace mlir::edsc::op;
     majorIvsPlusOffsets.push_back(iv + off);
-    if (xferOp.isMaskedDim(leadingRank + idx)) {
+    if (!xferOp.isDimInBounds(leadingRank + idx)) {
       Value inBoundsCond = onTheFlyFoldSLT(majorIvsPlusOffsets.back(), ub);
       if (inBoundsCond)
         inBoundsCondition = (inBoundsCondition)
@@ -276,14 +276,14 @@ LogicalResult NDTransferOpHelper<TransferReadOp>::doReplace() {
       Value memref = xferOp.source();
       auto map =
           getTransferMinorIdentityMap(xferOp.getShapedType(), minorVectorType);
-      ArrayAttr masked;
-      if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) {
+      ArrayAttr inBounds;
+      if (xferOp.isDimInBounds(xferOp.getVectorType().getRank() - 1)) {
         OpBuilder &b = ScopedContext::getBuilderRef();
-        masked = b.getBoolArrayAttr({false});
+        inBounds = b.getBoolArrayAttr({true});
       }
       return vector_transfer_read(minorVectorType, memref, indexing,
                                   AffineMapAttr::get(map), xferOp.padding(),
-                                  masked);
+                                  inBounds);
     };
 
     // 1. Compute the inBoundsCondition in the current loops ivs + offset
@@ -382,13 +382,13 @@ LogicalResult NDTransferOpHelper<TransferWriteOp>::doReplace() {
         result = memref_load(alloc, majorIvs);
       auto map =
           getTransferMinorIdentityMap(xferOp.getShapedType(), minorVectorType);
-      ArrayAttr masked;
-      if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) {
+      ArrayAttr inBounds;
+      if (xferOp.isDimInBounds(xferOp.getVectorType().getRank() - 1)) {
         OpBuilder &b = ScopedContext::getBuilderRef();
-        masked = b.getBoolArrayAttr({false});
+        inBounds = b.getBoolArrayAttr({true});
       }
       vector_transfer_write(result, xferOp.source(), indexing,
-                            AffineMapAttr::get(map), masked);
+                            AffineMapAttr::get(map), inBounds);
     };
 
     // 1. Compute the inBoundsCondition in the current loops ivs + offset