[CIR][CIRGen][Builtin] Support __builtin_memcpy_inline

clang/include/clang/CIR/Dialect/IR/CIROps.td

@@ -4003,19 +4003,14 @@ def CopyOp : CIR_Op<"copy",
 // MemCpyOp && MemMoveOp
 //===----------------------------------------------------------------------===//
 
-class CIR_MemCpyOp<string mnemonic>: CIR_Op<mnemonic, [AllTypesMatch<["dst", "src"]>]> {
-  let arguments = (ins Arg<VoidPtr, "", [MemWrite]>:$dst,
-                       Arg<VoidPtr, "", [MemRead]>:$src,
-                       PrimitiveUInt:$len);
+class CIR_MemOp<string mnemonic>
+    : CIR_Op<mnemonic, [AllTypesMatch<["dst", "src"]>]> {
+  dag commonArgs = (ins Arg<VoidPtr, "", [MemWrite]>:$dst,
+      Arg<VoidPtr, "", [MemRead]>:$src);
   let hasVerifier = 0;
-
-  let extraClassDeclaration = [{
-    /// Returns the byte length type.
-    cir::IntType getLenTy() { return getLen().getType(); }
-  }];
 }
 
-def MemCpyOp : CIR_MemCpyOp<"libc.memcpy"> {
+def MemCpyOp : CIR_MemOp<"libc.memcpy"> {
   let summary = "Equivalent to libc's `memcpy`";
   let description = [{
     Given two CIR pointers, `src` and `dst`, `cir.libc.memcpy` will copy `len`
@@ -4034,13 +4029,20 @@ def MemCpyOp : CIR_MemCpyOp<"libc.memcpy"> {
     ```
   }];
 
+  let arguments = !con(commonArgs, (ins PrimitiveUInt:$len));
+
   let assemblyFormat = [{
     $len `bytes` `from` $src `to` $dst attr-dict
     `:` type($len) `` `,` qualified(type($src)) `->` qualified(type($dst))
   }];
+
+  let extraClassDeclaration = [{
+    /// Returns the byte length type.
+    cir::IntType getLenTy() { return getLen().getType(); }
+  }];
 }
 
-def MemMoveOp : CIR_MemCpyOp<"libc.memmove"> {
+def MemMoveOp : CIR_MemOp<"libc.memmove"> {
   let summary = "Equivalent to libc's `memmove`";
   let description = [{
     Given two CIR pointers, `src` and `dst`, `cir.libc.memmove` will copy `len`
@@ -4057,12 +4059,49 @@ def MemMoveOp : CIR_MemCpyOp<"libc.memmove"> {
     ```
   }];
 
+  let arguments = !con(commonArgs, (ins PrimitiveUInt:$len));
 
   let assemblyFormat = [{
     $len `bytes` `from` $src `to` $dst attr-dict
     `:` qualified(type($dst)) `,` type($len)
   }];
+
+  let extraClassDeclaration = [{
+    /// Returns the byte length type.
+    cir::IntType getLenTy() { return getLen().getType(); }
+  }];
 }
+
+//===----------------------------------------------------------------------===//
+// MemCpyInlineOp
+//===----------------------------------------------------------------------===//
+
+def MemCpyInlineOp : CIR_MemOp<"memcpy_inline"> {
+  let summary = "Memory copy with constant length without calling"
+                "any external function";
+  let description = [{
+    Given two CIR pointers, `src` and `dst`, `memcpy_inline` will copy `len`
+    bytes from the memory pointed by `src` to the memory pointed by `dst`.
+
+    Unlike `cir.libc.memcpy`,  this Op guarantees that no external functions 
+    are called, and length of copied bytes is a constant.
+
+    Examples:
+
+    ```mlir
+      // Copying 2 bytes from one array to a struct:
+      cir.memcpy_inline 2 bytes from %arr to %struct : !cir.ptr<!arr> -> !cir.ptr<!struct>
+    ```
+  }];
+
+  let arguments = !con(commonArgs, (ins I64Attr:$len));
+
+  let assemblyFormat = [{
+    $len `bytes` `from` $src `to` $dst attr-dict
+    `:` qualified(type($src)) `->` qualified(type($dst))
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // MemSetOp
 //===----------------------------------------------------------------------===//

clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp

@@ -1408,8 +1408,22 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       return RValue::get(Dest.getPointer());
   }
 
-  case Builtin::BI__builtin_memcpy_inline:
-    llvm_unreachable("BI__builtin_memcpy_inline NYI");
+  case Builtin::BI__builtin_memcpy_inline: {
+    Address dest = emitPointerWithAlignment(E->getArg(0));
+    Address src = emitPointerWithAlignment(E->getArg(1));
+    emitNonNullArgCheck(RValue::get(dest.getPointer()), E->getArg(0)->getType(),
+                        E->getArg(0)->getExprLoc(), FD, 0);
+    emitNonNullArgCheck(RValue::get(src.getPointer()), E->getArg(1)->getType(),
+                        E->getArg(1)->getExprLoc(), FD, 1);
+    uint64_t size =
+        E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
+    builder.create<cir::MemCpyInlineOp>(
+        getLoc(E->getSourceRange()), dest.getPointer(), src.getPointer(),
+        mlir::IntegerAttr::get(mlir::IntegerType::get(builder.getContext(), 64),
+                               size));
+    // __builtin_memcpy_inline has no return value
+    return RValue::get(nullptr);
+  }
 
   case Builtin::BI__builtin_char_memchr:
   case Builtin::BI__builtin_memchr: {

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp

@@ -778,6 +778,21 @@ class CIRMemChrOpLowering : public mlir::OpConversionPattern<cir::MemChrOp> {
   }
 };
 
+class CIRMemCpyInlineOpLowering
+    : public mlir::OpConversionPattern<cir::MemCpyInlineOp> {
+public:
+  using mlir::OpConversionPattern<cir::MemCpyInlineOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::MemCpyInlineOp op, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyInlineOp>(
+        op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLenAttr(),
+        /*isVolatile=*/false);
+    return mlir::success();
+  }
+};
+
 class CIRMemMoveOpLowering : public mlir::OpConversionPattern<cir::MemMoveOp> {
 public:
   using mlir::OpConversionPattern<cir::MemMoveOp>::OpConversionPattern;
@@ -4331,8 +4346,8 @@ void populateCIRToLLVMConversionPatterns(
       CIRVAStartLowering, CIRVAEndLowering, CIRVACopyLowering, CIRVAArgLowering,
       CIRBrOpLowering, CIRGetMemberOpLowering, CIRGetRuntimeMemberOpLowering,
       CIRSwitchFlatOpLowering, CIRPtrDiffOpLowering, CIRCopyOpLowering,
-      CIRMemCpyOpLowering, CIRMemChrOpLowering, CIRFAbsOpLowering,
-      CIRExpectOpLowering, CIRVTableAddrPointOpLowering,
+      CIRMemCpyOpLowering, CIRMemChrOpLowering, CIRMemCpyInlineOpLowering,
+      CIRFAbsOpLowering, CIRExpectOpLowering, CIRVTableAddrPointOpLowering,
       CIRVectorCreateLowering, CIRVectorCmpOpLowering, CIRVectorSplatLowering,
       CIRVectorTernaryLowering, CIRVectorShuffleIntsLowering,
       CIRVectorShuffleVecLowering, CIRStackSaveLowering, CIRUnreachableLowering,
@@ -4376,27 +4391,27 @@ std::unique_ptr<cir::LowerModule> prepareLowerModule(mlir::ModuleOp module) {
 void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
                           mlir::DataLayout &dataLayout,
                           cir::LowerModule *lowerModule) {
-  converter.addConversion([&,
-                           lowerModule](cir::PointerType type) -> mlir::Type {
-    // Drop pointee type since LLVM dialect only allows opaque pointers.
-
-    auto addrSpace =
-        mlir::cast_if_present<cir::AddressSpaceAttr>(type.getAddrSpace());
-    // Null addrspace attribute indicates the default addrspace.
-    if (!addrSpace)
-      return mlir::LLVM::LLVMPointerType::get(type.getContext());
-
-    assert(lowerModule && "CIR AS map is not available");
-    // Pass through target addrspace and map CIR addrspace to LLVM addrspace by
-    // querying the target info.
-    unsigned targetAS =
-        addrSpace.isTarget()
-            ? addrSpace.getTargetValue()
-            : lowerModule->getTargetLoweringInfo()
-                  .getTargetAddrSpaceFromCIRAddrSpace(addrSpace);
-
-    return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS);
-  });
+  converter.addConversion(
+      [&, lowerModule](cir::PointerType type) -> mlir::Type {
+        // Drop pointee type since LLVM dialect only allows opaque pointers.
+
+        auto addrSpace =
+            mlir::cast_if_present<cir::AddressSpaceAttr>(type.getAddrSpace());
+        // Null addrspace attribute indicates the default addrspace.
+        if (!addrSpace)
+          return mlir::LLVM::LLVMPointerType::get(type.getContext());
+
+        assert(lowerModule && "CIR AS map is not available");
+        // Pass through target addrspace and map CIR addrspace to LLVM addrspace
+        // by querying the target info.
+        unsigned targetAS =
+            addrSpace.isTarget()
+                ? addrSpace.getTargetValue()
+                : lowerModule->getTargetLoweringInfo()
+                      .getTargetAddrSpaceFromCIRAddrSpace(addrSpace);
+
+        return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS);
+      });
   converter.addConversion([&](cir::DataMemberType type) -> mlir::Type {
     return mlir::IntegerType::get(type.getContext(),
                                   dataLayout.getTypeSizeInBits(type));

clang/test/CIR/CodeGen/builtins-memory.c

@@ -1,5 +1,8 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck %s --check-prefix=CIR --input-file=%t.cir
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu  -fclangir -emit-llvm %s -o - \
+// RUN:  | opt -S -passes=instcombine,mem2reg,simplifycfg -o %t.ll 
+// RUN: FileCheck  --check-prefix=LLVM --input-file=%t.ll %s
 
 typedef __SIZE_TYPE__ size_t;
 void test_memcpy_chk(void *dest, const void *src, size_t n) {
@@ -107,3 +110,38 @@ void test_memset_chk(void *dest, int ch, size_t n) {
   // CIR: cir.call @__memset_chk(%[[#DEST_LOAD]], %[[#CH_LOAD]], %[[#N_LOAD1]], %[[#N_LOAD2]])
   __builtin___memset_chk(dest, ch, n, n);
 }
+
+// FIXME: The test should test intrinsic argument alignment, however, 
+// currently we lack support for argument attributes. 
+// Thus, added `COM: LLVM:` lines so we can easily flip the test 
+// when the support of argument attributes is in.
+void test_memcpy_inline(void *dst, const void *src, size_t n) {
+
+  // CIR-LABEL: test_memcpy_inline
+  // CIR: cir.memcpy_inline 0 bytes from {{%.*}} to {{%.*}} : !cir.ptr<!void> -> !cir.ptr<!void>
+
+  // LLVM-LABEL: test_memcpy_inline
+  // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 0, i1 false)
+  // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 {{%.*}}, ptr align 1 {{%.*}}, i64 0, i1 false)
+  __builtin_memcpy_inline(dst, src, 0);
+
+  // CIR: cir.memcpy_inline 1 bytes from {{%.*}} to {{%.*}} : !cir.ptr<!void> -> !cir.ptr<!void>
+
+  // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 1, i1 false)
+  // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 {{%.*}}, ptr align 1 {{%.*}}, i64 1, i1 false)
+  __builtin_memcpy_inline(dst, src, 1);
+
+  // CIR: cir.memcpy_inline 4 bytes from {{%.*}} to {{%.*}} : !cir.ptr<!void> -> !cir.ptr<!void>
+
+  // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 4, i1 false)
+  // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 {{%.*}}, ptr align 1 {{%.*}}, i64 4, i1 false)
+  __builtin_memcpy_inline(dst, src, 4);
+}
+
+void test_memcpy_inline_aligned_buffers(unsigned long long *dst, const unsigned long long *src) {
+
+  // LLVM-LABEL: test_memcpy_inline_aligned_buffers
+  // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 4, i1 false)
+  // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 {{%.*}}, ptr align 8 {{%.*}}, i64 4, i1 false)
+  __builtin_memcpy_inline(dst, src, 4);
+}