Implement an optimized 'swap_ranges' for contiguous, trivially_relocatable ranges

clang/include/clang/AST/ASTContext.h

@@ -2491,6 +2491,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   TypeInfoChars getTypeInfoInChars(const Type *T) const;
   TypeInfoChars getTypeInfoInChars(QualType T) const;
 
+  CharUnits getStartOfValueRepresentation(QualType T) const;
+
   /// Determine if the alignment the type has was required using an
   /// alignment attribute.
   bool isAlignmentRequired(const Type *T) const;

clang/include/clang/AST/CXXRecordDeclDefinitionBits.def

@@ -224,6 +224,10 @@ FIELD(StructuralIfLiteral, 1, NO_MERGE)
 /// explicitly deleted or defaulted).
 FIELD(UserProvidedDefaultConstructor, 1, NO_MERGE)
 
+FIELD(UserProvidedMoveAssignment, 1, NO_MERGE)
+FIELD(UserProvidedCopyAssignment, 1, NO_MERGE)
+FIELD(ExplicitlyDeletedMoveAssignment, 1, NO_MERGE)
+
 /// The special members which have been declared for this class,
 /// either by the user or implicitly.
 FIELD(DeclaredSpecialMembers, 6, MERGE_OR)
@@ -253,4 +257,8 @@ FIELD(IsAnyDestructorNoReturn, 1, NO_MERGE)
 /// type that is intangible). HLSL only.
 FIELD(IsHLSLIntangible, 1, NO_MERGE)
 
+FIELD(IsTriviallyRelocatable, 1, NO_MERGE)
+
+FIELD(IsReplaceable, 1, NO_MERGE)
+
 #undef FIELD

clang/include/clang/AST/DeclCXX.h

@@ -127,6 +127,28 @@ class AccessSpecDecl : public Decl {
   static bool classofKind(Kind K) { return K == AccessSpec; }
 };
 
+enum class MemberwiseRelocatableOrReplaceableKind { Relocatable, Replaceable };
+
+template <MemberwiseRelocatableOrReplaceableKind MK>
+class BasicMemberwiseSpecifier {
+public:
+  BasicMemberwiseSpecifier() = default;
+  BasicMemberwiseSpecifier(SourceLocation Begin) : Loc(Begin) {}
+  void Set(SourceLocation Begin) { Loc = Begin; }
+
+  bool isSet() const { return !Loc.isInvalid(); }
+
+  SourceLocation getLocation() const { return Loc; }
+
+private:
+  SourceLocation Loc;
+};
+
+using TriviallyRelocatableSpecifier = BasicMemberwiseSpecifier<
+    MemberwiseRelocatableOrReplaceableKind::Relocatable>;
+using MemberwiseReplaceableSpecifier = BasicMemberwiseSpecifier<
+    MemberwiseRelocatableOrReplaceableKind::Replaceable>;
+
 /// Represents a base class of a C++ class.
 ///
 /// Each CXXBaseSpecifier represents a single, direct base class (or
@@ -349,6 +371,10 @@ class CXXRecordDecl : public RecordDecl {
     /// This is actually currently stored in reverse order.
     LazyDeclPtr FirstFriend;
 
+    TriviallyRelocatableSpecifier TriviallyRelocatable;
+
+    MemberwiseReplaceableSpecifier MemberwiseReplaceable;
+
     DefinitionData(CXXRecordDecl *D);
 
     /// Retrieve the set of direct base classes.
@@ -716,12 +742,19 @@ class CXXRecordDecl : public RecordDecl {
   /// \c true if a defaulted move constructor for this class would be
   /// deleted.
   bool defaultedMoveConstructorIsDeleted() const {
-    assert((!needsOverloadResolutionForMoveConstructor() ||
-            (data().DeclaredSpecialMembers & SMF_MoveConstructor)) &&
-           "this property has not yet been computed by Sema");
+    // assert((!needsOverloadResolutionForMoveConstructor() ||
+    //         (data().DeclaredSpecialMembers & SMF_MoveConstructor)) &&
+    //        "this property has not yet been computed by Sema");
     return data().DefaultedMoveConstructorIsDeleted;
   }
 
+  bool defaultedMoveAssignmentIsDeleted() const {
+    // assert((!needsOverloadResolutionForMoveAssignment() ||
+    //         (data().DeclaredSpecialMembers & SMF_MoveAssignment)) &&
+    //        "this property has not yet been computed by Sema");
+    return data().DefaultedMoveAssignmentIsDeleted;
+  }
+
   /// \c true if a defaulted destructor for this class would be deleted.
   bool defaultedDestructorIsDeleted() const {
     assert((!needsOverloadResolutionForDestructor() ||
@@ -806,6 +839,18 @@ class CXXRecordDecl : public RecordDecl {
     return data().UserDeclaredSpecialMembers & SMF_CopyConstructor;
   }
 
+  bool hasUserProvidedCopyAssignment() const {
+    return data().UserProvidedCopyAssignment;
+  }
+
+  bool hasUserProvidedMoveAssignment() const {
+    return data().UserProvidedCopyAssignment;
+  }
+
+  bool hasExplicitlyDeletedMoveAssignment() const {
+    return data().ExplicitlyDeletedMoveAssignment;
+  }
+
   /// Determine whether this class needs an implicit copy
   /// constructor to be lazily declared.
   bool needsImplicitCopyConstructor() const {
@@ -1464,6 +1509,24 @@ class CXXRecordDecl : public RecordDecl {
     return isLiteral() && data().StructuralIfLiteral;
   }
 
+  TriviallyRelocatableSpecifier getTriviallyRelocatableSpecifier() const {
+    return data().TriviallyRelocatable;
+  }
+
+  MemberwiseReplaceableSpecifier getMemberwiseReplaceableSpecifier() const {
+    return data().MemberwiseReplaceable;
+  }
+
+  bool isTriviallyRelocatable() const { return data().IsTriviallyRelocatable; }
+
+  void setIsTriviallyRelocatable(bool Set) {
+    data().IsTriviallyRelocatable = Set;
+  }
+
+  bool isReplaceable() const { return data().IsReplaceable; }
+
+  void setIsReplaceable(bool Set) { data().IsReplaceable = Set; }
+
   /// Notify the class that this destructor is now selected.
   ///
   /// Important properties of the class depend on destructor properties. Since
@@ -1898,6 +1961,13 @@ class CXXRecordDecl : public RecordDecl {
     return K >= firstCXXRecord && K <= lastCXXRecord;
   }
   void markAbstract() { data().Abstract = true; }
+
+  void setTriviallyRelocatableSpecifier(TriviallyRelocatableSpecifier TRS) {
+    data().TriviallyRelocatable = TRS;
+  }
+  void setMemberwiseReplaceableSpecifier(MemberwiseReplaceableSpecifier MRS) {
+    data().MemberwiseReplaceable = MRS;
+  }
 };
 
 /// Store information needed for an explicit specifier.

clang/include/clang/AST/Stmt.h

@@ -530,7 +530,7 @@ class alignas(void *) Stmt {
     unsigned : NumExprBits;
 
     LLVM_PREFERRED_TYPE(UnaryExprOrTypeTrait)
-    unsigned Kind : 3;
+    unsigned Kind : 4;
     LLVM_PREFERRED_TYPE(bool)
     unsigned IsType : 1; // true if operand is a type, false if an expression.
   };

clang/include/clang/AST/Type.h

@@ -1142,6 +1142,10 @@ class QualType {
   /// Return true if this is a trivially relocatable type.
   bool isTriviallyRelocatableType(const ASTContext &Context) const;
 
+  bool isCppTriviallyRelocatableType(const ASTContext &Context) const;
+
+  bool isReplaceableType(const ASTContext &Context) const;
+
   /// Returns true if it is a class and it might be dynamic.
   bool mayBeDynamicClass() const;
 

clang/include/clang/Basic/Builtins.td

@@ -2756,6 +2756,12 @@ def MemMove : LibBuiltin<"string.h"> {
   let AddBuiltinPrefixedAlias = 1;
 }
 
+def BuiltinTriviallyRelocate : Builtin {
+  let Spellings = ["__builtin_trivially_relocate"];
+  let Attributes = [FunctionWithBuiltinPrefix, CustomTypeChecking, NoThrow];
+  let Prototype = "void*(void*, void*, size_t)";
+}
+
 def StrCpy : LibBuiltin<"string.h"> {
   let Spellings = ["strcpy"];
   let Attributes = [NoThrow];

clang/include/clang/Basic/DiagnosticParseKinds.td

@@ -1049,6 +1049,9 @@ def err_access_specifier_interface : Error<
 def err_duplicate_class_virt_specifier : Error<
   "class already marked '%0'">;
 
+def err_duplicate_class_memberwise_specifier : Error<
+  "class already marked %select{'memberwise_trivially_relocatable'|'memberwise_replaceable'}0">;
+
 def err_duplicate_virt_specifier : Error<
   "class member already marked '%0'">;
 

clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -84,7 +84,8 @@ def err_typecheck_converted_constant_expression_indirect : Error<
 def err_expr_not_cce : Error<
   "%select{case value|enumerator value|non-type template argument|"
   "array size|explicit specifier argument|noexcept specifier argument|"
-  "call to 'size()'|call to 'data()'}0 is not a constant expression">;
+  "call to 'size()'|call to 'data()'|"
+  "trivially_relocatable specifier argument}0 is not a constant expression">;
 def ext_cce_narrowing : ExtWarn<
   "%select{case value|enumerator value|non-type template argument|"
   "array size|explicit specifier argument|noexcept specifier argument|"
@@ -2680,6 +2681,21 @@ def warn_final_dtor_non_final_class : Warning<
 def note_final_dtor_non_final_class_silence : Note<
   "mark %0 as '%select{final|sealed}1' to silence this warning">;
 
+def warn_trivially_relocatable_specifier_on_non_relocatable_class : Warning<
+  "invalid 'trivially_relocatable' specifier on non trivially-relocatable class %0">,
+   InGroup<DiagGroup<"never-relocatatable">>,
+   DefaultError;
+
+def warn_memberwise_replaceable_specifier_on_non_relocatable_class : Warning<
+  "invalid 'memberwise_replaceable' specifier on non memberwise replaceable class %0">,
+  InGroup<DiagGroup<"never-replaceable">>,
+  DefaultError;
+
+def note_trivially_relocatable: Note<
+  "because it %select{inherits from a non trivially-relocatable class %1|"
+  "has a virtual base class %1|"
+  "has a deleted move constructor or assignment operator}0">;
+
 // C++11 attributes
 def err_repeat_attribute : Error<"%0 attribute cannot be repeated">;
 

clang/include/clang/Basic/TokenKinds.def

@@ -320,6 +320,7 @@ KEYWORD(short                       , KEYALL)
 KEYWORD(signed                      , KEYALL)
 UNARY_EXPR_OR_TYPE_TRAIT(sizeof, SizeOf, KEYALL)
 UNARY_EXPR_OR_TYPE_TRAIT(__datasizeof, DataSizeOf, KEYCXX)
+UNARY_EXPR_OR_TYPE_TRAIT(__value_representation_begin, ValueRepresentationBegin, KEYCXX)
 KEYWORD(static                      , KEYALL)
 KEYWORD(struct                      , KEYALL)
 KEYWORD(switch                      , KEYALL)
@@ -557,6 +558,8 @@ TYPE_TRAIT_2(__reference_converts_from_temporary, ReferenceConvertsFromTemporary
 TYPE_TRAIT_2(/*EmptySpellingName*/, IsDeducible, KEYCXX)
 
 TYPE_TRAIT_1(__is_bitwise_cloneable, IsBitwiseCloneable, KEYALL)
+TYPE_TRAIT_1(__is_cpp_trivially_relocatable, IsCppTriviallyRelocatable, KEYCXX)
+TYPE_TRAIT_1(__builtin_is_replaceable, IsReplaceable, KEYCXX)
 
 // Embarcadero Expression Traits
 EXPRESSION_TRAIT(__is_lvalue_expr, IsLValueExpr, KEYCXX)

clang/include/clang/Parse/Parser.h

@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_PARSE_PARSER_H
 #define LLVM_CLANG_PARSE_PARSER_H
 
+#include "clang/AST/DeclCXX.h"
 #include "clang/Basic/OpenACCKinds.h"
 #include "clang/Basic/OperatorPrecedence.h"
 #include "clang/Lex/CodeCompletionHandler.h"
@@ -164,6 +165,8 @@ class Parser : public CodeCompletionHandler {
   mutable IdentifierInfo *Ident_final;
   mutable IdentifierInfo *Ident_GNU_final;
   mutable IdentifierInfo *Ident_override;
+  mutable IdentifierInfo *Ident_trivially_relocatable;
+  mutable IdentifierInfo *Ident_memberwise_replaceable;
 
   // C++2a contextual keywords.
   mutable IdentifierInfo *Ident_import;
@@ -3164,6 +3167,20 @@ class Parser : public CodeCompletionHandler {
                                           SourceLocation FriendLoc);
 
   bool isCXX11FinalKeyword() const;
+
+  bool isCXX2CTriviallyRelocatableKeyword(Token Tok) const;
+  bool isCXX2CTriviallyRelocatableKeyword() const;
+  bool SkipCXX2CTriviallyRelocatableSpecifier();
+  void ParseOptionalCXX2CTriviallyRelocatableSpecifier(
+      TriviallyRelocatableSpecifier &TRS);
+
+  bool isCXX2CMemberwiseReplaceableKeyword(Token Tok) const;
+  bool isCXX2CMemberwiseReplaceableKeyword() const;
+  void ParseOptionalCXX2CMemberwiseReplaceableSpecifier(
+      MemberwiseReplaceableSpecifier &MRS);
+  bool SkipCXX2CMemberwiseReplaceableSpecifier();
+
+  bool isClassCompatibleKeyword(Token Tok) const;
   bool isClassCompatibleKeyword() const;
 
   /// DeclaratorScopeObj - RAII object used in Parser::ParseDirectDeclarator to

clang/include/clang/Sema/Sema.h

@@ -20,6 +20,8 @@
 #include "clang/AST/AttrIterator.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/DeclBase.h"
+#include "clang/AST/Availability.h"
+#include "clang/AST/ComparisonCategories.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclarationName.h"
@@ -3983,20 +3985,31 @@ class Sema final : public SemaBase {
   /// Invoked when we enter a tag definition that we're skipping.
   SkippedDefinitionContext ActOnTagStartSkippedDefinition(Scope *S, Decl *TD);
 
+  TriviallyRelocatableSpecifier
+  ActOnTriviallyRelocatableSpecifier(SourceLocation Loc);
+
+  MemberwiseReplaceableSpecifier
+  ActOnMemberwiseReplaceableSpecifier(SourceLocation Loc);
+
   /// ActOnStartCXXMemberDeclarations - Invoked when we have parsed a
   /// C++ record definition's base-specifiers clause and are starting its
   /// member declarations.
-  void ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagDecl,
-                                       SourceLocation FinalLoc,
-                                       bool IsFinalSpelledSealed,
-                                       bool IsAbstract,
-                                       SourceLocation LBraceLoc);
+  void ActOnStartCXXMemberDeclarations(
+      Scope *S, Decl *TagDecl, SourceLocation FinalLoc,
+      bool IsFinalSpelledSealed, bool IsAbstract,
+      TriviallyRelocatableSpecifier TriviallyRelocatable,
+      MemberwiseReplaceableSpecifier MemberwiseReplaceable,
+      SourceLocation LBraceLoc);
 
   /// ActOnTagFinishDefinition - Invoked once we have finished parsing
   /// the definition of a tag (enumeration, class, struct, or union).
   void ActOnTagFinishDefinition(Scope *S, Decl *TagDecl,
                                 SourceRange BraceRange);
 
+  void CheckCXX2CTriviallyRelocatable(CXXRecordDecl *D);
+
+  void CheckMemberwiseReplaceable(CXXRecordDecl *D);
+
   void ActOnTagFinishSkippedDefinition(SkippedDefinitionContext Context);
 
   /// ActOnTagDefinitionError - Invoked when there was an unrecoverable
@@ -10039,6 +10052,8 @@ class Sema final : public SemaBase {
                                   ///< message.
     CCEK_StaticAssertMessageData, ///< Call to data() in a static assert
                                   ///< message.
+                                  ///
+    CCEK_TriviallyRelocatable,
   };
 
   ExprResult BuildConvertedConstantExpression(Expr *From, QualType T,

clang/lib/AST/ASTContext.cpp

@@ -1796,6 +1796,19 @@ TypeInfoChars ASTContext::getTypeInfoDataSizeInChars(QualType T) const {
   return Info;
 }
 
+CharUnits ASTContext::getStartOfValueRepresentation(QualType T) const {
+  if (getLangOpts().CPlusPlus) {
+    if (const auto *RT = T->getAs<RecordType>();
+        RT && !RT->getDecl()->isInvalidDecl()) {
+      const ASTRecordLayout &layout = getASTRecordLayout(RT->getDecl());
+      if (layout.getFieldCount())
+        return toCharUnitsFromBits(layout.getFieldOffset(0));
+      return layout.getDataSize();
+    }
+  }
+  return CharUnits::Zero();
+}
+
 /// getConstantArrayInfoInChars - Performing the computation in CharUnits
 /// instead of in bits prevents overflowing the uint64_t for some large arrays.
 TypeInfoChars

clang/lib/AST/ByteCode/Compiler.cpp

@@ -1780,7 +1780,8 @@ bool Compiler<Emitter>::VisitUnaryExprOrTypeTraitExpr(
   UnaryExprOrTypeTrait Kind = E->getKind();
   const ASTContext &ASTCtx = Ctx.getASTContext();
 
-  if (Kind == UETT_SizeOf || Kind == UETT_DataSizeOf) {
+  if (Kind == UETT_SizeOf || Kind == UETT_DataSizeOf ||
+      Kind == UETT_ValueRepresentationBegin) {
     QualType ArgType = E->getTypeOfArgument();
 
     // C++ [expr.sizeof]p2: "When applied to a reference or a reference type,
@@ -1797,8 +1798,10 @@ bool Compiler<Emitter>::VisitUnaryExprOrTypeTraitExpr(
 
       if (Kind == UETT_SizeOf)
         Size = ASTCtx.getTypeSizeInChars(ArgType);
-      else
+      else if (Kind == UETT_DataSizeOf)
         Size = ASTCtx.getTypeInfoDataSizeInChars(ArgType).Width;
+      else
+        Size = ASTCtx.getStartOfValueRepresentation(ArgType);
     }
 
     if (DiscardResult)

clang/lib/AST/Decl.cpp

@@ -4414,6 +4414,7 @@ unsigned FunctionDecl::getMemoryFunctionKind() const {
   case Builtin::BImempcpy:
     return Builtin::BImempcpy;
 
+  case Builtin::BI__builtin_trivially_relocate:
   case Builtin::BI__builtin_memmove:
   case Builtin::BI__builtin___memmove_chk:
   case Builtin::BImemmove: