Don't Serialize Declarations with package Access

This accidentally started happening when I adjusted getEffectiveAccess to return `Public` for `Package` declarations in swiftlang#69709. As a result, the optimizer thought it had more opportunities to go after declarations that used to be opaque. Unfortunately, this resulted in a miscompile as the devirtualizer was able to look through now-serialized package (static) functions. In one specific instance, the optimizer created direct calls to hidden accessors instead of going through the dispatch thunk.

Author: Robert Widmann

include/swift/AST/Decl.h

@@ -2905,6 +2905,10 @@ class ValueDecl : public Decl {
   /// if the base declaration is \c open, the override might have to be too.
   bool hasOpenAccess(const DeclContext *useDC) const;
 
+  /// Returns whether this declaration should be treated as having the \c
+  /// package access level.
+  bool hasPackageAccess() const;
+
   /// FIXME: This is deprecated.
   bool isRecursiveValidation() const;
 

lib/AST/Decl.cpp

@@ -4203,6 +4203,13 @@ bool ValueDecl::hasOpenAccess(const DeclContext *useDC) const {
   return access == AccessLevel::Open;
 }
 
+bool ValueDecl::hasPackageAccess() const {
+  AccessLevel access =
+      getAdjustedFormalAccess(this, /*useDC*/ nullptr,
+                              /*treatUsableFromInlineAsPublic*/ false);
+  return access == AccessLevel::Package;
+}
+
 /// Given the formal access level for using \p VD, compute the scope where
 /// \p VD may be accessed, taking \@usableFromInline, \@testable imports,
 /// \@_spi imports, and enclosing access levels into account.

lib/SIL/IR/SILDeclRef.cpp

@@ -844,7 +844,7 @@ IsSerialized_t SILDeclRef::isSerialized() const {
   }
 
   // Anything else that is not public is not serializable.
-  if (d->getEffectiveAccess() < AccessLevel::Public)
+  if (d->getEffectiveAccess() < AccessLevel::Public || d->hasPackageAccess())
     return IsNotSerialized;
 
   // Enum element constructors are serializable if the enum is

lib/SIL/IR/SILWitnessTable.cpp

@@ -171,7 +171,8 @@ bool SILWitnessTable::conformanceIsSerialized(
   if (normalConformance && normalConformance->isResilient())
     return false;
 
-  if (conformance->getProtocol()->getEffectiveAccess() < AccessLevel::Public)
+  if (conformance->getProtocol()->getEffectiveAccess() < AccessLevel::Public ||
+      conformance->getProtocol()->hasPackageAccess())
     return false;
 
   auto *nominal = conformance->getDeclContext()->getSelfNominalTypeDecl();

lib/SILGen/SILGenType.cpp

@@ -307,7 +307,7 @@ class SILGenVTable : public SILVTableVisitor<SILGenVTable> {
     IsSerialized_t serialized = IsNotSerialized;
     auto classIsPublic = theClass->getEffectiveAccess() >= AccessLevel::Public;
     // Only public, fixed-layout classes should have serialized vtables.
-    if (classIsPublic && !isResilient)
+    if (classIsPublic && !theClass->hasPackageAccess() && !isResilient)
       serialized = IsSerialized;
 
     // Finally, create the vtable.
@@ -1080,7 +1080,7 @@ void SILGenModule::emitNonCopyableTypeDeinitTable(NominalTypeDecl *nom) {
   auto serialized = IsSerialized_t::IsNotSerialized;
   bool nomIsPublic = nom->getEffectiveAccess() >= AccessLevel::Public;
   // We only serialize the deinit if the type is public and not resilient.
-  if (nomIsPublic && !nom->isResilient())
+  if (nomIsPublic && !nom->hasPackageAccess() && !nom->isResilient())
     serialized = IsSerialized;
   SILMoveOnlyDeinit::create(f->getModule(), nom, serialized, f);
 }