Merge pull request #66033 from DougGregor/parse-types-via-astgen

[ASTGen] Add experimental feature to use ASTGen in lieu of parsing types

Author: DougGregor

include/swift/AST/CASTBridging.h

@@ -112,6 +112,70 @@ typedef enum ENUM_EXTENSIBILITY_ATTR(open) BridgedMacroDefinitionKind : long {
   BridgedBuiltinExternalMacro
 } BridgedMacroDefinitionKind;
 
+/// Bridged parameter specifiers
+typedef enum ENUM_EXTENSIBILITY_ATTR(open) BridgedAttributedTypeSpecifier : long {
+  BridgedAttributedTypeSpecifierInOut,
+  BridgedAttributedTypeSpecifierBorrowing,
+  BridgedAttributedTypeSpecifierConsuming,
+  BridgedAttributedTypeSpecifierLegacyShared,
+  BridgedAttributedTypeSpecifierLegacyOwned,
+  BridgedAttributedTypeSpecifierConst,
+  BridgedAttributedTypeSpecifierIsolated,
+} BridgedAttributedTypeSpecifier;
+
+
+// Bridged type attribute kinds, which mirror TypeAttrKind exactly.
+typedef enum ENUM_EXTENSIBILITY_ATTR(closed) BridgedTypeAttrKind : long {
+  BridgedTypeAttrKind_autoclosure,
+  BridgedTypeAttrKind_convention,
+  BridgedTypeAttrKind_noescape,
+  BridgedTypeAttrKind_escaping,
+  BridgedTypeAttrKind_differentiable,
+  BridgedTypeAttrKind_noDerivative,
+  BridgedTypeAttrKind_async,
+  BridgedTypeAttrKind_Sendable,
+  BridgedTypeAttrKind_unchecked,
+  BridgedTypeAttrKind__local,
+  BridgedTypeAttrKind__noMetadata,
+  BridgedTypeAttrKind__opaqueReturnTypeOf,
+  BridgedTypeAttrKind_block_storage,
+  BridgedTypeAttrKind_box,
+  BridgedTypeAttrKind_dynamic_self,
+  BridgedTypeAttrKind_sil_weak,
+  BridgedTypeAttrKind_sil_unowned,
+  BridgedTypeAttrKind_sil_unmanaged,
+  BridgedTypeAttrKind_error,
+  BridgedTypeAttrKind_out,
+  BridgedTypeAttrKind_direct,
+  BridgedTypeAttrKind_in,
+  BridgedTypeAttrKind_inout,
+  BridgedTypeAttrKind_inout_aliasable,
+  BridgedTypeAttrKind_in_guaranteed,
+  BridgedTypeAttrKind_in_constant,
+  BridgedTypeAttrKind_pack_owned,
+  BridgedTypeAttrKind_pack_guaranteed,
+  BridgedTypeAttrKind_pack_inout,
+  BridgedTypeAttrKind_pack_out,
+  BridgedTypeAttrKind_owned,
+  BridgedTypeAttrKind_unowned_inner_pointer,
+  BridgedTypeAttrKind_guaranteed,
+  BridgedTypeAttrKind_autoreleased,
+  BridgedTypeAttrKind_callee_owned,
+  BridgedTypeAttrKind_callee_guaranteed,
+  BridgedTypeAttrKind_objc_metatype,
+  BridgedTypeAttrKind_opened,
+  BridgedTypeAttrKind_pack_element,
+  BridgedTypeAttrKind_pseudogeneric,
+  BridgedTypeAttrKind_yields,
+  BridgedTypeAttrKind_yield_once,
+  BridgedTypeAttrKind_yield_many,
+  BridgedTypeAttrKind_captures_generics,
+  BridgedTypeAttrKind_moveOnly,
+  BridgedTypeAttrKind_thin,
+  BridgedTypeAttrKind_thick,
+  BridgedTypeAttrKind_Count
+} BridgedTypeAttrKind;
+
 #ifdef __cplusplus
 extern "C" {
 
@@ -261,6 +325,19 @@ void *ImplicitlyUnwrappedOptionalTypeRepr_create(void *ctx, void *base,
                                                  void *exclamationLoc);
 void *MetatypeTypeRepr_create(void *ctx, void *baseType, void *typeLoc);
 void *ProtocolTypeRepr_create(void *ctx, void *baseType, void *protoLoc);
+
+BridgedTypeAttrKind getBridgedTypeAttrKindFromString(
+    const unsigned char * _Nullable str, long len);
+
+typedef void *BridgedTypeAttributes;
+BridgedTypeAttributes BridgedTypeAttributes_create(void);
+void BridgedTypeAttributes_addSimpleAttr(
+    BridgedTypeAttributes typeAttributes, BridgedTypeAttrKind kind, void *atLoc, void *attrLoc);
+void *AttributedTypeRepr_create(void *ctx, void *base, BridgedTypeAttributes typeAttributes);
+
+void *AttributedTypeSpecifierRepr_create(
+    void *ctx, void *base, BridgedAttributedTypeSpecifier specifier, void *specifierLoc);
+void *VarargTypeRepr_create(void *ctx, void *base, void *ellipsisLocPtr);
 void *PackExpansionTypeRepr_create(void *ctx, void *base, void *repeatLoc);
 void *TupleTypeRepr_create(void *ctx, BridgedArrayRef elements, void *lParenLoc,
                            void *rParenLoc);
@@ -269,8 +346,9 @@ void *MemberTypeRepr_create(void *ctx, void *baseComponent,
 void *GenericIdentTypeRepr_create(void *ctx, BridgedIdentifier name,
                                   void *nameLoc, BridgedArrayRef genericArgs,
                                   void *lAngle, void *rAngle);
+void *EmptyCompositionTypeRepr_create(void *ctx, void *anyLoc);
 void *CompositionTypeRepr_create(void *ctx, BridgedArrayRef types,
-                                 void *firstTypeLoc);
+                                 void *firstTypeLoc, void *firstAmpLoc);
 void *FunctionTypeRepr_create(void *ctx, void *argsTy, void *_Nullable asyncLoc,
                               void *_Nullable throwsLoc, void *arrowLoc,
                               void *returnType);

include/swift/Basic/Features.def

@@ -179,6 +179,10 @@ EXPERIMENTAL_FEATURE(ImplicitSome, false)
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(ParserASTGen, false)
 
+/// Use the syntax tree produced by the Swift (swift-syntax) parser for type
+/// parsing, using ASTGen to translate them into AST nodes.
+EXPERIMENTAL_FEATURE(ASTGenTypes, false)
+
 /// Parse using the Swift (swift-syntax) parser and use ASTGen to generate the
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(BuiltinMacros, false)

include/swift/Parse/Parser.h

@@ -24,6 +24,7 @@
 #include "swift/AST/LayoutConstraint.h"
 #include "swift/AST/ParseRequests.h"
 #include "swift/AST/Pattern.h"
+#include "swift/AST/SourceFile.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Config.h"
@@ -1330,6 +1331,50 @@ class Parser {
   /// Get the location for a type error.
   SourceLoc getTypeErrorLoc() const;
 
+  /// Callback function used for creating a C++ AST from the syntax node at the given source location.
+  ///
+  /// The arguments to this callback are the source file to pass into ASTGen (the exported source file)
+  /// and the source location pointer to pass into ASTGen (to find the syntax node).
+  ///
+  /// The callback returns the new AST node and the ending location of the syntax node. If the AST node
+  /// is NULL, something went wrong.
+  template<typename T>
+  using ASTFromSyntaxTreeCallback = std::pair<T*, const void *>(
+      void *sourceFile, const void *sourceLoc
+  );
+
+  /// Parse by constructing a C++ AST node from the Swift syntax tree via ASTGen.
+  template<typename T>
+  ParserResult<T> parseASTFromSyntaxTree(
+      llvm::function_ref<ASTFromSyntaxTreeCallback<T>> body
+  ) {
+    if (!Context.LangOpts.hasFeature(Feature::ASTGenTypes))
+      return nullptr;
+
+    auto exportedSourceFile = SF.getExportedSourceFile();
+    if (!exportedSourceFile)
+      return nullptr;
+
+    // Perform the translation.
+    auto sourceLoc = Tok.getLoc().getOpaquePointerValue();
+    T* astNode;
+    const void *endLocPtr;
+    std::tie(astNode, endLocPtr) = body(exportedSourceFile, sourceLoc);
+
+    if (!astNode) {
+      assert(false && "Could not build AST node from syntax tree");
+      return nullptr;
+    }
+
+    // Reset the lexer to the ending location.
+    StringRef contents =
+        SourceMgr.extractText(SourceMgr.getRangeForBuffer(L->getBufferID()));
+    L->resetToOffset((const char *)endLocPtr - contents.data());
+    L->lex(Tok);
+
+    return makeParserResult(astNode);
+  }
+
   //===--------------------------------------------------------------------===//
   // Type Parsing
 

lib/AST/ASTPrinter.cpp

@@ -3321,6 +3321,10 @@ static bool usesFeatureParserASTGen(Decl *decl) {
   return false;
 }
 
+static bool usesFeatureASTGenTypes(Decl *decl) {
+  return false;
+}
+
 static bool usesFeatureBuiltinMacros(Decl *decl) {
   return false;
 }

lib/AST/CASTBridging.cpp

@@ -119,6 +119,15 @@ void SwiftDiagnostic_finish(BridgedDiagnostic diagPtr) {
 BridgedIdentifier
 SwiftASTContext_getIdentifier(void *ctx, const unsigned char *_Nullable str,
                               long len) {
+  if (len == 1 && str[0] == '_')
+    return BridgedIdentifier();
+
+  // If this was a back-ticked identifier, drop the back-ticks.
+  if (len >= 2 && str[0] == '`' && str[len-1] == '`') {
+    ++str;
+    len -= 2;
+  }
+
   return const_cast<void *>(
       static_cast<ASTContext *>(ctx)
           ->getIdentifier(
@@ -484,6 +493,87 @@ void *PackExpansionTypeRepr_create(void *ctx, void *base, void *repeatLoc) {
       getSourceLocFromPointer(repeatLoc), (TypeRepr *)base);
 }
 
+static BridgedTypeAttrKind bridgeTypeAttrKind(TypeAttrKind kind) {
+  switch (kind) {
+#define TYPE_ATTR(X) case TAK_##X: return BridgedTypeAttrKind_##X;
+#include "swift/AST/Attr.def"
+    case TAK_Count: return BridgedTypeAttrKind_Count;
+  }
+}
+
+static TypeAttrKind bridgeTypeAttrKind(BridgedTypeAttrKind kind) {
+  switch (kind) {
+#define TYPE_ATTR(X) case BridgedTypeAttrKind_##X: return TAK_##X;
+#include "swift/AST/Attr.def"
+    case BridgedTypeAttrKind_Count: return TAK_Count;
+  }
+}
+
+BridgedTypeAttrKind getBridgedTypeAttrKindFromString(
+    const unsigned char *str, intptr_t len) {
+  return bridgeTypeAttrKind(
+      TypeAttributes::getAttrKindFromString(StringRef((const char *)str, len)));
+}
+
+BridgedTypeAttributes BridgedTypeAttributes_create() {
+  return new TypeAttributes();
+}
+
+void BridgedTypeAttributes_addSimpleAttr(
+    BridgedTypeAttributes typeAttributesPtr, BridgedTypeAttrKind kind,
+    void *atLoc, void *attrLoc
+) {
+  TypeAttributes *typeAttributes = (TypeAttributes *)typeAttributesPtr;
+  typeAttributes->setAttr(
+      bridgeTypeAttrKind(kind), getSourceLocFromPointer(attrLoc));
+  if (typeAttributes->AtLoc.isInvalid())
+    typeAttributes->AtLoc = getSourceLocFromPointer(atLoc);
+}
+
+void *AttributedTypeRepr_create(
+    void *ctx, void *base, BridgedTypeAttributes typeAttributesPtr) {
+  TypeAttributes *typeAttributes = (TypeAttributes *)typeAttributesPtr;
+  if (typeAttributes->empty())
+    return base;
+
+  ASTContext &Context = *static_cast<ASTContext *>(ctx);
+  auto attributedType =
+    new (Context) AttributedTypeRepr(*typeAttributes, (TypeRepr *)base);
+  delete typeAttributes;
+  return attributedType;
+}
+
+void *AttributedTypeSpecifierRepr_create(
+    void *ctx, void *base, BridgedAttributedTypeSpecifier specifier, void *specifierLoc
+) {
+  ASTContext &Context = *static_cast<ASTContext *>(ctx);
+  SourceLoc loc = getSourceLocFromPointer(specifierLoc);
+  TypeRepr *baseType = (TypeRepr *)base;
+  switch (specifier) {
+  case BridgedAttributedTypeSpecifierInOut:
+    return new (Context) OwnershipTypeRepr(baseType, ParamSpecifier::InOut, loc);
+  case BridgedAttributedTypeSpecifierBorrowing:
+    return new (Context) OwnershipTypeRepr(baseType, ParamSpecifier::Borrowing, loc);
+  case BridgedAttributedTypeSpecifierConsuming:
+    return new (Context) OwnershipTypeRepr(baseType, ParamSpecifier::Consuming, loc);
+  case BridgedAttributedTypeSpecifierLegacyShared:
+    return new (Context) OwnershipTypeRepr(baseType, ParamSpecifier::LegacyShared, loc);
+  case BridgedAttributedTypeSpecifierLegacyOwned:
+    return new (Context) OwnershipTypeRepr(baseType, ParamSpecifier::LegacyOwned, loc);
+  case BridgedAttributedTypeSpecifierConst:
+    return new (Context) CompileTimeConstTypeRepr(baseType, loc);
+  case BridgedAttributedTypeSpecifierIsolated:
+    return new (Context) IsolatedTypeRepr(baseType, loc);
+  }
+}
+
+void *VarargTypeRepr_create(void *ctx, void *base, void *ellipsisLocPtr) {
+  ASTContext &Context = *static_cast<ASTContext *>(ctx);
+  SourceLoc ellipsisLoc = getSourceLocFromPointer(ellipsisLocPtr);
+  TypeRepr *baseType = (TypeRepr *)base;
+  return new (Context) VarargTypeRepr(baseType, ellipsisLoc);
+}
+
 void *TupleTypeRepr_create(void *ctx, BridgedArrayRef elements, void *lParenLoc,
                            void *rParenLoc) {
   ASTContext &Context = *static_cast<ASTContext *>(ctx);
@@ -518,12 +608,21 @@ void *MemberTypeRepr_create(void *ctx, void *baseComponent,
                                 memberComponents);
 }
 
-void *CompositionTypeRepr_create(void *ctx, BridgedArrayRef types,
-                                 void *firstTypeLoc) {
+void *EmptyCompositionTypeRepr_create(void *ctx, void *anyLocPtr) {
+  ASTContext &Context = *static_cast<ASTContext *>(ctx);
+  SourceLoc anyLoc = getSourceLocFromPointer(anyLocPtr);
+  return CompositionTypeRepr::createEmptyComposition(Context, anyLoc);
+}
+
+void *CompositionTypeRepr_create(void *ctx, BridgedArrayRef typesPtr,
+                                 void *firstTypeLoc, void *firstAmpLocPtr) {
   ASTContext &Context = *static_cast<ASTContext *>(ctx);
   SourceLoc firstType = getSourceLocFromPointer(firstTypeLoc);
-  return CompositionTypeRepr::create(Context, getArrayRef<TypeRepr *>(types),
-                                     firstType, SourceRange{});
+  SourceLoc firstAmpLoc = getSourceLocFromPointer(firstAmpLocPtr);
+  auto types = getArrayRef<TypeRepr *>(typesPtr);
+  return CompositionTypeRepr::create(
+      Context, types, firstType,
+      SourceRange{firstAmpLoc, types.back()->getEndLoc()});
 }
 
 void *FunctionTypeRepr_create(void *ctx, void *argsTy, void *_Nullable asyncLoc,

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -586,10 +586,11 @@ func expandFreestandingMacroInProcess(
 }
 
 /// Retrieve a syntax node in the given source file, with the given type.
-private func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
+func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
   sourceFilePtr: UnsafeRawPointer,
   sourceLocationPtr: UnsafePointer<UInt8>?,
-  type: Node.Type
+  type: Node.Type,
+  wantOutermost: Bool = false
 ) -> Node? {
   guard let sourceLocationPtr = sourceLocationPtr else {
     return nil
@@ -615,16 +616,45 @@ private func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
   }
 
   var currentSyntax = Syntax(token)
+  var resultSyntax: Node? = nil
   while let parentSyntax = currentSyntax.parent {
     if let typedParent = parentSyntax.as(type) {
-      return typedParent
+      resultSyntax = typedParent
+      break
     }
 
     currentSyntax = parentSyntax
   }
 
-  print("unable to find node: \(token.debugDescription)")
-  return nil
+  // If we didn't find anything, complain and fail.
+  guard var resultSyntax else {
+    print("unable to find node: \(token.debugDescription)")
+    return nil
+  }
+
+  // If we want the outermost node, keep looking.
+  // FIXME: This is VERY SPECIFIC to handling of types. We must be able to
+  // do better.
+  if wantOutermost {
+    while let parentSyntax = resultSyntax.parent {
+      // Look through type compositions.
+      if let compositionElement = parentSyntax.as(CompositionTypeElementSyntax.self),
+         let compositionList = compositionElement.parent?.as(CompositionTypeElementListSyntax.self),
+         let typedParent = compositionList.parent?.as(type) {
+        resultSyntax = typedParent
+        continue
+      }
+
+      guard let typedParent = parentSyntax.as(type),
+            typedParent.position == resultSyntax.position else {
+        break
+      }
+
+      resultSyntax = typedParent
+    }
+  }
+
+  return resultSyntax
 }
 
 @_cdecl("swift_ASTGen_expandAttachedMacro")