Skip to content

Commit 1a211e6

Browse files
rintarorintaro
authored
Merge pull request #26403 from rintaro/gsoc-2019-part1
Reapply: [Parser] Decouple the parser from AST creation (part 1)
2 parents 482d062 + 94f0c14 commit 1a211e6

24 files changed

+971
-318
lines changed

‎include/swift/Parse/ASTGen.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
//===--- ASTGen.h ---------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef SWIFT_PARSE_ASTGEN_H
14+
#define SWIFT_PARSE_ASTGEN_H
15+
16+
#include "swift/AST/ASTContext.h"
17+
#include "swift/AST/Expr.h"
18+
#include "swift/Syntax/SyntaxNodes.h"
19+
#include "llvm/ADT/DenseMap.h"
20+
21+
namespace swift {
22+
/// Generates AST nodes from Syntax nodes.
23+
class ASTGen {
24+
ASTContext &Context;
25+
// A stack of source locations of syntax constructs. Allows us to get the
26+
// SourceLoc necessary to create AST nodes for nodes in not-yet-complete
27+
// Syntax tree. The topmost item should always correspond to the token/node
28+
// that has been parsed/transformed most recently.
29+
// todo [gsoc]: remove when possible
30+
llvm::SmallVector<SourceLoc, 16> LocStack;
31+
32+
public:
33+
explicit ASTGen(ASTContext &Context) : Context(Context) {}
34+
35+
IntegerLiteralExpr *generate(syntax::IntegerLiteralExprSyntax &Expr);
36+
FloatLiteralExpr *generate(syntax::FloatLiteralExprSyntax &Expr);
37+
NilLiteralExpr *generate(syntax::NilLiteralExprSyntax &Expr);
38+
BooleanLiteralExpr *generate(syntax::BooleanLiteralExprSyntax &Expr);
39+
MagicIdentifierLiteralExpr *generate(syntax::PoundFileExprSyntax &Expr);
40+
MagicIdentifierLiteralExpr *generate(syntax::PoundLineExprSyntax &Expr);
41+
MagicIdentifierLiteralExpr *generate(syntax::PoundColumnExprSyntax &Expr);
42+
MagicIdentifierLiteralExpr *generate(syntax::PoundFunctionExprSyntax &Expr);
43+
MagicIdentifierLiteralExpr *generate(syntax::PoundDsohandleExprSyntax &Expr);
44+
Expr *generate(syntax::UnknownExprSyntax &Expr);
45+
46+
/// Stores source location necessary for AST creation.
47+
void pushLoc(SourceLoc Loc);
48+
49+
/// Copy a numeric literal value into AST-owned memory, stripping underscores
50+
/// so the semantic part of the value can be parsed by APInt/APFloat parsers.
51+
static StringRef copyAndStripUnderscores(StringRef Orig, ASTContext &Context);
52+
53+
private:
54+
StringRef copyAndStripUnderscores(StringRef Orig);
55+
56+
SourceLoc topLoc();
57+
58+
MagicIdentifierLiteralExpr *
59+
generateMagicIdentifierLiteralExpr(const syntax::TokenSyntax &PoundToken);
60+
61+
/// Map magic literal tokens such as #file to their MagicIdentifierLiteralExpr
62+
/// kind.
63+
static MagicIdentifierLiteralExpr::Kind
64+
getMagicIdentifierLiteralKind(tok Kind);
65+
};
66+
} // namespace swift
67+
68+
#endif // SWIFT_PARSE_ASTGEN_H

‎include/swift/Parse/HiddenLibSyntaxAction.h‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
//===--- HiddenLibSyntaxAction.h ------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
14+
#define SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
15+
16+
#include "swift/Parse/SyntaxParseActions.h"
17+
#include "swift/SyntaxParse/SyntaxTreeCreator.h"
18+
#include "llvm/Support/Allocator.h"
19+
20+
namespace swift {
21+
namespace syntax {
22+
class RawSyntax;
23+
}
24+
25+
/// Holds an explicitly provided action and uses it to handle all function
26+
/// calls. Also hides an implicit SyntaxTreeCreator and ensures libSyntax nodes
27+
/// are always created. Provides an interface to map results of the explicitly
28+
/// provided action to the hidden libSyntax action.
29+
// todo [gsoc]: remove when possible
30+
class HiddenLibSyntaxAction : public SyntaxParseActions {
31+
32+
struct Node {
33+
OpaqueSyntaxNode ExplicitActionNode;
34+
OpaqueSyntaxNode LibSyntaxNode;
35+
36+
Node(OpaqueSyntaxNode ExplicitActionNode, OpaqueSyntaxNode LibSyntaxNode)
37+
: ExplicitActionNode(ExplicitActionNode), LibSyntaxNode(LibSyntaxNode) {
38+
}
39+
};
40+
41+
std::shared_ptr<SyntaxParseActions> ExplicitAction;
42+
std::shared_ptr<SyntaxTreeCreator> LibSyntaxAction;
43+
llvm::SpecificBumpPtrAllocator<Node> NodeAllocator;
44+
45+
bool areBothLibSyntax() {
46+
return ExplicitAction->getOpaqueKind() == OpaqueSyntaxNodeKind::LibSyntax;
47+
}
48+
49+
OpaqueSyntaxNode makeHiddenNode(OpaqueSyntaxNode explicitActionNode,
50+
OpaqueSyntaxNode libSyntaxNode);
51+
52+
public:
53+
HiddenLibSyntaxAction(
54+
const std::shared_ptr<SyntaxParseActions> &SPActions,
55+
const std::shared_ptr<SyntaxTreeCreator> &LibSyntaxAction)
56+
: ExplicitAction(SPActions != nullptr ? SPActions : LibSyntaxAction),
57+
LibSyntaxAction(LibSyntaxAction){};
58+
59+
OpaqueSyntaxNode recordToken(tok tokenKind,
60+
ArrayRef<ParsedTriviaPiece> leadingTrivia,
61+
ArrayRef<ParsedTriviaPiece> trailingTrivia,
62+
CharSourceRange range) override;
63+
64+
OpaqueSyntaxNode recordMissingToken(tok tokenKind, SourceLoc loc) override;
65+
66+
OpaqueSyntaxNode recordRawSyntax(syntax::SyntaxKind kind,
67+
ArrayRef<OpaqueSyntaxNode> elements,
68+
CharSourceRange range) override;
69+
70+
std::pair<size_t, OpaqueSyntaxNode>
71+
lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) override;
72+
73+
OpaqueSyntaxNodeKind getOpaqueKind() override {
74+
return ExplicitAction->getOpaqueKind();
75+
}
76+
77+
/// Returns the libSyntax node from the specified node that has been created
78+
/// by this action.
79+
syntax::RawSyntax *getLibSyntaxNodeFor(OpaqueSyntaxNode node);
80+
81+
/// Returns the node created by explicit syntax action from the specified
82+
/// node that has been created by this action.
83+
OpaqueSyntaxNode getExplicitNodeFor(OpaqueSyntaxNode node);
84+
85+
bool isReleaseNeeded() {
86+
return ExplicitAction == LibSyntaxAction || !areBothLibSyntax();
87+
}
88+
89+
/// Returns the underlying libSyntax SyntaxTreeCreator.
90+
std::shared_ptr<SyntaxTreeCreator> getLibSyntaxAction() {
91+
return LibSyntaxAction;
92+
}
93+
};
94+
} // namespace swift
95+
96+
#endif // SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H

‎include/swift/Parse/LibSyntaxGenerator.h‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
//===----------- LibSyntaxGenerator.h -------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef SWIFT_PARSE_LIBSYNTAXGENERATOR_H
14+
#define SWIFT_PARSE_LIBSYNTAXGENERATOR_H
15+
16+
#include "swift/Parse/HiddenLibSyntaxAction.h"
17+
#include "swift/Parse/ParsedRawSyntaxNode.h"
18+
#include "swift/Parse/ParsedRawSyntaxRecorder.h"
19+
#include "swift/Parse/SyntaxParsingCache.h"
20+
#include "swift/Syntax/RawSyntax.h"
21+
#include "swift/SyntaxParse/SyntaxTreeCreator.h"
22+
23+
namespace swift {
24+
/// Generates libSyntax nodes either by looking them up using
25+
/// HiddenLibSyntaxAction (based on provided OpaqueSyntaxNode) or by recording
26+
/// them with ParsedRawSyntaxRecorder.
27+
// todo [gsoc]: remove when possible
28+
class LibSyntaxGenerator {
29+
std::shared_ptr<HiddenLibSyntaxAction> Actions;
30+
ParsedRawSyntaxRecorder Recorder;
31+
32+
public:
33+
explicit LibSyntaxGenerator(std::shared_ptr<HiddenLibSyntaxAction> TwoActions)
34+
: Actions(std::move(TwoActions)),
35+
Recorder(Actions->getLibSyntaxAction()) {}
36+
37+
TokenSyntax createToken(ParsedRawSyntaxNode Node) {
38+
assert(Node.isDeferredToken());
39+
40+
auto Kind = Node.getTokenKind();
41+
auto Range = Node.getDeferredTokenRangeWithTrivia();
42+
auto LeadingTriviaPieces = Node.getDeferredLeadingTriviaPieces();
43+
auto TrailingTriviaPieces = Node.getDeferredTrailingTriviaPieces();
44+
45+
auto Recorded = Recorder.recordToken(Kind, Range, LeadingTriviaPieces,
46+
TrailingTriviaPieces);
47+
auto Raw = static_cast<RawSyntax *>(Recorded.getOpaqueNode());
48+
return make<TokenSyntax>(Raw);
49+
}
50+
51+
template <typename SyntaxNode>
52+
SyntaxNode createNode(ParsedRawSyntaxNode Node) {
53+
assert(Node.isDeferredLayout());
54+
auto Kind = Node.getKind();
55+
auto Children = Node.getDeferredChildren();
56+
57+
auto Recorded = Recorder.recordRawSyntax(Kind, Children);
58+
RC<RawSyntax> Raw {static_cast<RawSyntax *>(Recorded.getOpaqueNode()) };
59+
Raw->Release(); // -1 since it's transfer of ownership.
60+
return make<SyntaxNode>(Raw);
61+
}
62+
63+
TokenSyntax getLibSyntaxTokenFor(OpaqueSyntaxNode Node) {
64+
return getLibSyntaxNodeFor<TokenSyntax>(Node);
65+
}
66+
67+
template <typename SyntaxNode>
68+
SyntaxNode getLibSyntaxNodeFor(OpaqueSyntaxNode Node) {
69+
return make<SyntaxNode>(Actions->getLibSyntaxNodeFor(Node));
70+
}
71+
72+
OpaqueSyntaxNode finalizeNode(OpaqueSyntaxNode Node) {
73+
if (Actions->isReleaseNeeded())
74+
Actions->getLibSyntaxNodeFor(Node)->Release();
75+
return Actions->getExplicitNodeFor(Node);
76+
}
77+
};
78+
} // namespace swift
79+
80+
#endif // SWIFT_PARSE_LIBSYNTAXGENERATOR_H

‎include/swift/Parse/ParsedRawSyntaxNode.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,19 @@ class ParsedRawSyntaxNode {
156156

157157
// Deferred Token Data =====================================================//
158158

159+
CharSourceRange getDeferredTokenRangeWithTrivia() const {
160+
assert(DK == DataKind::DeferredToken);
161+
auto leadTriviaPieces = getDeferredLeadingTriviaPieces();
162+
auto trailTriviaPieces = getDeferredTrailingTriviaPieces();
163+
164+
auto leadTriviaLen = ParsedTriviaPiece::getTotalLength(leadTriviaPieces);
165+
auto trailTriviaLen = ParsedTriviaPiece::getTotalLength(trailTriviaPieces);
166+
167+
SourceLoc begin = DeferredToken.TokLoc.getAdvancedLoc(-leadTriviaLen);
168+
unsigned len = leadTriviaLen + DeferredToken.TokLength + trailTriviaLen;
169+
170+
return CharSourceRange{begin, len};
171+
}
159172
CharSourceRange getDeferredTokenRangeWithoutBackticks() const {
160173
assert(DK == DataKind::DeferredToken);
161174
return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};

‎include/swift/Parse/ParsedSyntaxRecorder.h.gyb‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,11 +66,31 @@ public:
6666

6767
static Parsed${node.name} makeBlank${node.syntax_kind}(SourceLoc loc,
6868
SyntaxParsingContext &SPCtx);
69+
% elif node.is_unknown():
70+
private:
71+
static Parsed${node.name} record${node.syntax_kind}(
72+
ArrayRef<ParsedSyntax> elts,
73+
ParsedRawSyntaxRecorder &rec);
74+
75+
public:
76+
static Parsed${node.name} defer${node.syntax_kind}(
77+
ArrayRef<ParsedSyntax> elts,
78+
SyntaxParsingContext &SPCtx);
79+
80+
static Parsed${node.name} make${node.syntax_kind}(
81+
ArrayRef<ParsedSyntax> elts,
82+
SyntaxParsingContext &SPCtx);
6983
% end
7084
% end
7185

7286
#pragma mark - Convenience APIs
7387

88+
public:
89+
static ParsedTokenSyntax makeToken(const Token &Tok,
90+
const ParsedTrivia &LeadingTrivia,
91+
const ParsedTrivia &TrailingTrivia,
92+
SyntaxParsingContext &SPCtx);
93+
7494
/// Records an unlabelled TupleTypeElementSyntax with the provided type and
7595
/// optional trailing comma.
7696
static ParsedTupleTypeElementSyntax

‎include/swift/Parse/Parser.h‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@
2525
#include "swift/AST/Pattern.h"
2626
#include "swift/AST/Stmt.h"
2727
#include "swift/Basic/OptionSet.h"
28+
#include "swift/Parse/ASTGen.h"
2829
#include "swift/Parse/Lexer.h"
2930
#include "swift/Parse/LocalContext.h"
3031
#include "swift/Parse/PersistentParserState.h"
3132
#include "swift/Parse/Token.h"
33+
#include "swift/Parse/ParsedSyntaxNodes.h"
3234
#include "swift/Parse/ParserPosition.h"
3335
#include "swift/Parse/ParserResult.h"
3436
#include "swift/Parse/SyntaxParserResult.h"
@@ -376,6 +378,9 @@ class Parser {
376378
/// Current syntax parsing context where call backs should be directed to.
377379
SyntaxParsingContext *SyntaxContext;
378380

381+
/// The AST generator.
382+
ASTGen Generator;
383+
379384
public:
380385
Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
381386
SILParserTUStateBase *SIL,
@@ -397,7 +402,7 @@ class Parser {
397402

398403
/// Calling this function to finalize libSyntax tree creation without destroying
399404
/// the parser instance.
400-
ParsedRawSyntaxNode finalizeSyntaxTree() {
405+
OpaqueSyntaxNode finalizeSyntaxTree() {
401406
assert(Tok.is(tok::eof) && "not done parsing yet");
402407
return SyntaxContext->finalizeRoot();
403408
}
@@ -518,6 +523,12 @@ class Parser {
518523
assert(Tok.is(K) && "Consuming wrong token kind");
519524
return consumeToken();
520525
}
526+
/// Consume a token without providing it to the SyntaxParsingContext.
527+
ParsedTokenSyntax consumeTokenSyntax();
528+
ParsedTokenSyntax consumeTokenSyntax(tok K) {
529+
assert(Tok.is(K) && "Consuming wrong token kind");
530+
return consumeTokenSyntax();
531+
}
521532

522533
SourceLoc consumeIdentifier(Identifier *Result = nullptr,
523534
bool allowDollarIdentifier = false) {
@@ -1324,6 +1335,15 @@ class Parser {
13241335
ParserResult<Expr> parseExprSuper();
13251336
ParserResult<Expr> parseExprStringLiteral();
13261337

1338+
// todo [gsoc]: create new result type for ParsedSyntax
1339+
// todo [gsoc]: turn into proper non-templated methods later
1340+
template <typename SyntaxNode>
1341+
ParsedExprSyntax parseExprSyntax();
1342+
1343+
// todo [gsoc]: remove when possible
1344+
template <typename SyntaxNode>
1345+
ParserResult<Expr> parseExprAST();
1346+
13271347
StringRef copyAndStripUnderscores(StringRef text);
13281348

13291349
ParserStatus parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,

‎include/swift/Parse/SyntaxParseActions.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ namespace syntax {
3333

3434
typedef void *OpaqueSyntaxNode;
3535

36+
// todo [gsoc]: remove when possible
37+
enum class OpaqueSyntaxNodeKind {
38+
SwiftSyntax,
39+
LibSyntax,
40+
};
41+
3642
class SyntaxParseActions {
3743
virtual void _anchor();
3844

@@ -60,6 +66,9 @@ class SyntaxParseActions {
6066
lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) {
6167
return std::make_pair(0, nullptr);
6268
}
69+
70+
/// Returns what kind of OpaqueSyntaxNode is created by `recordXXX` methods.
71+
virtual OpaqueSyntaxNodeKind getOpaqueKind() = 0;
6372
};
6473

6574
} // end namespace swift

0 commit comments

Comments
 (0)