diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index f23075043387e..af0e49f7faa83 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -791,6 +791,10 @@ class ASTContext final { /// \param IDC The context whose member decls should be lazily parsed. void parseMembers(IterableDeclContext *IDC); + /// Use the lazy parsers associated with the context to check whether the decl + /// context has been parsed. + bool hasUnparsedMembers(const IterableDeclContext *IDC) const; + /// Get the lazy function data for the given generic context. /// /// \param lazyLoader If non-null, the lazy loader to use when creating the diff --git a/include/swift/AST/LazyResolver.h b/include/swift/AST/LazyResolver.h index 987f6ae1dc73e..a600fdf9135ce 100644 --- a/include/swift/AST/LazyResolver.h +++ b/include/swift/AST/LazyResolver.h @@ -101,6 +101,12 @@ class LazyMemberParser { /// /// The implementation should add the members to IDC. virtual void parseMembers(IterableDeclContext *IDC) = 0; + + /// Return whether the iterable decl context needs parsing. + virtual bool hasUnparsedMembers(const IterableDeclContext *IDC) = 0; + + /// Parse all delayed decl list members. + virtual void parseAllDelayedDeclLists() = 0; }; /// Context data for generic contexts. diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index e4fa2a5dc3d0f..97622f59b331b 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -552,7 +552,8 @@ class CompilerInstance { /// Parses the input file but does no type-checking or module imports. /// Note that this only supports parsing an invocation with a single file. - void performParseOnly(bool EvaluateConditionals = false); + void performParseOnly(bool EvaluateConditionals = false, + bool ParseDelayedBodyOnEnd = false); /// Parses and performs name binding on all input files. /// diff --git a/include/swift/Parse/Parser.h b/include/swift/Parse/Parser.h index c74b7fa73a440..0bffd70888ec0 100644 --- a/include/swift/Parse/Parser.h +++ b/include/swift/Parse/Parser.h @@ -346,6 +346,9 @@ class Parser { SyntaxParsingContext *SyntaxContext; public: + Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags, + SILParserTUStateBase *SIL, + PersistentParserState *PersistentState); Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL, PersistentParserState *PersistentState = nullptr); Parser(std::unique_ptr Lex, SourceFile &SF, diff --git a/include/swift/Parse/PersistentParserState.h b/include/swift/Parse/PersistentParserState.h index cc436361dc25f..7c6d52a456666 100644 --- a/include/swift/Parse/PersistentParserState.h +++ b/include/swift/Parse/PersistentParserState.h @@ -174,10 +174,12 @@ class PersistentParserState: public LazyMemberParser { std::unique_ptr takeDelayedDeclListState(IterableDeclContext *IDC); - bool hasDelayedDeclList(IterableDeclContext *IDC) { + bool hasUnparsedMembers(const IterableDeclContext *IDC) override { return DelayedDeclListStates.find(IDC) != DelayedDeclListStates.end(); } + void parseAllDelayedDeclLists() override; + TopLevelContext &getTopLevelContext() { return TopLevelCode; } diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 14527661a28ea..10842063c5acf 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -573,6 +573,7 @@ void ASTContext::addLazyParser(LazyMemberParser *lazyParser) { void ASTContext::removeLazyParser(LazyMemberParser *lazyParser) { auto removed = getImpl().lazyParsers.erase(lazyParser); + (void)removed; assert(removed && "Removing an non-existing lazy parser."); } @@ -1902,9 +1903,16 @@ LazyContextData *ASTContext::getOrCreateLazyContextData( return contextData; } +bool ASTContext::hasUnparsedMembers(const IterableDeclContext *IDC) const { + auto parsers = getImpl().lazyParsers; + return std::any_of(parsers.begin(), parsers.end(), + [IDC](LazyMemberParser *p) { return p->hasUnparsedMembers(IDC); }); +} + void ASTContext::parseMembers(IterableDeclContext *IDC) { for (auto *p: getImpl().lazyParsers) { - p->parseMembers(IDC); + if (p->hasUnparsedMembers(IDC)) + p->parseMembers(IDC); } } diff --git a/lib/AST/ASTVerifier.cpp b/lib/AST/ASTVerifier.cpp index 7eeaf89e15dd0..e4566a120a3c4 100644 --- a/lib/AST/ASTVerifier.cpp +++ b/lib/AST/ASTVerifier.cpp @@ -709,6 +709,8 @@ class Verifier : public ASTWalker { pushScope(fn); \ if (fn->hasLazyMembers()) \ return false; \ + if (fn->getASTContext().hasUnparsedMembers(fn)) \ + return false; \ return shouldVerify(cast::BaseTy>(fn));\ } \ void cleanup(NODE *fn) { \ diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 8fe3b3643f3d7..35a117b534172 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -925,7 +925,8 @@ SourceFile *CompilerInstance::createSourceFileForMainModule( return inputFile; } -void CompilerInstance::performParseOnly(bool EvaluateConditionals) { +void CompilerInstance::performParseOnly(bool EvaluateConditionals, + bool ParseDelayedBodyOnEnd) { const InputFileKind Kind = Invocation.getInputKind(); ModuleDecl *const MainModule = getMainModule(); Context->LoadedModules[MainModule->getName()] = MainModule; @@ -947,6 +948,10 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals) { } PersistentParserState PersistentState(getASTContext()); + SWIFT_DEFER { + if (ParseDelayedBodyOnEnd) + PersistentState.parseAllDelayedDeclLists(); + }; PersistentState.PerformConditionEvaluation = EvaluateConditionals; // Parse all the library files. for (auto BufferID : InputSourceCodeBufferIDs) { diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index fae5ca0043be3..bb455654ead12 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -902,8 +902,12 @@ static bool performCompile(CompilerInstance &Instance, return compileLLVMIR(Invocation, Instance, Stats); if (FrontendOptions::shouldActionOnlyParse(Action)) { + bool ParseDelayedDeclListsOnEnd = + Action == FrontendOptions::ActionType::DumpParse || + Invocation.getDiagnosticOptions().VerifyMode != DiagnosticOptions::NoVerify; Instance.performParseOnly(/*EvaluateConditionals*/ - Action == FrontendOptions::ActionType::EmitImportedModules); + Action == FrontendOptions::ActionType::EmitImportedModules, + ParseDelayedDeclListsOnEnd); } else if (Action == FrontendOptions::ActionType::ResolveImports) { Instance.performParseAndResolveImportsOnly(); } else { diff --git a/lib/Parse/ParseDecl.cpp b/lib/Parse/ParseDecl.cpp index 3c904dcf4efcf..5d31927994253 100644 --- a/lib/Parse/ParseDecl.cpp +++ b/lib/Parse/ParseDecl.cpp @@ -181,18 +181,21 @@ namespace { } // end anonymous namespace void PersistentParserState::parseMembers(IterableDeclContext *IDC) { - if (!hasDelayedDeclList(IDC)) - return; SourceFile &SF = *IDC->getDecl()->getDeclContext()->getParentSourceFile(); assert(!SF.hasInterfaceHash() && "Cannot delay parsing if we care about the interface hash."); + assert(SF.Kind != SourceFileKind::SIL && "cannot delay parsing SIL"); unsigned BufferID = *SF.getBufferID(); + // MarkedPos is not useful for delayed parsing because we know where we should // jump the parser to. However, we should recover the MarkedPos here in case // the PersistentParserState will be used to continuously parse the rest of // the file linearly. llvm::SaveAndRestore Pos(MarkedPos, ParserPosition()); - Parser TheParser(BufferID, SF, nullptr, this); + + // Lexer diaganostics have been emitted during skipping, so we disable lexer's + // diagnostic engine here. + Parser TheParser(BufferID, SF, /*No Lexer Diags*/nullptr, nullptr, this); // Disable libSyntax creation in the delayed parsing. TheParser.SyntaxContext->disable(); TheParser.parseDeclListDelayed(IDC); @@ -2292,7 +2295,8 @@ static unsigned skipUntilMatchingRBrace(Parser &P, bool &HasPoundDirective, SyntaxParsingContext BodyContext(SyntaxContext, SyntaxKind::TokenList); unsigned OpenBraces = 1; while (OpenBraces != 0 && P.Tok.isNot(tok::eof)) { - HasPoundDirective |= P.Tok.isAny(tok::pound_sourceLocation, tok::pound_line); + HasPoundDirective |= P.Tok.isAny(tok::pound_sourceLocation, tok::pound_line, + tok::pound_if, tok::pound_else, tok::pound_endif, tok::pound_elseif); if (P.consumeIf(tok::l_brace)) { OpenBraces++; continue; @@ -2902,6 +2906,15 @@ void Parser::parseDeclListDelayed(IterableDeclContext *IDC) { ParseDeclOptions(DelayedState->Flags), [&] (Decl *D) { ext->addMember(D); }); ext->setBraces({LBLoc, RBLoc}); + } else if (auto *cd = dyn_cast(D)) { + auto handler = [&] (Decl *D) { + cd->addMember(D); + if (isa(D)) + cd->setHasDestructor(); + }; + parseDeclList(cd->getBraces().Start, RBLoc, Id, + ParseDeclOptions(DelayedState->Flags), handler); + cd->setBraces({LBLoc, RBLoc}); } else { auto *ntd = cast(D); parseDeclList(ntd->getBraces().Start, RBLoc, Id, @@ -3349,6 +3362,10 @@ bool Parser::parseDeclList(SourceLoc LBLoc, SourceLoc &RBLoc, } bool Parser::canDelayMemberDeclParsing() { + // There's no fundamental reasons that SIL cannnot be lasily parsed. We need + // to keep SILParserTUStateBase persistent to make it happen. + if (isInSILMode()) + return false; // Calculating interface hash requires tokens consumed in the original order. if (SF.hasInterfaceHash()) return false; @@ -5831,6 +5848,7 @@ ParserResult Parser::parseDeclStruct(ParseDeclOptions Flags, // Make the entities of the struct as a code block. SyntaxParsingContext BlockContext(SyntaxContext, SyntaxKind::MemberDeclBlock); SourceLoc LBLoc, RBLoc; + SourceLoc PosBeforeLB = Tok.getLoc(); if (parseToken(tok::l_brace, LBLoc, diag::expected_lbrace_struct)) { LBLoc = PreviousLoc; RBLoc = LBLoc; @@ -5839,9 +5857,20 @@ ParserResult Parser::parseDeclStruct(ParseDeclOptions Flags, // Parse the body. Scope S(this, ScopeKind::StructBody); ParseDeclOptions Options(PD_HasContainerType | PD_InStruct); - if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_struct, - Options, [&](Decl *D) {SD->addMember(D);})) - Status.setIsParseError(); + if (canDelayMemberDeclParsing()) { + if (Tok.is(tok::r_brace)) { + RBLoc = consumeToken(); + } else { + RBLoc = Tok.getLoc(); + Status.setIsParseError(); + } + State->delayDeclList(SD, Options.toRaw(), CurDeclContext, { LBLoc, RBLoc }, + PosBeforeLB); + } else { + if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_struct, + Options, [&](Decl *D) {SD->addMember(D);})) + Status.setIsParseError(); + } } SD->setBraces({LBLoc, RBLoc}); @@ -5942,6 +5971,7 @@ ParserResult Parser::parseDeclClass(ParseDeclOptions Flags, SyntaxParsingContext BlockContext(SyntaxContext, SyntaxKind::MemberDeclBlock); SourceLoc LBLoc, RBLoc; + auto PosBeforeLB = Tok.getLoc(); if (parseToken(tok::l_brace, LBLoc, diag::expected_lbrace_class)) { LBLoc = PreviousLoc; RBLoc = LBLoc; @@ -5951,14 +5981,25 @@ ParserResult Parser::parseDeclClass(ParseDeclOptions Flags, Scope S(this, ScopeKind::ClassBody); ParseDeclOptions Options(PD_HasContainerType | PD_AllowDestructor | PD_InClass); - auto Handler = [&] (Decl *D) { - CD->addMember(D); - if (isa(D)) - CD->setHasDestructor(); - }; - if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_class, - Options, Handler)) - Status.setIsParseError(); + if (canDelayMemberDeclParsing()) { + if (Tok.is(tok::r_brace)) { + RBLoc = consumeToken(); + } else { + RBLoc = Tok.getLoc(); + Status.setIsParseError(); + } + State->delayDeclList(CD, Options.toRaw(), CurDeclContext, { LBLoc, RBLoc }, + PosBeforeLB); + } else { + auto Handler = [&] (Decl *D) { + CD->addMember(D); + if (isa(D)) + CD->setHasDestructor(); + }; + if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_class, + Options, Handler)) + Status.setIsParseError(); + } } CD->setBraces({LBLoc, RBLoc}); @@ -6040,6 +6081,7 @@ parseDeclProtocol(ParseDeclOptions Flags, DeclAttributes &Attributes) { SyntaxParsingContext BlockContext(SyntaxContext, SyntaxKind::MemberDeclBlock); SourceLoc LBraceLoc; SourceLoc RBraceLoc; + SourceLoc PosBeforeLB = Tok.getLoc(); if (parseToken(tok::l_brace, LBraceLoc, diag::expected_lbrace_protocol)) { LBraceLoc = PreviousLoc; RBraceLoc = LBraceLoc; @@ -6049,9 +6091,21 @@ parseDeclProtocol(ParseDeclOptions Flags, DeclAttributes &Attributes) { ParseDeclOptions Options(PD_HasContainerType | PD_DisallowInit | PD_InProtocol); - if (parseDeclList(LBraceLoc, RBraceLoc, diag::expected_rbrace_protocol, - Options, [&](Decl *D) {Proto->addMember(D);})) - Status.setIsParseError(); + if (canDelayMemberDeclParsing()) { + if (Tok.is(tok::r_brace)) { + RBraceLoc = consumeToken(); + } else { + RBraceLoc = Tok.getLoc(); + Status.setIsParseError(); + } + State->delayDeclList(Proto, Options.toRaw(), CurDeclContext, + { LBraceLoc, RBraceLoc }, + PosBeforeLB); + } else { + if (parseDeclList(LBraceLoc, RBraceLoc, diag::expected_rbrace_protocol, + Options, [&](Decl *D) {Proto->addMember(D);})) + Status.setIsParseError(); + } } // Install the protocol elements. diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp index 9200f3f00c53a..e0fd9200c1de6 100644 --- a/lib/Parse/Parser.cpp +++ b/lib/Parse/Parser.cpp @@ -331,12 +331,18 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM, // Setup and Helper Methods //===----------------------------------------------------------------------===// + Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL, PersistentParserState *PersistentState) + : Parser(BufferID, SF, &SF.getASTContext().Diags, SIL, PersistentState) {} + +Parser::Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags, + SILParserTUStateBase *SIL, + PersistentParserState *PersistentState) : Parser( std::unique_ptr(new Lexer( SF.getASTContext().LangOpts, SF.getASTContext().SourceMgr, - BufferID, &SF.getASTContext().Diags, + BufferID, LexerDiags, /*InSILMode=*/SIL != nullptr, SF.Kind == SourceFileKind::Main ? HashbangMode::Allowed diff --git a/lib/Parse/PersistentParserState.cpp b/lib/Parse/PersistentParserState.cpp index a6b2f5b4e5967..384d0584b087d 100644 --- a/lib/Parse/PersistentParserState.cpp +++ b/lib/Parse/PersistentParserState.cpp @@ -81,6 +81,16 @@ void PersistentParserState::delayDeclList(IterableDeclContext* D, ParentContext, BodyRange, PreviousLoc, ScopeInfo.saveCurrentScope()); } +void PersistentParserState::parseAllDelayedDeclLists() { + std::vector AllDelayed; + for (auto &P: DelayedDeclListStates) { + AllDelayed.push_back(P.first); + } + for (auto *D: AllDelayed) { + parseMembers(D); + } +} + void PersistentParserState::delayTopLevel(TopLevelCodeDecl *TLCD, SourceRange BodyRange, SourceLoc PreviousLoc) { diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp index bad0d2928fe7d..adb6f2dbea661 100644 --- a/tools/swift-ide-test/swift-ide-test.cpp +++ b/tools/swift-ide-test/swift-ide-test.cpp @@ -1476,7 +1476,7 @@ static int doInputCompletenessTest(StringRef SourceFilename) { llvm::raw_ostream &OS = llvm::outs(); OS << SourceFilename << ": "; if (isSourceInputComplete(std::move(FileBufOrErr.get()), - SourceFileKind::Main).IsComplete) { + SourceFileKind::REPL).IsComplete) { OS << "IS_COMPLETE\n"; } else { OS << "IS_INCOMPLETE\n"; diff --git a/utils/scale-test b/utils/scale-test index fd290a083d89e..7388f14095f89 100755 --- a/utils/scale-test +++ b/utils/scale-test @@ -109,6 +109,8 @@ def run_once_with_primary(args, ast, rng, primary_idx): mode = "-c" if args.typecheck: mode = "-typecheck" + if args.parse: + mode = "-parse" focus = ["-primary-file", primary] if args.whole_module_optimization: @@ -716,6 +718,9 @@ def main(): '--exponential-threshold', type=float, default=1.2, help='minimum base for exponential fit to consider "bad scaling"') + parser.add_argument( + '-parse', '--parse', action='store_true', + default=False, help='only run compiler with -parse') parser.add_argument( '-typecheck', '--typecheck', action='store_true', default=False, help='only run compiler with -typecheck') diff --git a/validation-test/compiler_scale/nominal_bodies.gyb b/validation-test/compiler_scale/nominal_bodies.gyb new file mode 100644 index 0000000000000..371207659e506 --- /dev/null +++ b/validation-test/compiler_scale/nominal_bodies.gyb @@ -0,0 +1,9 @@ +// RUN: %scale-test --sum-multi --parse --begin 5 --end 16 --step 5 --select NumIterableDeclContextParsed %s +// REQUIRES: OS=macosx +// REQUIRES: asserts + +struct S${N} {} +class C${N} {} +enum E${N} {} +extension C${N} {} +protocol P${N} {}