From db02c218f442a251639bdca22c5db38e14b55f0e Mon Sep 17 00:00:00 2001 From: dcode Date: Thu, 12 Jan 2023 14:44:11 +0100 Subject: [PATCH 1/2] Maintain onNewLine state on subsequent lookahead --- src/tokenizer.ts | 18 +++++++++--------- tests/parser/asi.ts | 8 +++++++- tests/parser/asi.ts.fixture.ts | 4 ++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 767ee03489..4fef11b283 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -973,19 +973,19 @@ export class Tokenizer extends DiagnosticEmitter { while (nextToken == Token.Invalid); this.nextToken = nextToken; this.nextTokenPos = this.tokenPos; - if (checkOnNewLine) { - this.nextTokenOnNewLine = false; - for (let pos = posBefore, end = this.nextTokenPos; pos < end; ++pos) { - if (isLineBreak(text.charCodeAt(pos))) { - this.nextTokenOnNewLine = true; - break; - } - } - } this.pos = posBefore; this.token = tokenBefore; this.tokenPos = tokenPosBefore; } + if (checkOnNewLine) { + this.nextTokenOnNewLine = false; + for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) { + if (isLineBreak(text.charCodeAt(pos))) { + this.nextTokenOnNewLine = true; + break; + } + } + } return this.nextToken; } diff --git a/tests/parser/asi.ts b/tests/parser/asi.ts index 542d063520..cb15d11762 100644 --- a/tests/parser/asi.ts +++ b/tests/parser/asi.ts @@ -16,4 +16,10 @@ function successCloseBrace(): i32 { function successCloseParen(): i32 { return ( 123 ) -} \ No newline at end of file +} + +function successAfterLet(): i32 { + // multiple tn.peeks + let a = 0 + return a +} diff --git a/tests/parser/asi.ts.fixture.ts b/tests/parser/asi.ts.fixture.ts index d6d8e567b8..8db95c3eaf 100644 --- a/tests/parser/asi.ts.fixture.ts +++ b/tests/parser/asi.ts.fixture.ts @@ -17,6 +17,10 @@ function successCloseBrace(): i32 { function successCloseParen(): i32 { return (123); } +function successAfterLet(): i32 { + let a = 0; + return a; +} // ERROR 1012: "Unexpected token." in asi.ts(2,13+0) // ERROR 1012: "Unexpected token." in asi.ts(7,14+0) // ERROR 1012: "Unexpected token." in asi.ts(11,13+0) From 9c0d38860cac9fd1fffcb7af48b8152766169ff3 Mon Sep 17 00:00:00 2001 From: dcode Date: Thu, 12 Jan 2023 15:43:03 +0100 Subject: [PATCH 2/2] refactor and cache --- src/parser.ts | 40 +++++++++++++++++++------------------- src/tokenizer.ts | 50 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 27f0af6320..981eb803e9 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -290,12 +290,12 @@ export class Parser extends DiagnosticEmitter { tn.next(); let abstractStart = tn.tokenPos; let abstractEnd = tn.pos; - let next = tn.peek(true); - if (tn.nextTokenOnNewLine) { + if (tn.peekOnNewLine()) { tn.reset(state); statement = this.parseStatement(tn, true); break; } + let next = tn.peek(); if (next != Token.Class) { if (next == Token.Interface) { this.error( @@ -322,7 +322,7 @@ export class Parser extends DiagnosticEmitter { case Token.Namespace: { let state = tn.mark(); tn.next(); - if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) { + if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) { tn.discard(state); statement = this.parseNamespace(tn, flags, decorators, startPos); decorators = null; @@ -345,7 +345,7 @@ export class Parser extends DiagnosticEmitter { case Token.Type: { // also identifier let state = tn.mark(); tn.next(); - if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) { + if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) { tn.discard(state); statement = this.parseTypeDeclaration(tn, flags, decorators, startPos); decorators = null; @@ -358,7 +358,7 @@ export class Parser extends DiagnosticEmitter { case Token.Module: { // also identifier let state = tn.mark(); tn.next(); - if (tn.peek(true) == Token.StringLiteral && !tn.nextTokenOnNewLine) { + if (tn.peek() == Token.StringLiteral && !tn.peekOnNewLine()) { tn.discard(state); statement = this.parseModuleDeclaration(tn, flags); } else { @@ -1113,10 +1113,11 @@ export class Parser extends DiagnosticEmitter { let startPos = tn.tokenPos; let expr: Expression | null = null; + let nextToken = tn.peek(); if ( - tn.peek(true) != Token.Semicolon && - tn.nextToken != Token.CloseBrace && - !tn.nextTokenOnNewLine + nextToken != Token.Semicolon && + nextToken != Token.CloseBrace && + !tn.peekOnNewLine() ) { if (!(expr = this.parseExpression(tn))) return null; } @@ -2042,7 +2043,7 @@ export class Parser extends DiagnosticEmitter { let setEnd = 0; if (!isInterface) { if (tn.skip(Token.Get)) { - if (tn.peek(true, IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) { + if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) { flags |= CommonFlags.Get; isGetter = true; getStart = tn.tokenPos; @@ -2058,7 +2059,7 @@ export class Parser extends DiagnosticEmitter { tn.reset(state); } } else if (tn.skip(Token.Set)) { - if (tn.peek(true, IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) { + if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) { flags |= CommonFlags.Set; isSetter = true; setStart = tn.tokenPos; @@ -2966,7 +2967,7 @@ export class Parser extends DiagnosticEmitter { break; } case Token.Type: { // also identifier - if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) { + if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) { statement = this.parseTypeDeclaration(tn, CommonFlags.None, null, tn.tokenPos); break; } @@ -3020,7 +3021,7 @@ export class Parser extends DiagnosticEmitter { // at 'break': Identifier? ';'? let identifier: IdentifierExpression | null = null; - if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) { + if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) { tn.next(IdentifierHandling.Prefer); identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range()); } @@ -3036,7 +3037,7 @@ export class Parser extends DiagnosticEmitter { // at 'continue': Identifier? ';'? let identifier: IdentifierExpression | null = null; - if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) { + if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) { tn.next(IdentifierHandling.Prefer); identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range()); } @@ -3948,7 +3949,7 @@ export class Parser extends DiagnosticEmitter { if (tn.skip(Token.TemplateLiteral)) { return this.parseTemplateLiteral(tn, identifier); } - if (tn.peek(true) == Token.Equals_GreaterThan && !tn.nextTokenOnNewLine) { + if (tn.peek() == Token.Equals_GreaterThan && !tn.peekOnNewLine()) { return this.parseFunctionExpressionCommon( tn, Node.createEmptyIdentifierExpression(tn.range(startPos)), @@ -4405,8 +4406,8 @@ export class Parser extends DiagnosticEmitter { tn: Tokenizer ): void { // see: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion - let token = tn.peek(true); - if (tn.nextTokenOnNewLine || token == Token.EndOfFile || token == Token.CloseBrace) return; + let nextToken = tn.peek(); + if (nextToken == Token.EndOfFile || nextToken == Token.CloseBrace || tn.peekOnNewLine()) return; this.error( DiagnosticCode.Unexpected_token, tn.range(tn.nextTokenPos) @@ -4415,10 +4416,9 @@ export class Parser extends DiagnosticEmitter { /** Skips over a statement on errors in an attempt to reduce unnecessary diagnostic noise. */ skipStatement(tn: Tokenizer): void { - tn.peek(true); - if (tn.nextTokenOnNewLine) tn.next(); // if reset() to the previous line + if (tn.peekOnNewLine()) tn.next(); // if reset() to the previous line do { - let nextToken = tn.peek(true); + let nextToken = tn.peek(); if ( nextToken == Token.EndOfFile || // next step should handle this nextToken == Token.Semicolon // end of the statement for sure @@ -4426,7 +4426,7 @@ export class Parser extends DiagnosticEmitter { tn.next(); break; } - if (tn.nextTokenOnNewLine) break; // end of the statement maybe + if (tn.peekOnNewLine()) break; // end of the statement maybe switch (tn.next()) { case Token.Identifier: { tn.readIdentifier(); diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 4fef11b283..a4bf9e9aa4 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -449,6 +449,13 @@ export function operatorTokenToString(token: Token): string { /** Handler for intercepting comments while tokenizing. */ export type CommentHandler = (kind: CommentKind, text: string, range: Range) => void; +/** Whether a token begins on a new line, if known. */ +enum OnNewLine { + No, + Yes, + Unknown +} + /** Tokenizes a source to individual {@link Token}s. */ export class Tokenizer extends DiagnosticEmitter { @@ -461,7 +468,7 @@ export class Tokenizer extends DiagnosticEmitter { nextToken: Token = -1; nextTokenPos: i32 = 0; - nextTokenOnNewLine: bool = false; + nextTokenOnNewLine: OnNewLine = OnNewLine.Unknown; onComment: CommentHandler | null = null; @@ -504,7 +511,7 @@ export class Tokenizer extends DiagnosticEmitter { } next(identifierHandling: IdentifierHandling = IdentifierHandling.Default): Token { - this.nextToken = -1; + this.clearNextToken(); let token: Token; do token = this.unsafeNext(identifierHandling); while (token == Token.Invalid); @@ -959,34 +966,41 @@ export class Tokenizer extends DiagnosticEmitter { } peek( - checkOnNewLine: bool = false, identifierHandling: IdentifierHandling = IdentifierHandling.Default, maxCompoundLength: i32 = i32.MAX_VALUE ): Token { - let text = this.source.text; - if (this.nextToken < 0) { + let nextToken = this.nextToken; + if (nextToken < 0) { let posBefore = this.pos; let tokenBefore = this.token; let tokenPosBefore = this.tokenPos; - let nextToken: Token; do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength); while (nextToken == Token.Invalid); this.nextToken = nextToken; this.nextTokenPos = this.tokenPos; + this.nextTokenOnNewLine = OnNewLine.Unknown; this.pos = posBefore; this.token = tokenBefore; this.tokenPos = tokenPosBefore; } - if (checkOnNewLine) { - this.nextTokenOnNewLine = false; - for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) { - if (isLineBreak(text.charCodeAt(pos))) { - this.nextTokenOnNewLine = true; - break; - } + return nextToken; + } + + peekOnNewLine(): bool { + switch (this.nextTokenOnNewLine) { + case OnNewLine.No: return false; + case OnNewLine.Yes: return true; + } + this.peek(); + let text = this.source.text; + for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) { + if (isLineBreak(text.charCodeAt(pos))) { + this.nextTokenOnNewLine = OnNewLine.Yes; + return true; } } - return this.nextToken; + this.nextTokenOnNewLine = OnNewLine.No; + return false; } skipIdentifier(identifierHandling: IdentifierHandling = IdentifierHandling.Prefer): bool { @@ -1006,7 +1020,7 @@ export class Tokenizer extends DiagnosticEmitter { while (nextToken == Token.Invalid); if (nextToken == token) { this.token = token; - this.nextToken = -1; + this.clearNextToken(); return true; } else { this.pos = posBefore; @@ -1037,7 +1051,13 @@ export class Tokenizer extends DiagnosticEmitter { this.pos = state.pos; this.token = state.token; this.tokenPos = state.tokenPos; + this.clearNextToken(); + } + + clearNextToken(): void { this.nextToken = -1; + this.nextTokenPos = 0; + this.nextTokenOnNewLine = OnNewLine.Unknown; } range(start: i32 = -1, end: i32 = -1): Range {