From 21de343f89774ef82db4e45d498a78b9cc8fb612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20P=C3=A1nek?= Date: Wed, 27 Jul 2016 18:59:29 +0200 Subject: [PATCH 1/5] Add support for simple import *statement* to identifier I couldn't get import as an expression to work, so I resorted to define it as a statement for now. I'm pretty sure multi-line imports don't work either and there's no alias functionality yet. So this is still *heavily* WIP. --- src/grammar.coffee | 5 +++-- src/lexer.coffee | 11 ++++++++++- src/nodes.coffee | 20 +++++++++++++++++--- test/modules.coffee | 16 ++++++++-------- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/grammar.coffee b/src/grammar.coffee index d28127bf21..50854f4116 100644 --- a/src/grammar.coffee +++ b/src/grammar.coffee @@ -97,6 +97,8 @@ grammar = o 'Return' o 'Comment' o 'STATEMENT', -> new StatementLiteral $1 + o 'Import' + o 'Export' ] # All the different types of expressions in our language. The basic unit of @@ -117,8 +119,6 @@ grammar = o 'Class' o 'Throw' o 'Yield' - o 'Import' - o 'Export' ] Yield: [ @@ -353,6 +353,7 @@ grammar = Import: [ o 'IMPORT Expression', -> new Import $2 + o 'IMPORT Assignable FROM Expression', -> new Import $4, $2 ] # Ordinary function invocation, or a chained series of calls. diff --git a/src/lexer.coffee b/src/lexer.coffee index 2a83f8ffe2..aa1f5c54aa 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -258,6 +258,13 @@ exports.Lexer = class Lexer @token 'JS', (script = match[0])[1...-1], 0, script.length script.length + importToken: -> + match = @chunk.match(IMPORT) + + return 0 unless match + + @token('IMPORT', match[0], 0, match[0].length) + # Matches regular expression literals, as well as multiline extended ones. # Lexing regular expressions is difficult to distinguish from division, so we # borrow some basic heuristics from JavaScript and Ruby. @@ -774,7 +781,7 @@ JS_KEYWORDS = [ 'return', 'throw', 'break', 'continue', 'debugger', 'yield' 'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally' 'class', 'extends', 'super' - 'export', 'import', 'default' + 'export', 'import', 'from', 'default' ] # CoffeeScript-only keywords. @@ -849,6 +856,8 @@ MULTI_DENT = /^(?:\n[^\n\S]*)+/ JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/ +IMPORT = /^import .+/ + # String-matching-regexes. STRING_START = /^(?:'''|"""|'|")/ diff --git a/src/nodes.coffee b/src/nodes.coffee index c050929e22..662d2cbbdd 100644 --- a/src/nodes.coffee +++ b/src/nodes.coffee @@ -1226,9 +1226,9 @@ exports.Class = class Class extends Base #### Import exports.Import = class Import extends Base - constructor: (@expression) -> + constructor: (@expression, @identifier) -> - children: ['expression'] + children: ['expression', 'identifier'] isStatement: YES jumps: NO @@ -1236,8 +1236,22 @@ exports.Import = class Import extends Base makeReturn: THIS compileNode: (o) -> - [].concat @makeCode(@tab + 'import '), @expression.compileToFragments(o), @makeCode(';') + code = [] + + code.push @makeCode(@tab + 'import ') + + + if @identifier + code.push @makeCode("#{@identifier.value} from ") + + + if @expression.base.value? + code.push @makeCode(@expression.base.value) + else + code.push @makeCode(@expression.base) + code.push @makeCode(';') + code #### Assign diff --git a/test/modules.coffee b/test/modules.coffee index 18509ef372..d96db6e295 100644 --- a/test/modules.coffee +++ b/test/modules.coffee @@ -21,15 +21,15 @@ test "import module", -> # console.log toJS input eq toJS(input), output -# test "module import test, syntax #1", -> -# input = "import foo from 'lib'" -# output = "import foo from 'lib';" -# eq toJS(input), output +test "module import test, syntax #1", -> + input = "import foo from 'lib'" + output = "import foo from 'lib';" + eq toJS(input), output -# test "module import test, syntax #2", -> -# input = "import { foo } from 'lib'" -# output = "import { foo } from 'lib';" -# eq toJS(input), output +test "module import test, syntax #2", -> + input = "import { foo } from 'lib'" + output = "import { foo } from 'lib';" + eq toJS(input), output # test "module import test, syntax #3", -> # input = "import { default as foo } from 'lib'" From 118c11cf85174de9aa3623e6135d0287eba52ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20P=C3=A1nek?= Date: Thu, 28 Jul 2016 15:10:30 +0200 Subject: [PATCH 2/5] Add IdentifierList and NamedImport to the grammar as well as IMPORT_FROM and IMPORT_AS tokens in lexer --- src/grammar.coffee | 22 ++++++++++++++++++++-- src/lexer.coffee | 27 ++++++++++++++++++++------- src/nodes.coffee | 43 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/src/grammar.coffee b/src/grammar.coffee index 50854f4116..25e428958a 100644 --- a/src/grammar.coffee +++ b/src/grammar.coffee @@ -139,6 +139,14 @@ grammar = o 'IDENTIFIER', -> new IdentifierLiteral $1 ] + IdentifierList: [ + o 'Identifier', -> [$1] + o 'IdentifierList , Identifier', -> $1.concat $3 + o 'IdentifierList OptComma TERMINATOR Identifier', -> $1.concat $4 + o 'INDENT IdentifierList OptComma OUTDENT', -> $2 + o 'IdentifierList OptComma INDENT IdentifierList OptComma OUTDENT', -> $1.concat $4 + ] + Property: [ o 'PROPERTY', -> new PropertyName $1 ] @@ -351,9 +359,19 @@ grammar = o 'CLASS SimpleAssignable EXTENDS Expression Block', -> new Class $2, $4, $5 ] + NamedImports: [ + o '{ }', -> new IdentifierList [] + o '{ IdentifierList OptComma }', -> new IdentifierList $2 + ] + + ImportClause: [ + o 'NamedImports' + o 'Identifier' + ] + Import: [ - o 'IMPORT Expression', -> new Import $2 - o 'IMPORT Assignable FROM Expression', -> new Import $4, $2 + o 'IMPORT String', -> new Import $2 + o 'IMPORT ImportClause IMPORT_FROM String', -> new Import $4, $2 ] # Ordinary function invocation, or a chained series of calls. diff --git a/src/lexer.coffee b/src/lexer.coffee index aa1f5c54aa..21b0bbaaaa 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -43,6 +43,7 @@ exports.Lexer = class Lexer @ends = [] # The stack for pairing up tokens. @tokens = [] # Stream of parsed tokens in the form `['TYPE', value, location data]`. @seenFor = no # Used to recognize FORIN and FOROF tokens. + @seenImport = no # Used to recognize IMPORT FROM? AS? tokens. @chunkLine = opts.line or 0 # The start line for the current @chunk. @@ -113,7 +114,22 @@ exports.Lexer = class Lexer if id is 'from' and @tag() is 'YIELD' @token 'FROM', id return id.length + + if id is 'import' + @seenImport = yes + @token 'IMPORT', id, 0, idLength + return idLength + + if id is 'from' and @seenImport + @token 'IMPORT_FROM', id + return idLength + + if id is 'as' and @seenImport + @token 'IMPORT_AS', id + return idLength + [..., prev] = @tokens + tag = if colon or prev? and (prev[0] in ['.', '?.', '::', '?::'] or @@ -258,13 +274,6 @@ exports.Lexer = class Lexer @token 'JS', (script = match[0])[1...-1], 0, script.length script.length - importToken: -> - match = @chunk.match(IMPORT) - - return 0 unless match - - @token('IMPORT', match[0], 0, match[0].length) - # Matches regular expression literals, as well as multiline extended ones. # Lexing regular expressions is difficult to distinguish from division, so we # borrow some basic heuristics from JavaScript and Ruby. @@ -324,9 +333,13 @@ exports.Lexer = class Lexer lineToken: -> return 0 unless match = MULTI_DENT.exec @chunk indent = match[0] + @seenFor = no + @seenImport = no + size = indent.length - 1 - indent.lastIndexOf '\n' noNewlines = @unfinished() + if size - @indebt is @indent if noNewlines then @suppressNewlines() else @newlineToken 0 return indent.length diff --git a/src/nodes.coffee b/src/nodes.coffee index 662d2cbbdd..e7b2bc97a7 100644 --- a/src/nodes.coffee +++ b/src/nodes.coffee @@ -1238,18 +1238,49 @@ exports.Import = class Import extends Base compileNode: (o) -> code = [] - code.push @makeCode(@tab + 'import ') + console.log('@expression: ', @expression, '@identifier: ', @identifier) + code.push @makeCode(@tab + 'import ') if @identifier - code.push @makeCode("#{@identifier.value} from ") + if @identifier.value? + code.push @makeCode("#{@identifier.value} from ") + else + console.log @identifier.compileNode(o) + code.push fragment for fragment in @identifier.compileNode(o) + code.push @makeCode(' from ') + + if @expression.value? + code.push @makeCode(@expression.value) - if @expression.base.value? - code.push @makeCode(@expression.base.value) - else - code.push @makeCode(@expression.base) code.push @makeCode(';') + code + +exports.IdentifierList = class IdentifierList extends Base + constructor: (identifiers) -> + @identifiers = identifiers or [] + + children: ['identifiers'] + + compileNode: (o) -> + return [@makeCode('[]')] unless @identifiers.length + + o.indent += TAB + + code = [] + compiledList = (identifier.compileToFragments o, LEVEL_LIST for identifier in @identifiers) + + for fragments, index in compiledList + code.push @makeCode(', ') if index + code.push fragments... + + if fragmentsToText(code).includes('\n') + code.unshift @makeCode("{\n#{o.indent}") + code.push @makeCode("\n#{@tab}}") + else + code.unshift @makeCode("{ ") + code.push @makeCode(" }") code From d2e64818b42bd439d96ed4e519581fa75e246abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20P=C3=A1nek?= Date: Thu, 28 Jul 2016 15:12:48 +0200 Subject: [PATCH 3/5] Remove accidently committed console.log debug output --- src/nodes.coffee | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/nodes.coffee b/src/nodes.coffee index e7b2bc97a7..6c6491301f 100644 --- a/src/nodes.coffee +++ b/src/nodes.coffee @@ -1238,16 +1238,12 @@ exports.Import = class Import extends Base compileNode: (o) -> code = [] - console.log('@expression: ', @expression, '@identifier: ', @identifier) - code.push @makeCode(@tab + 'import ') if @identifier if @identifier.value? code.push @makeCode("#{@identifier.value} from ") else - console.log @identifier.compileNode(o) - code.push fragment for fragment in @identifier.compileNode(o) code.push @makeCode(' from ') From d86540afbbebf39b51330cbc15303d4d339721ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20P=C3=A1nek?= Date: Thu, 28 Jul 2016 15:15:44 +0200 Subject: [PATCH 4/5] Remove IMPORT regex that has never been used --- src/lexer.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lexer.coffee b/src/lexer.coffee index 21b0bbaaaa..52fd2917f0 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -869,8 +869,6 @@ MULTI_DENT = /^(?:\n[^\n\S]*)+/ JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/ -IMPORT = /^import .+/ - # String-matching-regexes. STRING_START = /^(?:'''|"""|'|")/ From de78f4832d9ff9934cbcea2cf3ed3dc462db225e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20P=C3=A1nek?= Date: Thu, 28 Jul 2016 16:02:01 +0200 Subject: [PATCH 5/5] NamedImports work now and can be mixed with Identifiers as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I think I'm getting a hang of how it's supposed to work. 🙌 What works now, as you can see in all the tests that are not commented out and should run through: ```coffee import foo import bar from lib import { foo } from lib import { foo as boo, bar } from lib import { foo, bar } from lib ``` --- src/grammar.coffee | 26 ++++++++++++++++---------- src/nodes.coffee | 10 +++++++++- test/modules.coffee | 29 +++++++++++++++++------------ 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/grammar.coffee b/src/grammar.coffee index 25e428958a..ad8c1cf0b1 100644 --- a/src/grammar.coffee +++ b/src/grammar.coffee @@ -139,14 +139,6 @@ grammar = o 'IDENTIFIER', -> new IdentifierLiteral $1 ] - IdentifierList: [ - o 'Identifier', -> [$1] - o 'IdentifierList , Identifier', -> $1.concat $3 - o 'IdentifierList OptComma TERMINATOR Identifier', -> $1.concat $4 - o 'INDENT IdentifierList OptComma OUTDENT', -> $2 - o 'IdentifierList OptComma INDENT IdentifierList OptComma OUTDENT', -> $1.concat $4 - ] - Property: [ o 'PROPERTY', -> new PropertyName $1 ] @@ -360,13 +352,27 @@ grammar = ] NamedImports: [ - o '{ }', -> new IdentifierList [] - o '{ IdentifierList OptComma }', -> new IdentifierList $2 + o '{ }', -> new ImportsList [] + o '{ ImportsList OptComma }', -> new ImportsList $2 ] ImportClause: [ o 'NamedImports' + o 'ImportSpecifier' + o '* IMPORT_AS Identifier' + ] + + ImportSpecifier: [ o 'Identifier' + o 'Identifier IMPORT_AS Identifier', -> new ImportSpecifier $1, $3 + ] + + ImportsList: [ + o 'ImportSpecifier', -> [$1] + o 'ImportsList , ImportSpecifier', -> $1.concat $3 + o 'ImportsList OptComma TERMINATOR ImportSpecifier', -> $1.concat $4 + o 'INDENT ImportsList OptComma OUTDENT', -> $2 + o 'ImportsList OptComma INDENT ImportsList OptComma OUTDENT', -> $1.concat $4 ] Import: [ diff --git a/src/nodes.coffee b/src/nodes.coffee index 6c6491301f..fbbd7708ec 100644 --- a/src/nodes.coffee +++ b/src/nodes.coffee @@ -1253,7 +1253,7 @@ exports.Import = class Import extends Base code.push @makeCode(';') code -exports.IdentifierList = class IdentifierList extends Base +exports.ImportsList = class ImportsList extends Base constructor: (identifiers) -> @identifiers = identifiers or [] @@ -1280,6 +1280,14 @@ exports.IdentifierList = class IdentifierList extends Base code +exports.ImportSpecifier = class ImportSpecifier extends Base + constructor: (@original, @alias) -> + + children: ['from', 'as'] + + compileNode: (o) -> + return [@makeCode("#{@original.value} as #{@alias.value}")] + #### Assign # The **Assign** is used to assign a local variable to value, or to set the diff --git a/test/modules.coffee b/test/modules.coffee index d96db6e295..323b56c4dd 100644 --- a/test/modules.coffee +++ b/test/modules.coffee @@ -31,20 +31,25 @@ test "module import test, syntax #2", -> output = "import { foo } from 'lib';" eq toJS(input), output -# test "module import test, syntax #3", -> -# input = "import { default as foo } from 'lib'" -# output = "import { default as foo } from 'lib';" -# eq toJS(input), output +test "module import test, syntax #3", -> + input = "import { bar as foo } from 'lib'" + output = "import { bar as foo } from 'lib';" + eq toJS(input), output -# test "module import test, syntax #4", -> -# input = "import { square, diag } from 'lib'" -# output = "import { square, diag } from 'lib';" -# eq toJS(input), output +test "module import test, syntax #3", -> + input = "import { oof, bar as foo } from 'lib'" + output = "import { oof, bar as foo } from 'lib';" + eq toJS(input), output -# test "module import test, syntax #5", -> -# input = "import { foo } from 'lib' # with a comment" -# output = "import { foo } from 'lib' ;" -# eq toJS(input), output +test "module import test, syntax #4", -> + input = "import { square, diag } from 'lib'" + output = "import { square, diag } from 'lib';" + eq toJS(input), output + +test "module import test, syntax #5", -> + input = "import { foo } from 'lib' # with a comment" + output = "import { foo } from 'lib';" + eq toJS(input), output # test "module export test, syntax #1", -> # input = "export default mixin"