From e42f480cc74ad54237c8583bc6921684101c3403 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 9 Mar 2022 17:12:30 +0100 Subject: [PATCH 01/68] Add test command to parse a file. --- analysis/src/Commands.ml | 20 +++++++++++++++++++ analysis/tests/src/Completion.res | 2 ++ .../tests/src/DefinitionWithInterface.resi | 1 + .../tests/src/expected/Completion.res.txt | 3 +++ .../expected/DefinitionWithInterface.resi.txt | 3 +++ 5 files changed, 29 insertions(+) diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index 74504e8ba..4cb9f269b 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -349,6 +349,26 @@ let test ~path = close_out cout; completion ~path ~line ~col ~currentFile; Sys.remove currentFile + | "par" -> + print_endline ("Parse " ^ path); + if Filename.check_suffix path ".res" then + let parser = + Res_driver.parsingEngine.parseImplementation ~forPrinter:false + in + let {Res_driver.parsetree = structure; diagnostics} = + parser ~filename:path + in + Printf.printf "structure items:%d diagnostics:%d \n" + (List.length structure) (List.length diagnostics) + else + let parser = + Res_driver.parsingEngine.parseInterface ~forPrinter:false + in + let {Res_driver.parsetree = signature; diagnostics} = + parser ~filename:path + in + Printf.printf "signature items:%d diagnostics:%d \n" + (List.length signature) (List.length diagnostics) | _ -> ()); print_newline ()) in diff --git a/analysis/tests/src/Completion.res b/analysis/tests/src/Completion.res index 0d433ad25..9c3ee4fee 100644 --- a/analysis/tests/src/Completion.res +++ b/analysis/tests/src/Completion.res @@ -100,3 +100,5 @@ let make = () => { } // ^com Obj.object[" + +// ^par diff --git a/analysis/tests/src/DefinitionWithInterface.resi b/analysis/tests/src/DefinitionWithInterface.resi index 8b357372c..9230148bf 100644 --- a/analysis/tests/src/DefinitionWithInterface.resi +++ b/analysis/tests/src/DefinitionWithInterface.resi @@ -2,3 +2,4 @@ let y: int // ^def type t +// ^par \ No newline at end of file diff --git a/analysis/tests/src/expected/Completion.res.txt b/analysis/tests/src/expected/Completion.res.txt index e9cb17fca..60617fdbd 100644 --- a/analysis/tests/src/expected/Completion.res.txt +++ b/analysis/tests/src/expected/Completion.res.txt @@ -744,3 +744,6 @@ Complete tests/src/Completion.res 100:3 "documentation": null }] +Parse tests/src/Completion.res +structure items:19 diagnostics:0 + diff --git a/analysis/tests/src/expected/DefinitionWithInterface.resi.txt b/analysis/tests/src/expected/DefinitionWithInterface.resi.txt index 60c8ac515..614b2da0d 100644 --- a/analysis/tests/src/expected/DefinitionWithInterface.resi.txt +++ b/analysis/tests/src/expected/DefinitionWithInterface.resi.txt @@ -1,3 +1,6 @@ Definition tests/src/DefinitionWithInterface.resi 0:4 {"uri": "DefinitionWithInterface.res", "range": {"start": {"line": 0, "character": 4}, "end": {"line": 0, "character": 5}}} +Parse tests/src/DefinitionWithInterface.resi +signature items:2 diagnostics:0 + From bae0ce625c9a0c9078dd4bbd84a00fb92b5a732a Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 10 Mar 2022 10:30:19 +0100 Subject: [PATCH 02/68] Separate parser test. --- analysis/src/Commands.ml | 39 +++++++++---------- analysis/tests/src/Completion.res | 2 - .../tests/src/DefinitionWithInterface.resi | 1 - analysis/tests/src/Parser.res | 2 + .../tests/src/expected/Completion.res.txt | 3 -- analysis/tests/src/expected/Debug.res.txt | 3 +- .../expected/DefinitionWithInterface.resi.txt | 3 -- analysis/tests/src/expected/Parser.res.txt | 3 ++ 8 files changed, 26 insertions(+), 30 deletions(-) create mode 100644 analysis/tests/src/Parser.res create mode 100644 analysis/tests/src/expected/Parser.res.txt diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index 4cb9f269b..1dd652eb1 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -282,6 +282,24 @@ let rename ~path ~line ~col ~newName = in print_endline result +let parser ~path = + if Filename.check_suffix path ".res" then + let parser = + Res_driver.parsingEngine.parseImplementation ~forPrinter:false + in + let {Res_driver.parsetree = structure; diagnostics} = + parser ~filename:path + in + Printf.printf "structure items:%d diagnostics:%d \n" (List.length structure) + (List.length diagnostics) + else + let parser = Res_driver.parsingEngine.parseInterface ~forPrinter:false in + let {Res_driver.parsetree = signature; diagnostics} = + parser ~filename:path + in + Printf.printf "signature items:%d diagnostics:%d \n" (List.length signature) + (List.length diagnostics) + let test ~path = Uri2.stripPath := true; match Files.readFile path with @@ -314,7 +332,6 @@ let test ~path = print_endline ("Hover " ^ path ^ " " ^ string_of_int line ^ ":" ^ string_of_int col); - hover ~path ~line ~col | "ref" -> print_endline @@ -331,7 +348,6 @@ let test ~path = ("Rename " ^ path ^ " " ^ string_of_int line ^ ":" ^ string_of_int col ^ " " ^ newName) in - rename ~path ~line ~col ~newName | "com" -> print_endline @@ -351,24 +367,7 @@ let test ~path = Sys.remove currentFile | "par" -> print_endline ("Parse " ^ path); - if Filename.check_suffix path ".res" then - let parser = - Res_driver.parsingEngine.parseImplementation ~forPrinter:false - in - let {Res_driver.parsetree = structure; diagnostics} = - parser ~filename:path - in - Printf.printf "structure items:%d diagnostics:%d \n" - (List.length structure) (List.length diagnostics) - else - let parser = - Res_driver.parsingEngine.parseInterface ~forPrinter:false - in - let {Res_driver.parsetree = signature; diagnostics} = - parser ~filename:path - in - Printf.printf "signature items:%d diagnostics:%d \n" - (List.length signature) (List.length diagnostics) + parser ~path | _ -> ()); print_newline ()) in diff --git a/analysis/tests/src/Completion.res b/analysis/tests/src/Completion.res index 9c3ee4fee..0d433ad25 100644 --- a/analysis/tests/src/Completion.res +++ b/analysis/tests/src/Completion.res @@ -100,5 +100,3 @@ let make = () => { } // ^com Obj.object[" - -// ^par diff --git a/analysis/tests/src/DefinitionWithInterface.resi b/analysis/tests/src/DefinitionWithInterface.resi index 9230148bf..8b357372c 100644 --- a/analysis/tests/src/DefinitionWithInterface.resi +++ b/analysis/tests/src/DefinitionWithInterface.resi @@ -2,4 +2,3 @@ let y: int // ^def type t -// ^par \ No newline at end of file diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res new file mode 100644 index 000000000..8f79691a2 --- /dev/null +++ b/analysis/tests/src/Parser.res @@ -0,0 +1,2 @@ +let _ = +// ^par \ No newline at end of file diff --git a/analysis/tests/src/expected/Completion.res.txt b/analysis/tests/src/expected/Completion.res.txt index 60617fdbd..e9cb17fca 100644 --- a/analysis/tests/src/expected/Completion.res.txt +++ b/analysis/tests/src/expected/Completion.res.txt @@ -744,6 +744,3 @@ Complete tests/src/Completion.res 100:3 "documentation": null }] -Parse tests/src/Completion.res -structure items:19 diagnostics:0 - diff --git a/analysis/tests/src/expected/Debug.res.txt b/analysis/tests/src/expected/Debug.res.txt index f63aaa815..180b6861e 100644 --- a/analysis/tests/src/expected/Debug.res.txt +++ b/analysis/tests/src/expected/Debug.res.txt @@ -4,7 +4,7 @@ Dependencies: @rescript/react Source directories: tests/node_modules/@rescript/react/src tests/node_modules/@rescript/react/src/legacy Source files: tests/node_modules/@rescript/react/src/React.res tests/node_modules/@rescript/react/src/ReactDOM.res tests/node_modules/@rescript/react/src/ReactDOMServer.res tests/node_modules/@rescript/react/src/ReactDOMStyle.res tests/node_modules/@rescript/react/src/ReactEvent.res tests/node_modules/@rescript/react/src/ReactEvent.resi tests/node_modules/@rescript/react/src/ReactTestUtils.res tests/node_modules/@rescript/react/src/ReactTestUtils.resi tests/node_modules/@rescript/react/src/RescriptReactErrorBoundary.res tests/node_modules/@rescript/react/src/RescriptReactErrorBoundary.resi tests/node_modules/@rescript/react/src/RescriptReactRouter.res tests/node_modules/@rescript/react/src/RescriptReactRouter.resi tests/node_modules/@rescript/react/src/legacy/ReactDOMRe.res tests/node_modules/@rescript/react/src/legacy/ReasonReact.res Source directories: tests/src tests/src/expected -Source files: tests/src/Auto.res tests/src/CompletePrioritize1.res tests/src/CompletePrioritize2.res tests/src/Completion.res tests/src/Component.res tests/src/Component.resi tests/src/Cross.res tests/src/Debug.res tests/src/Definition.res tests/src/DefinitionWithInterface.res tests/src/DefinitionWithInterface.resi tests/src/Div.res tests/src/Fragment.res tests/src/Hover.res tests/src/Jsx.res tests/src/Jsx.resi tests/src/LongIdentTest.res tests/src/Obj.res tests/src/Patterns.res tests/src/RecModules.res tests/src/RecordCompletion.res tests/src/References.res tests/src/ReferencesWithInterface.res tests/src/ReferencesWithInterface.resi tests/src/Rename.res tests/src/RenameWithInterface.res tests/src/RenameWithInterface.resi tests/src/TableclothMap.ml tests/src/TableclothMap.mli tests/src/TypeDefinition.res +Source files: tests/src/Auto.res tests/src/CompletePrioritize1.res tests/src/CompletePrioritize2.res tests/src/Completion.res tests/src/Component.res tests/src/Component.resi tests/src/Cross.res tests/src/Debug.res tests/src/Definition.res tests/src/DefinitionWithInterface.res tests/src/DefinitionWithInterface.resi tests/src/Div.res tests/src/Fragment.res tests/src/Hover.res tests/src/Jsx.res tests/src/Jsx.resi tests/src/LongIdentTest.res tests/src/Obj.res tests/src/Parser.res tests/src/Patterns.res tests/src/RecModules.res tests/src/RecordCompletion.res tests/src/References.res tests/src/ReferencesWithInterface.res tests/src/ReferencesWithInterface.resi tests/src/Rename.res tests/src/RenameWithInterface.res tests/src/RenameWithInterface.resi tests/src/TableclothMap.ml tests/src/TableclothMap.mli tests/src/TypeDefinition.res Impl cmt:tests/lib/bs/src/Auto.cmt res:tests/src/Auto.res Impl cmt:tests/lib/bs/src/CompletePrioritize1.cmt res:tests/src/CompletePrioritize1.res Impl cmt:tests/lib/bs/src/CompletePrioritize2.cmt res:tests/src/CompletePrioritize2.res @@ -20,6 +20,7 @@ Impl cmt:tests/lib/bs/src/Hover.cmt res:tests/src/Hover.res IntfAndImpl cmti:tests/lib/bs/src/Jsx.cmti resi:tests/src/Jsx.resi cmt:tests/lib/bs/src/Jsx.cmt res:tests/src/Jsx.res Impl cmt:tests/lib/bs/src/LongIdentTest.cmt res:tests/src/LongIdentTest.res Impl cmt:tests/lib/bs/src/Obj.cmt res:tests/src/Obj.res +Impl cmt:tests/lib/bs/src/Parser.cmt res:tests/src/Parser.res Impl cmt:tests/lib/bs/src/Patterns.cmt res:tests/src/Patterns.res Impl cmt:tests/lib/bs/src/RecModules.cmt res:tests/src/RecModules.res Impl cmt:tests/lib/bs/src/RecordCompletion.cmt res:tests/src/RecordCompletion.res diff --git a/analysis/tests/src/expected/DefinitionWithInterface.resi.txt b/analysis/tests/src/expected/DefinitionWithInterface.resi.txt index 614b2da0d..60c8ac515 100644 --- a/analysis/tests/src/expected/DefinitionWithInterface.resi.txt +++ b/analysis/tests/src/expected/DefinitionWithInterface.resi.txt @@ -1,6 +1,3 @@ Definition tests/src/DefinitionWithInterface.resi 0:4 {"uri": "DefinitionWithInterface.res", "range": {"start": {"line": 0, "character": 4}, "end": {"line": 0, "character": 5}}} -Parse tests/src/DefinitionWithInterface.resi -signature items:2 diagnostics:0 - diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt new file mode 100644 index 000000000..ca76cf57b --- /dev/null +++ b/analysis/tests/src/expected/Parser.res.txt @@ -0,0 +1,3 @@ +Parse tests/src/Parser.res +structure items:1 diagnostics:0 + From cba7ba65c3e520ec81dfb66d878505d246925fe5 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 10 Mar 2022 11:42:59 +0100 Subject: [PATCH 03/68] Add part of core logic to recognise JSX uppercase/lowercase, type arguments, binary operators. --- analysis/src/Commands.ml | 129 ++++++++++++++++++++- analysis/tests/src/Parser.res | 36 +++++- analysis/tests/src/expected/Parser.res.txt | 15 ++- 3 files changed, 175 insertions(+), 5 deletions(-) diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index 1dd652eb1..f69989cd7 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -283,7 +283,7 @@ let rename ~path ~line ~col ~newName = print_endline result let parser ~path = - if Filename.check_suffix path ".res" then + if Filename.check_suffix path ".res" then ( let parser = Res_driver.parsingEngine.parseImplementation ~forPrinter:false in @@ -291,7 +291,132 @@ let parser ~path = parser ~filename:path in Printf.printf "structure items:%d diagnostics:%d \n" (List.length structure) - (List.length diagnostics) + (List.length diagnostics); + + let jsxName lident = + let rec flatten acc lident = + match lident with + | Longident.Lident txt -> txt :: acc + | Ldot (lident, txt) -> + let acc = if txt = "createElement" then acc else txt :: acc in + flatten acc lident + | _ -> acc + in + match lident with + | Longident.Lident txt -> txt + | _ as lident -> + let segments = flatten [] lident in + segments |> String.concat "." + in + let locToString (loc : Location.t) = + let lineStart, colStart = Utils.tupleOfLexing loc.loc_start in + let lineEnd, colEnd = Utils.tupleOfLexing loc.loc_end in + Printf.sprintf "(%d,%d)->(%d,%d)" lineStart colStart lineEnd colEnd + in + let rec processExpression (expr : Parsetree.expression) = + match expr.pexp_desc with + | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) + when Res_parsetree_viewer.isJsxExpression expr -> + let rec isSelfClosing args = + match args with + | [] -> false + | [ + ( Asttypes.Labelled "children", + { + Parsetree.pexp_desc = + Pexp_construct ({txt = Longident.Lident "[]"}, None); + } ); + _; + ] -> + true + | _ :: rest -> isSelfClosing rest + in + Printf.printf "JsxOpen: %s %s\n" (jsxName lident.txt) + (locToString pexp_loc); + (if not (isSelfClosing args) then + let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in + let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in + let size = if lineStart = lineEnd then colEnd - colStart else 0 in + let lineEndWhole, colEndWhole = + Utils.tupleOfLexing expr.pexp_loc.loc_end + in + if size > 0 && colEndWhole > size then + Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" lineEndWhole + (colEndWhole - size - 1) + lineEndWhole (colEndWhole - 1)); + args |> List.iter (fun (_lbl, e) -> processExpression e) + | Pexp_apply ({pexp_loc}, args) + when Res_parsetree_viewer.isBinaryExpression expr -> + Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); + args |> List.iter (fun (_lbl, e) -> processExpression e) + | Pexp_apply (f, args) -> + processExpression f; + args |> List.iter (fun (_lbl, e) -> processExpression e) + | Pexp_construct (_lidend, expOpt) -> processExpressionOption expOpt + | Pexp_tuple exprs -> exprs |> List.iter processExpression + | Pexp_ident _ -> () + | Pexp_constant _ -> () + | Pexp_unreachable -> assert false + | Pexp_let (_, _, _) -> assert false + | Pexp_function _ -> assert false + | Pexp_fun (_, _, _, _) -> assert false + | Pexp_match (_, _) -> assert false + | Pexp_try (_, _) -> assert false + | Pexp_variant (_, _) -> assert false + | Pexp_record (_, _) -> assert false + | Pexp_field (_, _) -> assert false + | Pexp_setfield (_, _, _) -> assert false + | Pexp_array _ -> assert false + | Pexp_ifthenelse (_, _, _) -> assert false + | Pexp_sequence (_, _) -> assert false + | Pexp_while (_, _) -> assert false + | Pexp_for (_, _, _, _, _) -> assert false + | Pexp_constraint (_, _) -> assert false + | Pexp_coerce (_, _, _) -> assert false + | Pexp_send (_, _) -> assert false + | Pexp_new _ -> assert false + | Pexp_setinstvar (_, _) -> assert false + | Pexp_override _ -> assert false + | Pexp_letmodule (_, _, _) -> assert false + | Pexp_letexception (_, _) -> assert false + | Pexp_assert _ -> assert false + | Pexp_lazy _ -> assert false + | Pexp_poly (_, _) -> assert false + | Pexp_object _ -> assert false + | Pexp_newtype (_, _) -> assert false + | Pexp_pack _ -> assert false + | Pexp_open (_, _, _) -> assert false + | Pexp_extension _ -> assert false + and processExpressionOption = function + | None -> () + | Some e -> processExpression e + in + + let processValueBinding (binding : Parsetree.value_binding) = + processExpression binding.pvb_expr + in + let rec processTypeArg (coreType : Parsetree.core_type) = + Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc); + processCoreType coreType + and processCoreType (coreType : Parsetree.core_type) = + match coreType.ptyp_desc with + | Ptyp_constr (_lident, args) -> args |> List.iter processTypeArg + | _ -> () + in + let processTypeDeclaration (typeDecl : Parsetree.type_declaration) = + match typeDecl.ptype_manifest with + | Some t -> processCoreType t + | None -> () + in + let processStructureItem (item : Parsetree.structure_item) = + match item.pstr_desc with + | Pstr_value (_recFlag, bindings) -> + bindings |> List.iter processValueBinding + | Pstr_type (_recFlat, typeDecls) -> + typeDecls |> List.iter processTypeDeclaration + | _ -> () + in + structure |> List.iter processStructureItem) else let parser = Res_driver.parsingEngine.parseInterface ~forPrinter:false in let {Res_driver.parsetree = signature; diagnostics} = diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 8f79691a2..97f36a166 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -1,2 +1,34 @@ -let _ = -// ^par \ No newline at end of file +module M = { + module C = Component +} + +let _c = + +let _mc = + +let _d =
+ +let _d2 = +
+ {React.string("abc")} +
{React.string("abc")}
+ {React.string("abc")} + {React.string("abc")} +
+ +type pair<'x, 'y> = ('x, 'y) + +type looooooooooooooooooooooooooooooooooooooong_int = int + +type looooooooooooooooooooooooooooooooooooooong_string = string + +type pairIntString = list< + pair< + looooooooooooooooooooooooooooooooooooooong_int, + looooooooooooooooooooooooooooooooooooooong_string, + >, +> + +let _ = 3 < 4 || 3 > 4 + +// ^par diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index ca76cf57b..b6fbfec3f 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,3 +1,16 @@ Parse tests/src/Parser.res -structure items:1 diagnostics:0 +structure items:10 diagnostics:0 +JsxOpen: Component (4,10)->(4,19) +JsxOpen: M.C (6,11)->(6,14) +JsxOpen: div (8,10)->(8,13) +JsxOpen: div (11,3)->(11,6) +JsxClose: (16,4)->(16,7) +JsxOpen: div (13,5)->(13,8) +JsxClose: (13,34)->(13,37) +TypeArg: (25,2)->(28,3) +TypeArg: (26,4)->(26,50) +TypeArg: (27,4)->(27,53) +BinaryExp: (31,14)->(31,16) +BinaryExp: (31,10)->(31,11) +BinaryExp: (31,19)->(31,20) From b1c3d0a094a648c25033f8b15676f19135a07c77 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 11 Mar 2022 10:18:53 +0100 Subject: [PATCH 04/68] Basic setup of extension analysis bin communication. From https://github.com/zth --- analysis/src/Cli.ml | 3 +- analysis/src/Commands.ml | 25 +++++++++++++++ server/src/server.ts | 66 ++++++++++++++++++++++++++-------------- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/analysis/src/Cli.ml b/analysis/src/Cli.ml index cc08302ea..126335f0c 100644 --- a/analysis/src/Cli.ml +++ b/analysis/src/Cli.ml @@ -70,6 +70,7 @@ let main () = ~col:(int_of_string col) | _ :: "dump" :: files -> Commands.dump files | [_; "documentSymbol"; path] -> Commands.documentSymbol ~path + | [_; "semanticTokens"; _path] -> Commands.parserTest () | [_; "hover"; path; line; col] -> Commands.hover ~path ~line:(int_of_string line) ~col:(int_of_string col) | [_; "references"; path; line; col] -> @@ -83,6 +84,6 @@ let main () = | _ -> prerr_endline help; exit 1 - ;; + main () diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index f69989cd7..2b35bd26c 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -282,6 +282,31 @@ let rename ~path ~line ~col ~newName = in print_endline result +type tokenLegend = {tokenTypes : string array; tokenModifiers : string array} + +(* This needs to stay synced with the same legend in `server.ts` *) +let tokenLegend = {tokenTypes = [|"keyword"|]; tokenModifiers = [||]} + +(* These are not used yet, but taken from https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) +type tokenAtLoc = { + line : int; + startChar : int; + length : int; + tokenType : int; + tokenModifiers : int; +} + +type tokenAtDelta = { + deltaLine : int; + deltaStartChar : int; + length : int; + tokenType : int; + tokenModifiers : int; +} + +(* TEST: This will color the first 3 letters of any ReScript file as a keyword.*) +let parserTest () = Printf.printf "{\"data\":[0,0,3,0,0]}" + let parser ~path = if Filename.check_suffix path ".res" then ( let parser = diff --git a/server/src/server.ts b/server/src/server.ts index a31c2a6b1..4bb94bc73 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -15,10 +15,9 @@ import * as utils from "./utils"; import * as c from "./constants"; import * as chokidar from "chokidar"; import { assert } from "console"; -import { fileURLToPath, pathToFileURL } from "url"; +import { fileURLToPath } from "url"; import { ChildProcess } from "child_process"; import { WorkspaceEdit } from "vscode-languageserver"; -import { TextEdit } from "vscode-languageserver-types"; // https://microsoft.github.io/language-server-protocol/specification#initialize // According to the spec, there could be requests before the 'initialize' request. Link in comment tells how to handle them. @@ -38,17 +37,17 @@ let projectsFiles: Map< // ^ caching AND states AND distributed system. Why does LSP has to be stupid like this // will be properly defined later depending on the mode (stdio/node-rpc) -let send: (msg: m.Message) => void = (_) => { }; +let send: (msg: m.Message) => void = (_) => {}; interface CreateInterfaceRequestParams { uri: string; } -let createInterfaceRequest = - new v.RequestType< - CreateInterfaceRequestParams, - string, - void>("rescript-vscode.create_interface"); +let createInterfaceRequest = new v.RequestType< + CreateInterfaceRequestParams, + string, + void +>("rescript-vscode.create_interface"); interface OpenCompiledFileParams { uri: string; @@ -66,9 +65,8 @@ let sendUpdatedDiagnostics = () => { path.join(projectRootPath, c.compilerLogPartialPath), { encoding: "utf-8" } ); - let { done, result: filesAndErrors } = utils.parseCompilerLogOutput( - content - ); + let { done, result: filesAndErrors } = + utils.parseCompilerLogOutput(content); // diff Object.keys(filesAndErrors).forEach((file) => { @@ -290,7 +288,12 @@ function typeDefinition(msg: p.RequestMessage) { let filePath = fileURLToPath(params.textDocument.uri); let response = utils.runAnalysisCommand( filePath, - ["typeDefinition", filePath, params.position.line, params.position.character], + [ + "typeDefinition", + filePath, + params.position.line, + params.position.character, + ], msg ); return response; @@ -323,7 +326,7 @@ function prepareRename(msg: p.RequestMessage): m.ResponseMessage { ); let result: p.Range | null = null; if (locations !== null) { - locations.forEach(loc => { + locations.forEach((loc) => { if ( path.normalize(fileURLToPath(loc.uri)) === path.normalize(fileURLToPath(params.textDocument.uri)) @@ -337,14 +340,14 @@ function prepareRename(msg: p.RequestMessage): m.ResponseMessage { end.line >= pos.line ) { result = loc.range; - }; + } } }); - }; + } return { jsonrpc: c.jsonrpcVersion, id: msg.id, - result + result, }; } @@ -352,23 +355,22 @@ function rename(msg: p.RequestMessage) { // https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_rename let params = msg.params as p.RenameParams; let filePath = fileURLToPath(params.textDocument.uri); - let documentChanges: - | (p.RenameFile | p.TextDocumentEdit)[] - | null = utils.runAnalysisAfterSanityCheck(filePath, [ + let documentChanges: (p.RenameFile | p.TextDocumentEdit)[] | null = + utils.runAnalysisAfterSanityCheck(filePath, [ "rename", filePath, params.position.line, params.position.character, - params.newName + params.newName, ]); let result: WorkspaceEdit | null = null; if (documentChanges !== null) { result = { documentChanges }; - }; + } let response: m.ResponseMessage = { jsonrpc: c.jsonrpcVersion, id: msg.id, - result + result, }; return response; } @@ -385,6 +387,18 @@ function documentSymbol(msg: p.RequestMessage) { return response; } +function semanticTokens(msg: p.RequestMessage) { + // https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens + let params = msg.params as p.SemanticTokensParams; + let filePath = fileURLToPath(params.textDocument.uri); + let response = utils.runAnalysisCommand( + filePath, + ["semanticTokens", filePath], + msg + ); + return response; +} + function completion(msg: p.RequestMessage) { let params = msg.params as p.ReferenceParams; let filePath = fileURLToPath(params.textDocument.uri); @@ -739,6 +753,12 @@ function onMessage(msg: m.Message) { // disabled right now until we use the parser to show non-stale symbols per keystroke // documentSymbolProvider: true, completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, + semanticTokensProvider: { + legend: { tokenTypes: ["keyword"], tokenModifiers: [] }, + documentSelector: null, + // TODO: Support range for full, and add delta support + full: true, + }, }, }; let response: m.ResponseMessage = { @@ -797,6 +817,8 @@ function onMessage(msg: m.Message) { send(documentSymbol(msg)); } else if (msg.method === p.CompletionRequest.method) { send(completion(msg)); + } else if (msg.method === p.SemanticTokensRequest.method) { + send(semanticTokens(msg)); } else if (msg.method === p.DocumentFormattingRequest.method) { let responses = format(msg); responses.forEach((response) => send(response)); From 7bf68196225840a6523a8d549776b8acf5a64f94 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 11 Mar 2022 11:08:55 +0100 Subject: [PATCH 05/68] Simple token emitter --- analysis/src/Cli.ml | 2 +- analysis/src/Commands.ml | 53 +++++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/analysis/src/Cli.ml b/analysis/src/Cli.ml index 126335f0c..9077a867f 100644 --- a/analysis/src/Cli.ml +++ b/analysis/src/Cli.ml @@ -70,7 +70,7 @@ let main () = ~col:(int_of_string col) | _ :: "dump" :: files -> Commands.dump files | [_; "documentSymbol"; path] -> Commands.documentSymbol ~path - | [_; "semanticTokens"; _path] -> Commands.parserTest () + | [_; "semanticTokens"; _path] -> Commands.semanticTokensTest () | [_; "hover"; path; line; col] -> Commands.hover ~path ~line:(int_of_string line) ~col:(int_of_string col) | [_; "references"; path; line; col] -> diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index 2b35bd26c..3304449b8 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -282,30 +282,43 @@ let rename ~path ~line ~col ~newName = in print_endline result -type tokenLegend = {tokenTypes : string array; tokenModifiers : string array} +module Token = struct + type legend = {tokenTypes : string array; tokenModifiers : string array} -(* This needs to stay synced with the same legend in `server.ts` *) -let tokenLegend = {tokenTypes = [|"keyword"|]; tokenModifiers = [||]} + (* This needs to stay synced with the same legend in `server.ts` *) + (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) + type tokenType = Keyword + type tokenModifiers = NoModifier -(* These are not used yet, but taken from https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) -type tokenAtLoc = { - line : int; - startChar : int; - length : int; - tokenType : int; - tokenModifiers : int; -} + let tokenTypeToString = function Keyword -> "0" + let tokenModifiersToString = function NoModifier -> "0" -type tokenAtDelta = { - deltaLine : int; - deltaStartChar : int; - length : int; - tokenType : int; - tokenModifiers : int; -} + type emitter = { + buf : Buffer.t; + mutable lastLine : int; + mutable lastChar : int; + } -(* TEST: This will color the first 3 letters of any ReScript file as a keyword.*) -let parserTest () = Printf.printf "{\"data\":[0,0,3,0,0]}" + let createEmitter () = {buf = Buffer.create 0; lastLine = 0; lastChar = 0} + + let emit ~line ~char ~length ~type_ ?(modifiers = NoModifier) e = + let deltaLine = line - e.lastLine in + let deltaChar = char - e.lastChar in + e.lastLine <- line; + e.lastChar <- char; + if Buffer.length e.buf > 0 then Buffer.add_char e.buf ','; + Buffer.add_string e.buf + (string_of_int deltaLine ^ "," ^ string_of_int deltaChar ^ "," + ^ string_of_int length ^ "," ^ tokenTypeToString type_ ^ "," + ^ tokenModifiersToString modifiers); + () +end + +let semanticTokensTest () = + let emitter = Token.createEmitter () in + emitter |> Token.emit ~line:0 ~char:0 ~length:3 ~type_:Token.Keyword; + emitter |> Token.emit ~line:1 ~char:2 ~length:3 ~type_:Token.Keyword; + Printf.printf "{\"data\":[%s]}" (Buffer.contents emitter.buf) let parser ~path = if Filename.check_suffix path ".res" then ( From c042982fd86d6607dd71edf8057c4f3b1c44dff1 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 11 Mar 2022 11:31:03 +0100 Subject: [PATCH 06/68] Pass current file contents to binary command. --- analysis/src/Cli.ml | 2 +- analysis/src/Commands.ml | 2 +- server/src/server.ts | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/analysis/src/Cli.ml b/analysis/src/Cli.ml index 9077a867f..f9d0afd9b 100644 --- a/analysis/src/Cli.ml +++ b/analysis/src/Cli.ml @@ -70,7 +70,7 @@ let main () = ~col:(int_of_string col) | _ :: "dump" :: files -> Commands.dump files | [_; "documentSymbol"; path] -> Commands.documentSymbol ~path - | [_; "semanticTokens"; _path] -> Commands.semanticTokensTest () + | [_; "semanticTokens"; currentFile] -> Commands.semanticTokensTest ~currentFile | [_; "hover"; path; line; col] -> Commands.hover ~path ~line:(int_of_string line) ~col:(int_of_string col) | [_; "references"; path; line; col] -> diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index 3304449b8..61e61a401 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -314,7 +314,7 @@ module Token = struct () end -let semanticTokensTest () = +let semanticTokensTest ~currentFile = let emitter = Token.createEmitter () in emitter |> Token.emit ~line:0 ~char:0 ~length:3 ~type_:Token.Keyword; emitter |> Token.emit ~line:1 ~char:2 ~length:3 ~type_:Token.Keyword; diff --git a/server/src/server.ts b/server/src/server.ts index 4bb94bc73..48cc39d0f 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -391,11 +391,15 @@ function semanticTokens(msg: p.RequestMessage) { // https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens let params = msg.params as p.SemanticTokensParams; let filePath = fileURLToPath(params.textDocument.uri); + let code = getOpenedFileContent(params.textDocument.uri); + let tmpname = utils.createFileInTempDir(); + fs.writeFileSync(tmpname, code, { encoding: "utf-8" }); let response = utils.runAnalysisCommand( filePath, - ["semanticTokens", filePath], + ["semanticTokens", tmpname], msg ); + fs.unlink(tmpname, () => null); return response; } From c837fc3692c384ca9dda270a2567308502336508 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 11 Mar 2022 11:50:05 +0100 Subject: [PATCH 07/68] Use an ast mapper. --- analysis/src/Commands.ml | 199 ++++++++++++++------------------------- 1 file changed, 73 insertions(+), 126 deletions(-) diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index 61e61a401..c3ba72c70 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -321,6 +321,76 @@ let semanticTokensTest ~currentFile = Printf.printf "{\"data\":[%s]}" (Buffer.contents emitter.buf) let parser ~path = + let jsxName lident = + let rec flatten acc lident = + match lident with + | Longident.Lident txt -> txt :: acc + | Ldot (lident, txt) -> + let acc = if txt = "createElement" then acc else txt :: acc in + flatten acc lident + | _ -> acc + in + match lident with + | Longident.Lident txt -> txt + | _ as lident -> + let segments = flatten [] lident in + segments |> String.concat "." + in + let locToString (loc : Location.t) = + let lineStart, colStart = Utils.tupleOfLexing loc.loc_start in + let lineEnd, colEnd = Utils.tupleOfLexing loc.loc_end in + Printf.sprintf "(%d,%d)->(%d,%d)" lineStart colStart lineEnd colEnd + in + let processTypeArg (coreType : Parsetree.core_type) = + Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) + in + + let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = + match coreType.ptyp_desc with + | Ptyp_constr (_lident, args) -> + args |> List.iter processTypeArg; + Ast_mapper.default_mapper.typ mapper coreType + | _ -> Ast_mapper.default_mapper.typ mapper coreType + in + let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = + match e.pexp_desc with + | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) + when Res_parsetree_viewer.isJsxExpression e -> + let rec isSelfClosing args = + match args with + | [] -> false + | [ + ( Asttypes.Labelled "children", + { + Parsetree.pexp_desc = + Pexp_construct ({txt = Longident.Lident "[]"}, None); + } ); + _; + ] -> + true + | _ :: rest -> isSelfClosing rest + in + Printf.printf "JsxOpen: %s %s\n" (jsxName lident.txt) + (locToString pexp_loc); + (if not (isSelfClosing args) then + let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in + let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in + let size = if lineStart = lineEnd then colEnd - colStart else 0 in + let lineEndWhole, colEndWhole = Utils.tupleOfLexing e.pexp_loc.loc_end in + if size > 0 && colEndWhole > size then + Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" lineEndWhole + (colEndWhole - size - 1) + lineEndWhole (colEndWhole - 1)); + Ast_mapper.default_mapper.expr mapper e + | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e + -> + Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); + Ast_mapper.default_mapper.expr mapper e + | _ -> Ast_mapper.default_mapper.expr mapper e + in + + let mapper = {Ast_mapper.default_mapper with expr; typ} in + if Filename.check_suffix path ".res" then ( let parser = Res_driver.parsingEngine.parseImplementation ~forPrinter:false @@ -330,138 +400,15 @@ let parser ~path = in Printf.printf "structure items:%d diagnostics:%d \n" (List.length structure) (List.length diagnostics); - - let jsxName lident = - let rec flatten acc lident = - match lident with - | Longident.Lident txt -> txt :: acc - | Ldot (lident, txt) -> - let acc = if txt = "createElement" then acc else txt :: acc in - flatten acc lident - | _ -> acc - in - match lident with - | Longident.Lident txt -> txt - | _ as lident -> - let segments = flatten [] lident in - segments |> String.concat "." - in - let locToString (loc : Location.t) = - let lineStart, colStart = Utils.tupleOfLexing loc.loc_start in - let lineEnd, colEnd = Utils.tupleOfLexing loc.loc_end in - Printf.sprintf "(%d,%d)->(%d,%d)" lineStart colStart lineEnd colEnd - in - let rec processExpression (expr : Parsetree.expression) = - match expr.pexp_desc with - | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) - when Res_parsetree_viewer.isJsxExpression expr -> - let rec isSelfClosing args = - match args with - | [] -> false - | [ - ( Asttypes.Labelled "children", - { - Parsetree.pexp_desc = - Pexp_construct ({txt = Longident.Lident "[]"}, None); - } ); - _; - ] -> - true - | _ :: rest -> isSelfClosing rest - in - Printf.printf "JsxOpen: %s %s\n" (jsxName lident.txt) - (locToString pexp_loc); - (if not (isSelfClosing args) then - let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in - let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in - let size = if lineStart = lineEnd then colEnd - colStart else 0 in - let lineEndWhole, colEndWhole = - Utils.tupleOfLexing expr.pexp_loc.loc_end - in - if size > 0 && colEndWhole > size then - Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" lineEndWhole - (colEndWhole - size - 1) - lineEndWhole (colEndWhole - 1)); - args |> List.iter (fun (_lbl, e) -> processExpression e) - | Pexp_apply ({pexp_loc}, args) - when Res_parsetree_viewer.isBinaryExpression expr -> - Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); - args |> List.iter (fun (_lbl, e) -> processExpression e) - | Pexp_apply (f, args) -> - processExpression f; - args |> List.iter (fun (_lbl, e) -> processExpression e) - | Pexp_construct (_lidend, expOpt) -> processExpressionOption expOpt - | Pexp_tuple exprs -> exprs |> List.iter processExpression - | Pexp_ident _ -> () - | Pexp_constant _ -> () - | Pexp_unreachable -> assert false - | Pexp_let (_, _, _) -> assert false - | Pexp_function _ -> assert false - | Pexp_fun (_, _, _, _) -> assert false - | Pexp_match (_, _) -> assert false - | Pexp_try (_, _) -> assert false - | Pexp_variant (_, _) -> assert false - | Pexp_record (_, _) -> assert false - | Pexp_field (_, _) -> assert false - | Pexp_setfield (_, _, _) -> assert false - | Pexp_array _ -> assert false - | Pexp_ifthenelse (_, _, _) -> assert false - | Pexp_sequence (_, _) -> assert false - | Pexp_while (_, _) -> assert false - | Pexp_for (_, _, _, _, _) -> assert false - | Pexp_constraint (_, _) -> assert false - | Pexp_coerce (_, _, _) -> assert false - | Pexp_send (_, _) -> assert false - | Pexp_new _ -> assert false - | Pexp_setinstvar (_, _) -> assert false - | Pexp_override _ -> assert false - | Pexp_letmodule (_, _, _) -> assert false - | Pexp_letexception (_, _) -> assert false - | Pexp_assert _ -> assert false - | Pexp_lazy _ -> assert false - | Pexp_poly (_, _) -> assert false - | Pexp_object _ -> assert false - | Pexp_newtype (_, _) -> assert false - | Pexp_pack _ -> assert false - | Pexp_open (_, _, _) -> assert false - | Pexp_extension _ -> assert false - and processExpressionOption = function - | None -> () - | Some e -> processExpression e - in - - let processValueBinding (binding : Parsetree.value_binding) = - processExpression binding.pvb_expr - in - let rec processTypeArg (coreType : Parsetree.core_type) = - Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc); - processCoreType coreType - and processCoreType (coreType : Parsetree.core_type) = - match coreType.ptyp_desc with - | Ptyp_constr (_lident, args) -> args |> List.iter processTypeArg - | _ -> () - in - let processTypeDeclaration (typeDecl : Parsetree.type_declaration) = - match typeDecl.ptype_manifest with - | Some t -> processCoreType t - | None -> () - in - let processStructureItem (item : Parsetree.structure_item) = - match item.pstr_desc with - | Pstr_value (_recFlag, bindings) -> - bindings |> List.iter processValueBinding - | Pstr_type (_recFlat, typeDecls) -> - typeDecls |> List.iter processTypeDeclaration - | _ -> () - in - structure |> List.iter processStructureItem) + mapper.structure mapper structure |> ignore) else let parser = Res_driver.parsingEngine.parseInterface ~forPrinter:false in let {Res_driver.parsetree = signature; diagnostics} = parser ~filename:path in Printf.printf "signature items:%d diagnostics:%d \n" (List.length signature) - (List.length diagnostics) + (List.length diagnostics); + mapper.signature mapper signature |> ignore let test ~path = Uri2.stripPath := true; From 097872c862c07bf44e2b43e0c482452b4894543d Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 11 Mar 2022 13:59:12 +0100 Subject: [PATCH 08/68] Semantic highlighting for JSX open and close. --- analysis/src/Cli.ml | 3 +- analysis/src/Commands.ml | 132 +------------------------- analysis/src/SemanticTokens.ml | 166 +++++++++++++++++++++++++++++++++ server/src/server.ts | 3 +- 4 files changed, 173 insertions(+), 131 deletions(-) create mode 100644 analysis/src/SemanticTokens.ml diff --git a/analysis/src/Cli.ml b/analysis/src/Cli.ml index f9d0afd9b..cf56d3bbc 100644 --- a/analysis/src/Cli.ml +++ b/analysis/src/Cli.ml @@ -70,7 +70,8 @@ let main () = ~col:(int_of_string col) | _ :: "dump" :: files -> Commands.dump files | [_; "documentSymbol"; path] -> Commands.documentSymbol ~path - | [_; "semanticTokens"; currentFile] -> Commands.semanticTokensTest ~currentFile + | [_; "semanticTokens"; currentFile] -> + SemanticTokens.testCommand ~currentFile | [_; "hover"; path; line; col] -> Commands.hover ~path ~line:(int_of_string line) ~col:(int_of_string col) | [_; "references"; path; line; col] -> diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index c3ba72c70..a9945d469 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -282,134 +282,6 @@ let rename ~path ~line ~col ~newName = in print_endline result -module Token = struct - type legend = {tokenTypes : string array; tokenModifiers : string array} - - (* This needs to stay synced with the same legend in `server.ts` *) - (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword - type tokenModifiers = NoModifier - - let tokenTypeToString = function Keyword -> "0" - let tokenModifiersToString = function NoModifier -> "0" - - type emitter = { - buf : Buffer.t; - mutable lastLine : int; - mutable lastChar : int; - } - - let createEmitter () = {buf = Buffer.create 0; lastLine = 0; lastChar = 0} - - let emit ~line ~char ~length ~type_ ?(modifiers = NoModifier) e = - let deltaLine = line - e.lastLine in - let deltaChar = char - e.lastChar in - e.lastLine <- line; - e.lastChar <- char; - if Buffer.length e.buf > 0 then Buffer.add_char e.buf ','; - Buffer.add_string e.buf - (string_of_int deltaLine ^ "," ^ string_of_int deltaChar ^ "," - ^ string_of_int length ^ "," ^ tokenTypeToString type_ ^ "," - ^ tokenModifiersToString modifiers); - () -end - -let semanticTokensTest ~currentFile = - let emitter = Token.createEmitter () in - emitter |> Token.emit ~line:0 ~char:0 ~length:3 ~type_:Token.Keyword; - emitter |> Token.emit ~line:1 ~char:2 ~length:3 ~type_:Token.Keyword; - Printf.printf "{\"data\":[%s]}" (Buffer.contents emitter.buf) - -let parser ~path = - let jsxName lident = - let rec flatten acc lident = - match lident with - | Longident.Lident txt -> txt :: acc - | Ldot (lident, txt) -> - let acc = if txt = "createElement" then acc else txt :: acc in - flatten acc lident - | _ -> acc - in - match lident with - | Longident.Lident txt -> txt - | _ as lident -> - let segments = flatten [] lident in - segments |> String.concat "." - in - let locToString (loc : Location.t) = - let lineStart, colStart = Utils.tupleOfLexing loc.loc_start in - let lineEnd, colEnd = Utils.tupleOfLexing loc.loc_end in - Printf.sprintf "(%d,%d)->(%d,%d)" lineStart colStart lineEnd colEnd - in - let processTypeArg (coreType : Parsetree.core_type) = - Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) - in - - let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = - match coreType.ptyp_desc with - | Ptyp_constr (_lident, args) -> - args |> List.iter processTypeArg; - Ast_mapper.default_mapper.typ mapper coreType - | _ -> Ast_mapper.default_mapper.typ mapper coreType - in - let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = - match e.pexp_desc with - | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) - when Res_parsetree_viewer.isJsxExpression e -> - let rec isSelfClosing args = - match args with - | [] -> false - | [ - ( Asttypes.Labelled "children", - { - Parsetree.pexp_desc = - Pexp_construct ({txt = Longident.Lident "[]"}, None); - } ); - _; - ] -> - true - | _ :: rest -> isSelfClosing rest - in - Printf.printf "JsxOpen: %s %s\n" (jsxName lident.txt) - (locToString pexp_loc); - (if not (isSelfClosing args) then - let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in - let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in - let size = if lineStart = lineEnd then colEnd - colStart else 0 in - let lineEndWhole, colEndWhole = Utils.tupleOfLexing e.pexp_loc.loc_end in - if size > 0 && colEndWhole > size then - Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" lineEndWhole - (colEndWhole - size - 1) - lineEndWhole (colEndWhole - 1)); - Ast_mapper.default_mapper.expr mapper e - | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e - -> - Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); - Ast_mapper.default_mapper.expr mapper e - | _ -> Ast_mapper.default_mapper.expr mapper e - in - - let mapper = {Ast_mapper.default_mapper with expr; typ} in - - if Filename.check_suffix path ".res" then ( - let parser = - Res_driver.parsingEngine.parseImplementation ~forPrinter:false - in - let {Res_driver.parsetree = structure; diagnostics} = - parser ~filename:path - in - Printf.printf "structure items:%d diagnostics:%d \n" (List.length structure) - (List.length diagnostics); - mapper.structure mapper structure |> ignore) - else - let parser = Res_driver.parsingEngine.parseInterface ~forPrinter:false in - let {Res_driver.parsetree = signature; diagnostics} = - parser ~filename:path - in - Printf.printf "signature items:%d diagnostics:%d \n" (List.length signature) - (List.length diagnostics); - mapper.signature mapper signature |> ignore - let test ~path = Uri2.stripPath := true; match Files.readFile path with @@ -477,7 +349,9 @@ let test ~path = Sys.remove currentFile | "par" -> print_endline ("Parse " ^ path); - parser ~path + SemanticTokens.parser ~debug:true + ~emitter:(SemanticTokens.Token.createEmitter ()) + ~path | _ -> ()); print_newline ()) in diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml new file mode 100644 index 000000000..b5d4c43a6 --- /dev/null +++ b/analysis/src/SemanticTokens.ml @@ -0,0 +1,166 @@ +module Token = struct + type legend = {tokenTypes : string array; tokenModifiers : string array} + + (* This needs to stay synced with the same legend in `server.ts` *) + (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) + type tokenType = Keyword + type tokenModifiers = NoModifier + + let tokenTypeToString = function Keyword -> "0" + let tokenModifiersToString = function NoModifier -> "0" + + type token = int * int * int * tokenType * tokenModifiers + + type emitter = { + mutable tokens : token list; + mutable lastLine : int; + mutable lastChar : int; + } + + let createEmitter () = {tokens = []; lastLine = 0; lastChar = 0} + + let add ~line ~char ~length ~type_ ?(modifiers = NoModifier) e = + e.tokens <- (line, char, length, type_, modifiers) :: e.tokens + + let emitToken buf (line, char, length, type_, modifiers) e = + let deltaLine = line - e.lastLine in + let deltaChar = if deltaLine = 0 then char - e.lastChar else char in + e.lastLine <- line; + e.lastChar <- char; + if Buffer.length buf > 0 then Buffer.add_char buf ','; + Buffer.add_string buf + (string_of_int deltaLine ^ "," ^ string_of_int deltaChar ^ "," + ^ string_of_int length ^ "," ^ tokenTypeToString type_ ^ "," + ^ tokenModifiersToString modifiers) + + let emit e = + let sortedTokens = + e.tokens + |> List.sort (fun (l1, c1, _, _, _) (l2, c2, _, _, _) -> + if l1 = l2 then compare c1 c2 else compare l1 l2) + in + let buf = Buffer.create 1 in + sortedTokens |> List.iter (fun t -> e |> emitToken buf t); + Buffer.contents buf +end + +let locToPositions (loc : Location.t) = + (Utils.tupleOfLexing loc.loc_start, Utils.tupleOfLexing loc.loc_end) + +let posToString (loc, col) = Printf.sprintf "(%d,%d)" loc col + +let locToString (loc : Location.t) = + let posStart, posEnd = locToPositions loc in + Printf.sprintf "%s->%s" (posToString posStart) (posToString posEnd) + +let emitFromLoc loc emitter = + let posStart, posEnd = locToPositions loc in + let length = + if fst posStart = fst posEnd then snd posEnd - snd posStart else 0 + in + if length > 0 then + emitter + |> Token.add ~line:(fst posStart) ~char:(snd posStart) ~length + ~type_:Token.Keyword + +let parser ~debug ~emitter ~path = + let jsxName lident = + let rec flatten acc lident = + match lident with + | Longident.Lident txt -> txt :: acc + | Ldot (lident, txt) -> + let acc = if txt = "createElement" then acc else txt :: acc in + flatten acc lident + | _ -> acc + in + match lident with + | Longident.Lident txt -> txt + | _ as lident -> + let segments = flatten [] lident in + segments |> String.concat "." + in + let processTypeArg (coreType : Parsetree.core_type) = + if debug then Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) + in + + let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = + match coreType.ptyp_desc with + | Ptyp_constr (_lident, args) -> + args |> List.iter processTypeArg; + Ast_mapper.default_mapper.typ mapper coreType + | _ -> Ast_mapper.default_mapper.typ mapper coreType + in + let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = + match e.pexp_desc with + | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) + when Res_parsetree_viewer.isJsxExpression e -> + let rec isSelfClosing args = + match args with + | [] -> false + | [ + ( Asttypes.Labelled "children", + { + Parsetree.pexp_desc = + Pexp_construct ({txt = Longident.Lident "[]"}, None); + } ); + _; + ] -> + true + | _ :: rest -> isSelfClosing rest + in + + if debug then + Printf.printf "JsxOpen: %s %s\n" (jsxName lident.txt) + (locToString pexp_loc); + emitter |> emitFromLoc pexp_loc; + + (if not (isSelfClosing args) then + let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in + let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in + let length = if lineStart = lineEnd then colEnd - colStart else 0 in + let lineEndWhole, colEndWhole = Utils.tupleOfLexing e.pexp_loc.loc_end in + if length > 0 && colEndWhole > length then ( + let line = lineEndWhole in + let char = colEndWhole - length - 1 in + + if debug then + Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" line char lineEndWhole + (colEndWhole - 1); + emitter |> Token.add ~line ~char ~length ~type_:Token.Keyword)); + + Ast_mapper.default_mapper.expr mapper e + | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e + -> + if debug then Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); + Ast_mapper.default_mapper.expr mapper e + | _ -> Ast_mapper.default_mapper.expr mapper e + in + + let mapper = {Ast_mapper.default_mapper with expr; typ} in + + if Filename.check_suffix path ".res" then ( + let parser = + Res_driver.parsingEngine.parseImplementation ~forPrinter:false + in + let {Res_driver.parsetree = structure; diagnostics} = + parser ~filename:path + in + if debug then + Printf.printf "structure items:%d diagnostics:%d \n" + (List.length structure) (List.length diagnostics); + mapper.structure mapper structure |> ignore) + else + let parser = Res_driver.parsingEngine.parseInterface ~forPrinter:false in + let {Res_driver.parsetree = signature; diagnostics} = + parser ~filename:path + in + if debug then + Printf.printf "signature items:%d diagnostics:%d \n" + (List.length signature) (List.length diagnostics); + mapper.signature mapper signature |> ignore + +let testCommand ~currentFile = + let emitter = Token.createEmitter () in + parser ~emitter ~debug:false ~path:currentFile; + (* emitter |> Token.add ~line:0 ~char:0 ~length:3 ~type_:Token.Keyword; *) + Printf.printf "{\"data\":[%s]}" (Token.emit emitter) diff --git a/server/src/server.ts b/server/src/server.ts index 48cc39d0f..c5ab3d767 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -392,7 +392,8 @@ function semanticTokens(msg: p.RequestMessage) { let params = msg.params as p.SemanticTokensParams; let filePath = fileURLToPath(params.textDocument.uri); let code = getOpenedFileContent(params.textDocument.uri); - let tmpname = utils.createFileInTempDir(); + let extension = path.extname(params.textDocument.uri); + let tmpname = utils.createFileInTempDir(extension); fs.writeFileSync(tmpname, code, { encoding: "utf-8" }); let response = utils.runAnalysisCommand( filePath, From c3fa07696b76a93b83a9fd4bddc40b5d5c72669e Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Sat, 12 Mar 2022 13:06:43 +0100 Subject: [PATCH 09/68] Emit variables. --- analysis/src/SemanticTokens.ml | 61 ++++++++++++++-------- analysis/tests/src/expected/Parser.res.txt | 7 +++ 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index b5d4c43a6..36051613d 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -53,15 +53,30 @@ let locToString (loc : Location.t) = let posStart, posEnd = locToPositions loc in Printf.sprintf "%s->%s" (posToString posStart) (posToString posEnd) -let emitFromLoc loc emitter = - let posStart, posEnd = locToPositions loc in +let emitFromPos posStart posEnd ~type_ emitter = let length = if fst posStart = fst posEnd then snd posEnd - snd posStart else 0 in if length > 0 then emitter - |> Token.add ~line:(fst posStart) ~char:(snd posStart) ~length - ~type_:Token.Keyword + |> Token.add ~line:(fst posStart) ~char:(snd posStart) ~length ~type_ + +let emitFromLoc ~loc ~type_ emitter = + let posStart, posEnd = locToPositions loc in + emitter |> emitFromPos posStart posEnd ~type_ + +let emitJsxOpen ~id ~debug ~loc emitter = + if debug then Printf.printf "JsxOpen: %s %s\n" id (locToString loc); + emitter |> emitFromLoc ~loc ~type_:Token.Keyword + +let emitVariable ~id ~debug ~loc emitter = + if debug then Printf.printf "Variable: %s %s\n" id (locToString loc); + emitter |> emitFromLoc ~loc ~type_:Token.Keyword + +let emitJsxClose ~debug ~posStart ~posEnd emitter = + let l1, c1 = posStart and l2, c2 = posEnd in + if debug then Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" l1 c1 l2 c2; + emitter |> emitFromPos posStart posEnd ~type_:Token.Keyword let parser ~debug ~emitter ~path = let jsxName lident = @@ -82,7 +97,10 @@ let parser ~debug ~emitter ~path = let processTypeArg (coreType : Parsetree.core_type) = if debug then Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) in - + let isLowercaseId id = + let c = id.[0] in + c == '_' || (c >= 'a' && c <= 'z') + in let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = match coreType.ptyp_desc with | Ptyp_constr (_lident, args) -> @@ -90,8 +108,18 @@ let parser ~debug ~emitter ~path = Ast_mapper.default_mapper.typ mapper coreType | _ -> Ast_mapper.default_mapper.typ mapper coreType in + let pat (mapper : Ast_mapper.mapper) (p : Parsetree.pattern) = + match p.ppat_desc with + | Ppat_var {loc; txt = id} -> + if isLowercaseId id then emitter |> emitVariable ~id ~debug ~loc; + Ast_mapper.default_mapper.pat mapper p + | _ -> Ast_mapper.default_mapper.pat mapper p + in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = match e.pexp_desc with + | Pexp_ident {txt = Lident id; loc} -> + if isLowercaseId id then emitter |> emitVariable ~id ~debug ~loc; + Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) when Res_parsetree_viewer.isJsxExpression e -> let rec isSelfClosing args = @@ -108,26 +136,17 @@ let parser ~debug ~emitter ~path = true | _ :: rest -> isSelfClosing rest in - - if debug then - Printf.printf "JsxOpen: %s %s\n" (jsxName lident.txt) - (locToString pexp_loc); - emitter |> emitFromLoc pexp_loc; - + emitter |> emitJsxOpen ~id:(jsxName lident.txt) ~debug ~loc:pexp_loc; (if not (isSelfClosing args) then let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in let length = if lineStart = lineEnd then colEnd - colStart else 0 in let lineEndWhole, colEndWhole = Utils.tupleOfLexing e.pexp_loc.loc_end in - if length > 0 && colEndWhole > length then ( - let line = lineEndWhole in - let char = colEndWhole - length - 1 in - - if debug then - Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" line char lineEndWhole - (colEndWhole - 1); - emitter |> Token.add ~line ~char ~length ~type_:Token.Keyword)); - + if length > 0 && colEndWhole > length then + emitter + |> emitJsxClose ~debug + ~posStart:(lineEndWhole, colEndWhole - length - 1) + ~posEnd:(lineEndWhole, colEndWhole - 1)); Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e -> @@ -136,7 +155,7 @@ let parser ~debug ~emitter ~path = | _ -> Ast_mapper.default_mapper.expr mapper e in - let mapper = {Ast_mapper.default_mapper with expr; typ} in + let mapper = {Ast_mapper.default_mapper with expr; pat; typ} in if Filename.check_suffix path ".res" then ( let parser = diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index b6fbfec3f..5d11b373f 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,12 +1,19 @@ Parse tests/src/Parser.res structure items:10 diagnostics:0 JsxOpen: Component (4,10)->(4,19) +Variable: _c (4,4)->(4,6) JsxOpen: M.C (6,11)->(6,14) +Variable: _mc (6,4)->(6,7) JsxOpen: div (8,10)->(8,13) +Variable: div (8,10)->(8,13) +Variable: _d (8,4)->(8,6) JsxOpen: div (11,3)->(11,6) JsxClose: (16,4)->(16,7) JsxOpen: div (13,5)->(13,8) JsxClose: (13,34)->(13,37) +Variable: div (13,5)->(13,8) +Variable: div (11,3)->(11,6) +Variable: _d2 (10,4)->(10,7) TypeArg: (25,2)->(28,3) TypeArg: (26,4)->(26,50) TypeArg: (27,4)->(27,53) From d1537db13048335854914ecc0dc28c46086b9de0 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Sat, 12 Mar 2022 13:12:11 +0100 Subject: [PATCH 10/68] Add token type "variable". --- analysis/src/SemanticTokens.ml | 6 +++--- server/src/server.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 36051613d..51dd8d712 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -3,10 +3,10 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword + type tokenType = Keyword | Variable type tokenModifiers = NoModifier - let tokenTypeToString = function Keyword -> "0" + let tokenTypeToString = function Keyword -> "0" | Variable -> "1" let tokenModifiersToString = function NoModifier -> "0" type token = int * int * int * tokenType * tokenModifiers @@ -71,7 +71,7 @@ let emitJsxOpen ~id ~debug ~loc emitter = let emitVariable ~id ~debug ~loc emitter = if debug then Printf.printf "Variable: %s %s\n" id (locToString loc); - emitter |> emitFromLoc ~loc ~type_:Token.Keyword + emitter |> emitFromLoc ~loc ~type_:Token.Variable let emitJsxClose ~debug ~posStart ~posEnd emitter = let l1, c1 = posStart and l2, c2 = posEnd in diff --git a/server/src/server.ts b/server/src/server.ts index c5ab3d767..880b168c8 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -759,7 +759,7 @@ function onMessage(msg: m.Message) { // documentSymbolProvider: true, completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, semanticTokensProvider: { - legend: { tokenTypes: ["keyword"], tokenModifiers: [] }, + legend: { tokenTypes: ["keyword", "variable"], tokenModifiers: [] }, documentSelector: null, // TODO: Support range for full, and add delta support full: true, From 763494fbac53a108546ac11ee00adfb533499ff2 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Sat, 12 Mar 2022 13:26:31 +0100 Subject: [PATCH 11/68] Emitting types too. --- analysis/src/SemanticTokens.ml | 27 ++++++++++++++++++---- analysis/tests/src/expected/Parser.res.txt | 10 ++++++++ server/src/server.ts | 5 +++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 51dd8d712..d1d790b68 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -3,10 +3,14 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword | Variable + type tokenType = Keyword | Variable | Type type tokenModifiers = NoModifier - let tokenTypeToString = function Keyword -> "0" | Variable -> "1" + let tokenTypeToString = function + | Keyword -> "0" + | Variable -> "1" + | Type -> "2" + let tokenModifiersToString = function NoModifier -> "0" type token = int * int * int * tokenType * tokenModifiers @@ -73,6 +77,10 @@ let emitVariable ~id ~debug ~loc emitter = if debug then Printf.printf "Variable: %s %s\n" id (locToString loc); emitter |> emitFromLoc ~loc ~type_:Token.Variable +let emitType ~id ~debug ~loc emitter = + if debug then Printf.printf "Type: %s %s\n" id (locToString loc); + emitter |> emitFromLoc ~loc ~type_:Token.Type + let emitJsxClose ~debug ~posStart ~posEnd emitter = let l1, c1 = posStart and l2, c2 = posEnd in if debug then Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" l1 c1 l2 c2; @@ -103,11 +111,20 @@ let parser ~debug ~emitter ~path = in let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = match coreType.ptyp_desc with - | Ptyp_constr (_lident, args) -> + | Ptyp_constr ({txt; loc}, args) -> + (match txt with + | Lident id -> emitter |> emitType ~id ~debug ~loc + | _ -> ()); args |> List.iter processTypeArg; Ast_mapper.default_mapper.typ mapper coreType | _ -> Ast_mapper.default_mapper.typ mapper coreType in + let type_declaration (mapper : Ast_mapper.mapper) + (tydecl : Parsetree.type_declaration) = + emitter + |> emitType ~id:tydecl.ptype_name.txt ~debug ~loc:tydecl.ptype_name.loc; + Ast_mapper.default_mapper.type_declaration mapper tydecl + in let pat (mapper : Ast_mapper.mapper) (p : Parsetree.pattern) = match p.ppat_desc with | Ppat_var {loc; txt = id} -> @@ -155,7 +172,9 @@ let parser ~debug ~emitter ~path = | _ -> Ast_mapper.default_mapper.expr mapper e in - let mapper = {Ast_mapper.default_mapper with expr; pat; typ} in + let mapper = + {Ast_mapper.default_mapper with expr; pat; typ; type_declaration} + in if Filename.check_suffix path ".res" then ( let parser = diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 5d11b373f..abff69a1b 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -14,9 +14,19 @@ JsxClose: (13,34)->(13,37) Variable: div (13,5)->(13,8) Variable: div (11,3)->(11,6) Variable: _d2 (10,4)->(10,7) +Type: pair (18,5)->(18,9) +Type: looooooooooooooooooooooooooooooooooooooong_int (20,5)->(20,51) +Type: int (20,54)->(20,57) +Type: looooooooooooooooooooooooooooooooooooooong_string (22,5)->(22,54) +Type: string (22,57)->(22,63) +Type: pairIntString (24,5)->(24,18) +Type: list (24,21)->(24,25) TypeArg: (25,2)->(28,3) +Type: pair (25,2)->(25,6) TypeArg: (26,4)->(26,50) TypeArg: (27,4)->(27,53) +Type: looooooooooooooooooooooooooooooooooooooong_int (26,4)->(26,50) +Type: looooooooooooooooooooooooooooooooooooooong_string (27,4)->(27,53) BinaryExp: (31,14)->(31,16) BinaryExp: (31,10)->(31,11) BinaryExp: (31,19)->(31,20) diff --git a/server/src/server.ts b/server/src/server.ts index 880b168c8..4d8952268 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -759,7 +759,10 @@ function onMessage(msg: m.Message) { // documentSymbolProvider: true, completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, semanticTokensProvider: { - legend: { tokenTypes: ["keyword", "variable"], tokenModifiers: [] }, + legend: { + tokenTypes: ["keyword", "variable", "type"], + tokenModifiers: [], + }, documentSelector: null, // TODO: Support range for full, and add delta support full: true, From 4653dda33e00322712aff0961efb40efe42fe08c Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Sat, 12 Mar 2022 14:00:01 +0100 Subject: [PATCH 12/68] add sample highlight files --- .../src/syntax/sample-highlighting.res | 43 ++++++++++++++++++ .../src/syntax/sample-highlighting.rs | 31 +++++++++++++ .../src/syntax/sample-highlighting.tsx | 44 +++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 analysis/examples/example-project/src/syntax/sample-highlighting.res create mode 100644 analysis/examples/example-project/src/syntax/sample-highlighting.rs create mode 100644 analysis/examples/example-project/src/syntax/sample-highlighting.tsx diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.res b/analysis/examples/example-project/src/syntax/sample-highlighting.res new file mode 100644 index 000000000..814c7faed --- /dev/null +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.res @@ -0,0 +1,43 @@ +// Bindings +let numberBinding = 123 + +let someFunction = (param: int): int => { + let innerBinding = param + 2 + innerBinding +} + +// Types +type someRecord<'typeParameter> = { + someField: int, + someOtherField: string, + theParam: typeParameter, +} + +type someEnum = + | SomeMember + | AnotherMember + | SomeMemberWithPayload(someRecord) + +// Destructuring +let destructuring = () => { + let someVar = (1, 2, 3) + let (one, two, three) = someVar + let someObj: someRecord = { + someField: 1, + someOtherField: "hello", + theParam: 2, + } + let {someField, someOtherField, theParam} = someObj + + someField +} + +// JSX +module SomeComponent = { + @react.component + let make = () => { + React.null + } +} + +let jsx =
diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.rs b/analysis/examples/example-project/src/syntax/sample-highlighting.rs new file mode 100644 index 000000000..7131ba600 --- /dev/null +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.rs @@ -0,0 +1,31 @@ +// Bindings +fn some_function(param: usize) -> usize { + let innerBinding = param + 2; + innerBinding +} + +// Types +struct someRecord { + someField: usize, + someOtherField: String, + theParam: typeParameter, +} + +enum someEnum { + SomeMember, + AnotherMember, + SomeMemberWithPayload(someRecord), +} + +// Destructuring +fn destructuring() -> usize { + let someVar = (1, 2, 3); + let (one, two, three) = someVar; + let someObj = someRecord:: { + someField: 1, + someOtherField: String::new("HEllo"), + theParam: 2, + }; + + someObj.someField +} diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx new file mode 100644 index 000000000..1919f6fc4 --- /dev/null +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -0,0 +1,44 @@ +// Bindings +let numberBinding = 123; + +let someFunction = (param: number): number => { + let innerBinding = param + 2; + return innerBinding; +}; + +// Types +type someRecord = { + someField: number; + someOtherField: string; + theParam: typeParameter; +}; + +enum someEnum { + SomeMember, + AnotherMember, +} + +// Destructuring +let destructuring = () => { + let someVar = [1, 2, 3]; + let [one, two, three] = someVar; + let someObj: someRecord = { + someField: 1, + someOtherField: "hello", + theParam: 2, + }; + let { someField, someOtherField, theParam } = someObj; + + return someField; +}; + +// JSX +const SomeComponent = () => { + return null; +}; + +let jsx = ( +
+ +
+); From 071b27388d6c3a87d60d8e3728972337ec9291ca Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Sat, 12 Mar 2022 14:00:30 +0100 Subject: [PATCH 13/68] add explicit semantic token for JSX tag --- analysis/src/SemanticTokens.ml | 7 ++++--- package.json | 7 +++++++ server/src/server.ts | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index d1d790b68..7c7d6ec2b 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -3,13 +3,14 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword | Variable | Type + type tokenType = Keyword | Variable | Type | JsxTag type tokenModifiers = NoModifier let tokenTypeToString = function | Keyword -> "0" | Variable -> "1" | Type -> "2" + | JsxTag -> "3" let tokenModifiersToString = function NoModifier -> "0" @@ -71,7 +72,7 @@ let emitFromLoc ~loc ~type_ emitter = let emitJsxOpen ~id ~debug ~loc emitter = if debug then Printf.printf "JsxOpen: %s %s\n" id (locToString loc); - emitter |> emitFromLoc ~loc ~type_:Token.Keyword + emitter |> emitFromLoc ~loc ~type_:Token.JsxTag let emitVariable ~id ~debug ~loc emitter = if debug then Printf.printf "Variable: %s %s\n" id (locToString loc); @@ -84,7 +85,7 @@ let emitType ~id ~debug ~loc emitter = let emitJsxClose ~debug ~posStart ~posEnd emitter = let l1, c1 = posStart and l2, c2 = posEnd in if debug then Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" l1 c1 l2 c2; - emitter |> emitFromPos posStart posEnd ~type_:Token.Keyword + emitter |> emitFromPos posStart posEnd ~type_:Token.JsxTag let parser ~debug ~emitter ~path = let jsxName lident = diff --git a/package.json b/package.json index 637b453b1..b8707b19a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,13 @@ ], "main": "./client/out/extension", "contributes": { + "semanticTokenScopes": [ + { + "scopes": { + "jsx-tag": ["entity.name.tag"] + } + } + ], "jsonValidation": [ { "fileMatch": "bsconfig.json", diff --git a/server/src/server.ts b/server/src/server.ts index 4d8952268..d6962b1e8 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -760,7 +760,7 @@ function onMessage(msg: m.Message) { completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, semanticTokensProvider: { legend: { - tokenTypes: ["keyword", "variable", "type"], + tokenTypes: ["keyword", "variable", "type", "jsx-tag"], tokenModifiers: [], }, documentSelector: null, From a19747681349b9029a5cd72534cf435c7a8a15ac Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Sat, 12 Mar 2022 20:16:17 +0100 Subject: [PATCH 14/68] Emit long idents in expressions, patterns, and jsx labels, uniformly. With upper case ids, emit Token.JsxTag With lower case ids, emit Token.Variable Still need to do module declarations and expressions (and e.g. open M etc). --- analysis/src/SemanticTokens.ml | 106 ++++++++++++++------- analysis/tests/src/expected/Parser.res.txt | 34 ++++--- 2 files changed, 90 insertions(+), 50 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 7c7d6ec2b..232ec296d 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -58,6 +58,14 @@ let locToString (loc : Location.t) = let posStart, posEnd = locToPositions loc in Printf.sprintf "%s->%s" (posToString posStart) (posToString posEnd) +let isLowercaseId id = + let c = id.[0] in + c == '_' || (c >= 'a' && c <= 'z') + +let isUppercaseId id = + let c = id.[0] in + c >= 'A' && c <= 'Z' + let emitFromPos posStart posEnd ~type_ emitter = let length = if fst posStart = fst posEnd then snd posEnd - snd posStart else 0 @@ -70,46 +78,64 @@ let emitFromLoc ~loc ~type_ emitter = let posStart, posEnd = locToPositions loc in emitter |> emitFromPos posStart posEnd ~type_ -let emitJsxOpen ~id ~debug ~loc emitter = - if debug then Printf.printf "JsxOpen: %s %s\n" id (locToString loc); - emitter |> emitFromLoc ~loc ~type_:Token.JsxTag +let emitLongident ~backwards ~pos ~jsx ~lid ~debug emitter = + let rec flatten acc lid = + match lid with + | Longident.Lident txt -> txt :: acc + | Ldot (lid, txt) -> + let acc = if jsx && txt = "createElement" then acc else txt :: acc in + flatten acc lid + | _ -> acc + in + let rec loop pos segments = + match segments with + | [id] when isUppercaseId id || isLowercaseId id -> + if debug then Printf.printf "Lident: %s %s\n" id (posToString pos); + emitter + |> emitFromPos pos + (fst pos, snd pos + String.length id) + ~type_:(if isUppercaseId id then Token.JsxTag else Token.Variable) + | id :: segments when isUppercaseId id || isLowercaseId id -> + if debug then Printf.printf "Ldot: %s %s\n" id (posToString pos); + let length = String.length id in + emitter + |> emitFromPos pos + (fst pos, snd pos + length) + ~type_:(if isUppercaseId id then Token.JsxTag else Token.Variable); + loop (fst pos, snd pos + length + 1) segments + | _ -> () + in + let segments = flatten [] lid in + let segments = if backwards then List.rev segments else segments in + if backwards then ( + let totalLength = segments |> String.concat "." |> String.length in + if snd pos >= totalLength then + loop (fst pos, snd pos - totalLength) segments) + else loop pos segments let emitVariable ~id ~debug ~loc emitter = - if debug then Printf.printf "Variable: %s %s\n" id (locToString loc); - emitter |> emitFromLoc ~loc ~type_:Token.Variable + emitter + |> emitLongident ~backwards:false + ~pos:(Utils.tupleOfLexing loc.Location.loc_start) + ~jsx:false ~lid:(Longident.Lident id) ~debug + +let emitJsxOpen ~lid ~debug ~loc emitter = + emitter + |> emitLongident ~backwards:false + ~pos:(Utils.tupleOfLexing loc.Location.loc_start) + ~lid ~jsx:true ~debug + +let emitJsxClose ~lid ~debug ~pos emitter = + emitter |> emitLongident ~backwards:true ~pos ~lid ~jsx:true ~debug let emitType ~id ~debug ~loc emitter = if debug then Printf.printf "Type: %s %s\n" id (locToString loc); emitter |> emitFromLoc ~loc ~type_:Token.Type -let emitJsxClose ~debug ~posStart ~posEnd emitter = - let l1, c1 = posStart and l2, c2 = posEnd in - if debug then Printf.printf "JsxClose: (%d,%d)->(%d,%d)\n" l1 c1 l2 c2; - emitter |> emitFromPos posStart posEnd ~type_:Token.JsxTag - let parser ~debug ~emitter ~path = - let jsxName lident = - let rec flatten acc lident = - match lident with - | Longident.Lident txt -> txt :: acc - | Ldot (lident, txt) -> - let acc = if txt = "createElement" then acc else txt :: acc in - flatten acc lident - | _ -> acc - in - match lident with - | Longident.Lident txt -> txt - | _ as lident -> - let segments = flatten [] lident in - segments |> String.concat "." - in let processTypeArg (coreType : Parsetree.core_type) = if debug then Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) in - let isLowercaseId id = - let c = id.[0] in - c == '_' || (c >= 'a' && c <= 'z') - in let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = match coreType.ptyp_desc with | Ptyp_constr ({txt; loc}, args) -> @@ -135,8 +161,11 @@ let parser ~debug ~emitter ~path = in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = match e.pexp_desc with - | Pexp_ident {txt = Lident id; loc} -> - if isLowercaseId id then emitter |> emitVariable ~id ~debug ~loc; + | Pexp_ident {txt = lid; loc} -> + emitter + |> emitLongident ~backwards:false + ~pos:(Utils.tupleOfLexing loc.loc_start) + ~lid ~jsx:false ~debug; Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) when Res_parsetree_viewer.isJsxExpression e -> @@ -154,7 +183,7 @@ let parser ~debug ~emitter ~path = true | _ :: rest -> isSelfClosing rest in - emitter |> emitJsxOpen ~id:(jsxName lident.txt) ~debug ~loc:pexp_loc; + emitter |> emitJsxOpen ~lid:lident.txt ~debug ~loc:pexp_loc; (if not (isSelfClosing args) then let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in @@ -162,10 +191,15 @@ let parser ~debug ~emitter ~path = let lineEndWhole, colEndWhole = Utils.tupleOfLexing e.pexp_loc.loc_end in if length > 0 && colEndWhole > length then emitter - |> emitJsxClose ~debug - ~posStart:(lineEndWhole, colEndWhole - length - 1) - ~posEnd:(lineEndWhole, colEndWhole - 1)); - Ast_mapper.default_mapper.expr mapper e + |> emitJsxClose ~debug ~lid:lident.txt + ~pos:(lineEndWhole, colEndWhole - 1)); + (* only process again arguments, not the jsx label *) + let _ = + args + |> List.map (fun (_lbl, arg) -> + Ast_mapper.default_mapper.expr mapper arg) + in + e | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e -> if debug then Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index abff69a1b..58dc57c4a 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,19 +1,25 @@ Parse tests/src/Parser.res structure items:10 diagnostics:0 -JsxOpen: Component (4,10)->(4,19) -Variable: _c (4,4)->(4,6) -JsxOpen: M.C (6,11)->(6,14) -Variable: _mc (6,4)->(6,7) -JsxOpen: div (8,10)->(8,13) -Variable: div (8,10)->(8,13) -Variable: _d (8,4)->(8,6) -JsxOpen: div (11,3)->(11,6) -JsxClose: (16,4)->(16,7) -JsxOpen: div (13,5)->(13,8) -JsxClose: (13,34)->(13,37) -Variable: div (13,5)->(13,8) -Variable: div (11,3)->(11,6) -Variable: _d2 (10,4)->(10,7) +Lident: Component (4,10) +Lident: _c (4,4) +Ldot: M (6,11) +Lident: C (6,13) +Lident: _mc (6,4) +Lident: div (8,10) +Lident: _d (8,4) +Lident: div (11,3) +Lident: div (16,4) +Ldot: React (12,5) +Lident: string (12,11) +Lident: div (13,5) +Lident: div (13,34) +Ldot: React (13,11) +Lident: string (13,17) +Ldot: React (14,5) +Lident: string (14,11) +Ldot: React (15,5) +Lident: string (15,11) +Lident: _d2 (10,4) Type: pair (18,5)->(18,9) Type: looooooooooooooooooooooooooooooooooooooong_int (20,5)->(20,51) Type: int (20,54)->(20,57) From a66eb039cb8109cf61210af9cc623d39f1557f33 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 09:09:14 +0100 Subject: [PATCH 15/68] Rename JsxTag to Module. --- analysis/src/SemanticTokens.ml | 8 ++++---- package.json | 2 +- server/src/server.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 232ec296d..dec4fd63f 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -3,14 +3,14 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword | Variable | Type | JsxTag + type tokenType = Keyword | Variable | Type | Module type tokenModifiers = NoModifier let tokenTypeToString = function | Keyword -> "0" | Variable -> "1" | Type -> "2" - | JsxTag -> "3" + | Module -> "3" let tokenModifiersToString = function NoModifier -> "0" @@ -94,14 +94,14 @@ let emitLongident ~backwards ~pos ~jsx ~lid ~debug emitter = emitter |> emitFromPos pos (fst pos, snd pos + String.length id) - ~type_:(if isUppercaseId id then Token.JsxTag else Token.Variable) + ~type_:(if isUppercaseId id then Module else Token.Variable) | id :: segments when isUppercaseId id || isLowercaseId id -> if debug then Printf.printf "Ldot: %s %s\n" id (posToString pos); let length = String.length id in emitter |> emitFromPos pos (fst pos, snd pos + length) - ~type_:(if isUppercaseId id then Token.JsxTag else Token.Variable); + ~type_:(if isUppercaseId id then Module else Token.Variable); loop (fst pos, snd pos + length + 1) segments | _ -> () in diff --git a/package.json b/package.json index b8707b19a..aa5f2b60b 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "semanticTokenScopes": [ { "scopes": { - "jsx-tag": ["entity.name.tag"] + "module-tag": ["entity.name.tag"] } } ], diff --git a/server/src/server.ts b/server/src/server.ts index d6962b1e8..2f6f919da 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -760,7 +760,7 @@ function onMessage(msg: m.Message) { completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, semanticTokensProvider: { legend: { - tokenTypes: ["keyword", "variable", "type", "jsx-tag"], + tokenTypes: ["keyword", "variable", "type", "module-tag"], tokenModifiers: [], }, documentSelector: null, From f11f6827cf979035932253ac971e0c242d9183ee Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 09:51:40 +0100 Subject: [PATCH 16/68] Other cases of module declarations and expressions. --- analysis/src/SemanticTokens.ml | 74 +++++++++++++++++++--- analysis/tests/src/Parser.res | 22 +++++++ analysis/tests/src/expected/Parser.res.txt | 18 +++++- 3 files changed, 105 insertions(+), 9 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index dec4fd63f..8896f3c00 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -78,7 +78,7 @@ let emitFromLoc ~loc ~type_ emitter = let posStart, posEnd = locToPositions loc in emitter |> emitFromPos posStart posEnd ~type_ -let emitLongident ~backwards ~pos ~jsx ~lid ~debug emitter = +let emitLongident ?(backwards = false) ?(jsx = false) ~pos ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -115,13 +115,13 @@ let emitLongident ~backwards ~pos ~jsx ~lid ~debug emitter = let emitVariable ~id ~debug ~loc emitter = emitter - |> emitLongident ~backwards:false + |> emitLongident ~pos:(Utils.tupleOfLexing loc.Location.loc_start) - ~jsx:false ~lid:(Longident.Lident id) ~debug + ~lid:(Longident.Lident id) ~debug let emitJsxOpen ~lid ~debug ~loc emitter = emitter - |> emitLongident ~backwards:false + |> emitLongident ~pos:(Utils.tupleOfLexing loc.Location.loc_start) ~lid ~jsx:true ~debug @@ -163,9 +163,7 @@ let parser ~debug ~emitter ~path = match e.pexp_desc with | Pexp_ident {txt = lid; loc} -> emitter - |> emitLongident ~backwards:false - ~pos:(Utils.tupleOfLexing loc.loc_start) - ~lid ~jsx:false ~debug; + |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) when Res_parsetree_viewer.isJsxExpression e -> @@ -206,9 +204,69 @@ let parser ~debug ~emitter ~path = Ast_mapper.default_mapper.expr mapper e | _ -> Ast_mapper.default_mapper.expr mapper e in + let module_expr (mapper : Ast_mapper.mapper) (me : Parsetree.module_expr) = + match me.pmod_desc with + | Pmod_ident {txt = lid; loc} -> + emitter + |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; + Ast_mapper.default_mapper.module_expr mapper me + | _ -> Ast_mapper.default_mapper.module_expr mapper me + in + let module_binding (mapper : Ast_mapper.mapper) + (mb : Parsetree.module_binding) = + emitter + |> emitLongident + ~pos:(Utils.tupleOfLexing mb.pmb_name.loc.loc_start) + ~lid:(Longident.Lident mb.pmb_name.txt) ~debug; + Ast_mapper.default_mapper.module_binding mapper mb + in + let module_declaration (mapper : Ast_mapper.mapper) + (md : Parsetree.module_declaration) = + emitter + |> emitLongident + ~pos:(Utils.tupleOfLexing md.pmd_name.loc.loc_start) + ~lid:(Longident.Lident md.pmd_name.txt) ~debug; + Ast_mapper.default_mapper.module_declaration mapper md + in + let module_type (mapper : Ast_mapper.mapper) (mt : Parsetree.module_type) = + match mt.pmty_desc with + | Pmty_ident {txt = lid; loc} -> + emitter + |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; + Ast_mapper.default_mapper.module_type mapper mt + | _ -> Ast_mapper.default_mapper.module_type mapper mt + in + let module_type_declaration (mapper : Ast_mapper.mapper) + (mtd : Parsetree.module_type_declaration) = + emitter + |> emitLongident + ~pos:(Utils.tupleOfLexing mtd.pmtd_name.loc.loc_start) + ~lid:(Longident.Lident mtd.pmtd_name.txt) ~debug; + Ast_mapper.default_mapper.module_type_declaration mapper mtd + in + let open_description (mapper : Ast_mapper.mapper) + (od : Parsetree.open_description) = + emitter + |> emitLongident + ~pos:(Utils.tupleOfLexing od.popen_lid.loc.loc_start) + ~lid:od.popen_lid.txt ~debug; + Ast_mapper.default_mapper.open_description mapper od + in let mapper = - {Ast_mapper.default_mapper with expr; pat; typ; type_declaration} + { + Ast_mapper.default_mapper with + expr; + module_declaration; + module_binding; + module_expr; + module_type; + module_type_declaration; + open_description; + pat; + typ; + type_declaration; + } in if Filename.check_suffix path ".res" then ( diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 97f36a166..631e9f303 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -31,4 +31,26 @@ type pairIntString = list< let _ = 3 < 4 || 3 > 4 +module type MT = { + module DDF: { + + } +} + +module DDF: MT = { + module DDF = { + + } +} + +module XX = { + module YY = { + type t = int + } +} + +open XX.YY + +type tt = t + // ^par diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 58dc57c4a..9e40a855b 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,8 @@ Parse tests/src/Parser.res -structure items:10 diagnostics:0 +structure items:15 diagnostics:0 +Lident: M (0,7) +Lident: C (1,9) +Lident: Component (1,13) Lident: Component (4,10) Lident: _c (4,4) Ldot: M (6,11) @@ -36,4 +39,17 @@ Type: looooooooooooooooooooooooooooooooooooooong_string (27,4)->(27,53) BinaryExp: (31,14)->(31,16) BinaryExp: (31,10)->(31,11) BinaryExp: (31,19)->(31,20) +Lident: MT (33,12) +Lident: DDF (34,9) +Lident: DDF (39,7) +Lident: MT (39,12) +Lident: DDF (40,9) +Lident: XX (45,7) +Lident: YY (46,9) +Type: t (47,9)->(47,10) +Type: int (47,13)->(47,16) +Ldot: XX (51,5) +Lident: YY (51,8) +Type: tt (53,5)->(53,7) +Type: t (53,10)->(53,11) From e557d688a8d4dbb91f0ad3fc52497bb084b1c1ad Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 10:07:44 +0100 Subject: [PATCH 17/68] For module types, use Token.Type. --- analysis/src/SemanticTokens.ml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 8896f3c00..d853a41e2 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -78,7 +78,8 @@ let emitFromLoc ~loc ~type_ emitter = let posStart, posEnd = locToPositions loc in emitter |> emitFromPos posStart posEnd ~type_ -let emitLongident ?(backwards = false) ?(jsx = false) ~pos ~lid ~debug emitter = +let emitLongident ?(backwards = false) ?(jsx = false) + ?(moduleToken = Token.Module) ~pos ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -94,14 +95,14 @@ let emitLongident ?(backwards = false) ?(jsx = false) ~pos ~lid ~debug emitter = emitter |> emitFromPos pos (fst pos, snd pos + String.length id) - ~type_:(if isUppercaseId id then Module else Token.Variable) + ~type_:(if isUppercaseId id then moduleToken else Variable) | id :: segments when isUppercaseId id || isLowercaseId id -> if debug then Printf.printf "Ldot: %s %s\n" id (posToString pos); let length = String.length id in emitter |> emitFromPos pos (fst pos, snd pos + length) - ~type_:(if isUppercaseId id then Module else Token.Variable); + ~type_:(if isUppercaseId id then moduleToken else Variable); loop (fst pos, snd pos + length + 1) segments | _ -> () in @@ -232,14 +233,16 @@ let parser ~debug ~emitter ~path = match mt.pmty_desc with | Pmty_ident {txt = lid; loc} -> emitter - |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; + |> emitLongident ~moduleToken:Token.Type + ~pos:(Utils.tupleOfLexing loc.loc_start) + ~lid ~debug; Ast_mapper.default_mapper.module_type mapper mt | _ -> Ast_mapper.default_mapper.module_type mapper mt in let module_type_declaration (mapper : Ast_mapper.mapper) (mtd : Parsetree.module_type_declaration) = emitter - |> emitLongident + |> emitLongident ~moduleToken:Token.Type ~pos:(Utils.tupleOfLexing mtd.pmtd_name.loc.loc_start) ~lid:(Longident.Lident mtd.pmtd_name.txt) ~debug; Ast_mapper.default_mapper.module_type_declaration mapper mtd From 167f7ce24ea7e1db30e8c8793112506a418619da Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 10:11:47 +0100 Subject: [PATCH 18/68] Add token type to debug print. --- analysis/src/SemanticTokens.ml | 26 +++++---- analysis/tests/src/expected/Parser.res.txt | 64 +++++++++++----------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index d853a41e2..a449422c5 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -12,6 +12,12 @@ module Token = struct | Type -> "2" | Module -> "3" + let tokenTypeDebug = function + | Keyword -> "Keyword" + | Variable -> "Variable" + | Type -> "Type" + | Module -> "Module" + let tokenModifiersToString = function NoModifier -> "0" type token = int * int * int * tokenType * tokenModifiers @@ -91,18 +97,18 @@ let emitLongident ?(backwards = false) ?(jsx = false) let rec loop pos segments = match segments with | [id] when isUppercaseId id || isLowercaseId id -> - if debug then Printf.printf "Lident: %s %s\n" id (posToString pos); - emitter - |> emitFromPos pos - (fst pos, snd pos + String.length id) - ~type_:(if isUppercaseId id then moduleToken else Variable) + let type_ = if isUppercaseId id then moduleToken else Variable in + if debug then + Printf.printf "Lident: %s %s %s\n" id (posToString pos) + (Token.tokenTypeDebug type_); + emitter |> emitFromPos pos (fst pos, snd pos + String.length id) ~type_ | id :: segments when isUppercaseId id || isLowercaseId id -> - if debug then Printf.printf "Ldot: %s %s\n" id (posToString pos); + let type_ = if isUppercaseId id then moduleToken else Variable in + if debug then + Printf.printf "Ldot: %s %s %s\n" id (posToString pos) + (Token.tokenTypeDebug type_); let length = String.length id in - emitter - |> emitFromPos pos - (fst pos, snd pos + length) - ~type_:(if isUppercaseId id then moduleToken else Variable); + emitter |> emitFromPos pos (fst pos, snd pos + length) ~type_; loop (fst pos, snd pos + length + 1) segments | _ -> () in diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 9e40a855b..c410c3585 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,28 +1,28 @@ Parse tests/src/Parser.res structure items:15 diagnostics:0 -Lident: M (0,7) -Lident: C (1,9) -Lident: Component (1,13) -Lident: Component (4,10) -Lident: _c (4,4) -Ldot: M (6,11) -Lident: C (6,13) -Lident: _mc (6,4) -Lident: div (8,10) -Lident: _d (8,4) -Lident: div (11,3) -Lident: div (16,4) -Ldot: React (12,5) -Lident: string (12,11) -Lident: div (13,5) -Lident: div (13,34) -Ldot: React (13,11) -Lident: string (13,17) -Ldot: React (14,5) -Lident: string (14,11) -Ldot: React (15,5) -Lident: string (15,11) -Lident: _d2 (10,4) +Lident: M (0,7) Module +Lident: C (1,9) Module +Lident: Component (1,13) Module +Lident: Component (4,10) Module +Lident: _c (4,4) Variable +Ldot: M (6,11) Module +Lident: C (6,13) Module +Lident: _mc (6,4) Variable +Lident: div (8,10) Variable +Lident: _d (8,4) Variable +Lident: div (11,3) Variable +Lident: div (16,4) Variable +Ldot: React (12,5) Module +Lident: string (12,11) Variable +Lident: div (13,5) Variable +Lident: div (13,34) Variable +Ldot: React (13,11) Module +Lident: string (13,17) Variable +Ldot: React (14,5) Module +Lident: string (14,11) Variable +Ldot: React (15,5) Module +Lident: string (15,11) Variable +Lident: _d2 (10,4) Variable Type: pair (18,5)->(18,9) Type: looooooooooooooooooooooooooooooooooooooong_int (20,5)->(20,51) Type: int (20,54)->(20,57) @@ -39,17 +39,17 @@ Type: looooooooooooooooooooooooooooooooooooooong_string (27,4)->(27,53) BinaryExp: (31,14)->(31,16) BinaryExp: (31,10)->(31,11) BinaryExp: (31,19)->(31,20) -Lident: MT (33,12) -Lident: DDF (34,9) -Lident: DDF (39,7) -Lident: MT (39,12) -Lident: DDF (40,9) -Lident: XX (45,7) -Lident: YY (46,9) +Lident: MT (33,12) Type +Lident: DDF (34,9) Module +Lident: DDF (39,7) Module +Lident: MT (39,12) Type +Lident: DDF (40,9) Module +Lident: XX (45,7) Module +Lident: YY (46,9) Module Type: t (47,9)->(47,10) Type: int (47,13)->(47,16) -Ldot: XX (51,5) -Lident: YY (51,8) +Ldot: XX (51,5) Module +Lident: YY (51,8) Module Type: tt (53,5)->(53,7) Type: t (53,10)->(53,11) From 931af3d834e3474ee1baaf8e200f3bda9c087560 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 10:26:23 +0100 Subject: [PATCH 19/68] Use JsxTag for lower-case jsx, and Namespace for modules. --- analysis/src/SemanticTokens.ml | 15 +++++--- analysis/tests/src/expected/Parser.res.txt | 44 +++++++++++----------- package.json | 2 +- server/src/server.ts | 2 +- 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index a449422c5..b48e9a5f4 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -3,20 +3,22 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword | Variable | Type | Module + type tokenType = Keyword | Variable | Type | JsxTag | Namespace type tokenModifiers = NoModifier let tokenTypeToString = function | Keyword -> "0" | Variable -> "1" | Type -> "2" - | Module -> "3" + | JsxTag -> "3" + | Namespace -> "4" let tokenTypeDebug = function | Keyword -> "Keyword" | Variable -> "Variable" | Type -> "Type" - | Module -> "Module" + | JsxTag -> "JsxTag" + | Namespace -> "Namespace" let tokenModifiersToString = function NoModifier -> "0" @@ -85,7 +87,8 @@ let emitFromLoc ~loc ~type_ emitter = emitter |> emitFromPos posStart posEnd ~type_ let emitLongident ?(backwards = false) ?(jsx = false) - ?(moduleToken = Token.Module) ~pos ~lid ~debug emitter = + ?(moduleToken = Token.Namespace) ~pos ~lid ~debug emitter = + let variableToken = if jsx then Token.JsxTag else Variable in let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -97,13 +100,13 @@ let emitLongident ?(backwards = false) ?(jsx = false) let rec loop pos segments = match segments with | [id] when isUppercaseId id || isLowercaseId id -> - let type_ = if isUppercaseId id then moduleToken else Variable in + let type_ = if isUppercaseId id then moduleToken else variableToken in if debug then Printf.printf "Lident: %s %s %s\n" id (posToString pos) (Token.tokenTypeDebug type_); emitter |> emitFromPos pos (fst pos, snd pos + String.length id) ~type_ | id :: segments when isUppercaseId id || isLowercaseId id -> - let type_ = if isUppercaseId id then moduleToken else Variable in + let type_ = if isUppercaseId id then moduleToken else variableToken in if debug then Printf.printf "Ldot: %s %s %s\n" id (posToString pos) (Token.tokenTypeDebug type_); diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index c410c3585..6d5b89d7e 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,26 +1,26 @@ Parse tests/src/Parser.res structure items:15 diagnostics:0 -Lident: M (0,7) Module -Lident: C (1,9) Module -Lident: Component (1,13) Module -Lident: Component (4,10) Module +Lident: M (0,7) Namespace +Lident: C (1,9) Namespace +Lident: Component (1,13) Namespace +Lident: Component (4,10) Namespace Lident: _c (4,4) Variable -Ldot: M (6,11) Module -Lident: C (6,13) Module +Ldot: M (6,11) Namespace +Lident: C (6,13) Namespace Lident: _mc (6,4) Variable -Lident: div (8,10) Variable +Lident: div (8,10) JsxTag Lident: _d (8,4) Variable -Lident: div (11,3) Variable -Lident: div (16,4) Variable -Ldot: React (12,5) Module +Lident: div (11,3) JsxTag +Lident: div (16,4) JsxTag +Ldot: React (12,5) Namespace Lident: string (12,11) Variable -Lident: div (13,5) Variable -Lident: div (13,34) Variable -Ldot: React (13,11) Module +Lident: div (13,5) JsxTag +Lident: div (13,34) JsxTag +Ldot: React (13,11) Namespace Lident: string (13,17) Variable -Ldot: React (14,5) Module +Ldot: React (14,5) Namespace Lident: string (14,11) Variable -Ldot: React (15,5) Module +Ldot: React (15,5) Namespace Lident: string (15,11) Variable Lident: _d2 (10,4) Variable Type: pair (18,5)->(18,9) @@ -40,16 +40,16 @@ BinaryExp: (31,14)->(31,16) BinaryExp: (31,10)->(31,11) BinaryExp: (31,19)->(31,20) Lident: MT (33,12) Type -Lident: DDF (34,9) Module -Lident: DDF (39,7) Module +Lident: DDF (34,9) Namespace +Lident: DDF (39,7) Namespace Lident: MT (39,12) Type -Lident: DDF (40,9) Module -Lident: XX (45,7) Module -Lident: YY (46,9) Module +Lident: DDF (40,9) Namespace +Lident: XX (45,7) Namespace +Lident: YY (46,9) Namespace Type: t (47,9)->(47,10) Type: int (47,13)->(47,16) -Ldot: XX (51,5) Module -Lident: YY (51,8) Module +Ldot: XX (51,5) Namespace +Lident: YY (51,8) Namespace Type: tt (53,5)->(53,7) Type: t (53,10)->(53,11) diff --git a/package.json b/package.json index aa5f2b60b..b8707b19a 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "semanticTokenScopes": [ { "scopes": { - "module-tag": ["entity.name.tag"] + "jsx-tag": ["entity.name.tag"] } } ], diff --git a/server/src/server.ts b/server/src/server.ts index 2f6f919da..bfece5d47 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -760,7 +760,7 @@ function onMessage(msg: m.Message) { completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, semanticTokensProvider: { legend: { - tokenTypes: ["keyword", "variable", "type", "module-tag"], + tokenTypes: ["keyword", "variable", "type", "jsx-tag", "namespace"], tokenModifiers: [], }, documentSelector: null, From 0a59a790dccbc4a5521d228b82c5423ca44760d3 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Mon, 14 Mar 2022 12:07:26 +0100 Subject: [PATCH 20/68] change let coloring to keyword, not control --- grammars/rescript.tmLanguage.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 5932d7eae..10c5b46cb 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -15,7 +15,11 @@ }, "RE_KEYWORDS": { "name": "keyword.control", - "match": "\\b(and|as|assert|constraint|downto|else|exception|external|false|for|if|in|include|lazy|let|module|mutable|of|open|rec|switch|to|true|try|type|when|while|with)\\b" + "match": "\\b(and|as|assert|constraint|downto|else|exception|external|false|for|if|in|include|lazy|module|mutable|of|open|rec|switch|to|true|try|type|when|while|with)\\b" + }, + "RE_LET": { + "name": "keyword", + "match": "\\b(let)\\b" }, "RE_LITERAL": { "name": "constant.language", @@ -91,6 +95,9 @@ "patterns": [ { "include": "#RE_KEYWORDS" + }, + { + "include": "#RE_LET" } ] }, From 8bbe1d0205c0407c046d29bfe9d8e1d2174bcc61 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 11:15:55 +0100 Subject: [PATCH 21/68] Extend example file. --- analysis/tests/src/Parser.res | 12 +++++++++++- analysis/tests/src/expected/Parser.res.txt | 9 ++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 631e9f303..4a94df57e 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -53,4 +53,14 @@ open XX.YY type tt = t -// ^par +// ^par + +module T = { + type someRecord<'typeParameter> = { + someField: int, + someOtherField: string, + theParam: 'typeParameter, + } +} + +let foo = x => x.T.someField diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 6d5b89d7e..79904df9b 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:15 diagnostics:0 +structure items:17 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -52,4 +52,11 @@ Ldot: XX (51,5) Namespace Lident: YY (51,8) Namespace Type: tt (53,5)->(53,7) Type: t (53,10)->(53,11) +Lident: T (57,7) Namespace +Type: someRecord (58,7)->(58,17) +Type: int (59,15)->(59,18) +Type: string (60,20)->(60,26) +Lident: x (65,15) Variable +Lident: x (65,10) Variable +Lident: foo (65,4) Variable From 14ff85f419b9bf03826901ac4506b572b262b9d9 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 11:17:04 +0100 Subject: [PATCH 22/68] Extend example. --- analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 4a94df57e..2014244e4 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -61,6 +61,8 @@ module T = { someOtherField: string, theParam: 'typeParameter, } + + type someEnum = A | B | C } let foo = x => x.T.someField diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 79904df9b..4037b1301 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -56,7 +56,8 @@ Lident: T (57,7) Namespace Type: someRecord (58,7)->(58,17) Type: int (59,15)->(59,18) Type: string (60,20)->(60,26) -Lident: x (65,15) Variable -Lident: x (65,10) Variable -Lident: foo (65,4) Variable +Type: someEnum (64,7)->(64,15) +Lident: x (67,15) Variable +Lident: x (67,10) Variable +Lident: foo (67,4) Variable From eb5bd2b437b0be066cd1a7d1edc0baebc5fb830f Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 11:49:11 +0100 Subject: [PATCH 23/68] Emit record labels as "property". --- analysis/src/SemanticTokens.ml | 52 +++++++++++++++++++--- analysis/tests/src/expected/Parser.res.txt | 5 +++ server/src/server.ts | 10 ++++- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index b48e9a5f4..956bedeec 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -3,7 +3,15 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) - type tokenType = Keyword | Variable | Type | JsxTag | Namespace + type tokenType = + | Keyword + | Variable + | Type + | JsxTag + | Namespace + | EnumMember + | Property + type tokenModifiers = NoModifier let tokenTypeToString = function @@ -12,6 +20,8 @@ module Token = struct | Type -> "2" | JsxTag -> "3" | Namespace -> "4" + | EnumMember -> "5" + | Property -> "6" let tokenTypeDebug = function | Keyword -> "Keyword" @@ -19,6 +29,8 @@ module Token = struct | Type -> "Type" | JsxTag -> "JsxTag" | Namespace -> "Namespace" + | EnumMember -> "EnumMember" + | Property -> "Property" let tokenModifiersToString = function NoModifier -> "0" @@ -87,8 +99,8 @@ let emitFromLoc ~loc ~type_ emitter = emitter |> emitFromPos posStart posEnd ~type_ let emitLongident ?(backwards = false) ?(jsx = false) - ?(moduleToken = Token.Namespace) ~pos ~lid ~debug emitter = - let variableToken = if jsx then Token.JsxTag else Variable in + ?(lowerCaseToken = if jsx then Token.JsxTag else Variable) + ?(upperCaseToken = Token.Namespace) ~pos ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -100,13 +112,13 @@ let emitLongident ?(backwards = false) ?(jsx = false) let rec loop pos segments = match segments with | [id] when isUppercaseId id || isLowercaseId id -> - let type_ = if isUppercaseId id then moduleToken else variableToken in + let type_ = if isUppercaseId id then upperCaseToken else lowerCaseToken in if debug then Printf.printf "Lident: %s %s %s\n" id (posToString pos) (Token.tokenTypeDebug type_); emitter |> emitFromPos pos (fst pos, snd pos + String.length id) ~type_ | id :: segments when isUppercaseId id || isLowercaseId id -> - let type_ = if isUppercaseId id then moduleToken else variableToken in + let type_ = if isUppercaseId id then upperCaseToken else lowerCaseToken in if debug then Printf.printf "Ldot: %s %s %s\n" id (posToString pos) (Token.tokenTypeDebug type_); @@ -142,6 +154,12 @@ let emitType ~id ~debug ~loc emitter = if debug then Printf.printf "Type: %s %s\n" id (locToString loc); emitter |> emitFromLoc ~loc ~type_:Token.Type +let emitRecordLabel ~(label : Longident.t Location.loc) ~debug emitter = + emitter + |> emitLongident ~lowerCaseToken:Token.Property + ~pos:(Utils.tupleOfLexing label.loc.loc_start) + ~lid:label.txt ~debug + let parser ~debug ~emitter ~path = let processTypeArg (coreType : Parsetree.core_type) = if debug then Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) @@ -167,6 +185,10 @@ let parser ~debug ~emitter ~path = | Ppat_var {loc; txt = id} -> if isLowercaseId id then emitter |> emitVariable ~id ~debug ~loc; Ast_mapper.default_mapper.pat mapper p + | Ppat_record (cases, _) -> + cases + |> List.iter (fun (label, _) -> emitter |> emitRecordLabel ~label ~debug); + Ast_mapper.default_mapper.pat mapper p | _ -> Ast_mapper.default_mapper.pat mapper p in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = @@ -212,6 +234,13 @@ let parser ~debug ~emitter ~path = -> if debug then Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); Ast_mapper.default_mapper.expr mapper e + | Pexp_record (cases, _) -> + cases + |> List.iter (fun (label, _) -> emitter |> emitRecordLabel ~label ~debug); + Ast_mapper.default_mapper.expr mapper e + | Pexp_field (_, label) | Pexp_setfield (_, label, _) -> + emitter |> emitRecordLabel ~label ~debug; + Ast_mapper.default_mapper.expr mapper e | _ -> Ast_mapper.default_mapper.expr mapper e in let module_expr (mapper : Ast_mapper.mapper) (me : Parsetree.module_expr) = @@ -242,7 +271,7 @@ let parser ~debug ~emitter ~path = match mt.pmty_desc with | Pmty_ident {txt = lid; loc} -> emitter - |> emitLongident ~moduleToken:Token.Type + |> emitLongident ~upperCaseToken:Token.Type ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; Ast_mapper.default_mapper.module_type mapper mt @@ -251,7 +280,7 @@ let parser ~debug ~emitter ~path = let module_type_declaration (mapper : Ast_mapper.mapper) (mtd : Parsetree.module_type_declaration) = emitter - |> emitLongident ~moduleToken:Token.Type + |> emitLongident ~upperCaseToken:Token.Type ~pos:(Utils.tupleOfLexing mtd.pmtd_name.loc.loc_start) ~lid:(Longident.Lident mtd.pmtd_name.txt) ~debug; Ast_mapper.default_mapper.module_type_declaration mapper mtd @@ -264,11 +293,20 @@ let parser ~debug ~emitter ~path = ~lid:od.popen_lid.txt ~debug; Ast_mapper.default_mapper.open_description mapper od in + let label_declaration (mapper : Ast_mapper.mapper) + (ld : Parsetree.label_declaration) = + emitter + |> emitRecordLabel + ~label:{loc = ld.pld_name.loc; txt = Longident.Lident ld.pld_name.txt} + ~debug; + Ast_mapper.default_mapper.label_declaration mapper ld + in let mapper = { Ast_mapper.default_mapper with expr; + label_declaration; module_declaration; module_binding; module_expr; diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 4037b1301..40e8e109c 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -54,9 +54,14 @@ Type: tt (53,5)->(53,7) Type: t (53,10)->(53,11) Lident: T (57,7) Namespace Type: someRecord (58,7)->(58,17) +Lident: someField (59,4) Property Type: int (59,15)->(59,18) +Lident: someOtherField (60,4) Property Type: string (60,20)->(60,26) +Lident: theParam (61,4) Property Type: someEnum (64,7)->(64,15) +Ldot: T (67,17) Namespace +Lident: someField (67,19) Property Lident: x (67,15) Variable Lident: x (67,10) Variable Lident: foo (67,4) Variable diff --git a/server/src/server.ts b/server/src/server.ts index bfece5d47..1d0656c10 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -760,7 +760,15 @@ function onMessage(msg: m.Message) { completionProvider: { triggerCharacters: [".", ">", "@", "~", '"'] }, semanticTokensProvider: { legend: { - tokenTypes: ["keyword", "variable", "type", "jsx-tag", "namespace"], + tokenTypes: [ + "keyword", + "variable", + "type", + "jsx-tag", + "namespace", + "enumMember", + "property", + ], tokenModifiers: [], }, documentSelector: null, From ec3407845f574af172f2e90e1e6d1e7d03634c28 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 12:04:20 +0100 Subject: [PATCH 24/68] Emit variants as enumMember. --- analysis/src/SemanticTokens.ml | 21 +++++++++++++++++++++ analysis/tests/src/expected/Parser.res.txt | 3 +++ 2 files changed, 24 insertions(+) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 956bedeec..abed444e6 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -160,6 +160,12 @@ let emitRecordLabel ~(label : Longident.t Location.loc) ~debug emitter = ~pos:(Utils.tupleOfLexing label.loc.loc_start) ~lid:label.txt ~debug +let emitVariant ~(name : Longident.t Location.loc) ~debug emitter = + emitter + |> emitLongident ~upperCaseToken:Token.EnumMember + ~pos:(Utils.tupleOfLexing name.loc.loc_start) + ~lid:name.txt ~debug + let parser ~debug ~emitter ~path = let processTypeArg (coreType : Parsetree.core_type) = if debug then Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) @@ -189,6 +195,9 @@ let parser ~debug ~emitter ~path = cases |> List.iter (fun (label, _) -> emitter |> emitRecordLabel ~label ~debug); Ast_mapper.default_mapper.pat mapper p + | Ppat_construct (name, _) -> + emitter |> emitVariant ~name ~debug; + Ast_mapper.default_mapper.pat mapper p | _ -> Ast_mapper.default_mapper.pat mapper p in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = @@ -241,6 +250,9 @@ let parser ~debug ~emitter ~path = | Pexp_field (_, label) | Pexp_setfield (_, label, _) -> emitter |> emitRecordLabel ~label ~debug; Ast_mapper.default_mapper.expr mapper e + | Pexp_construct (name, _) -> + emitter |> emitVariant ~name ~debug; + Ast_mapper.default_mapper.expr mapper e | _ -> Ast_mapper.default_mapper.expr mapper e in let module_expr (mapper : Ast_mapper.mapper) (me : Parsetree.module_expr) = @@ -301,10 +313,19 @@ let parser ~debug ~emitter ~path = ~debug; Ast_mapper.default_mapper.label_declaration mapper ld in + let constructor_declaration (mapper : Ast_mapper.mapper) + (cd : Parsetree.constructor_declaration) = + emitter + |> emitVariant + ~name:{loc = cd.pcd_name.loc; txt = Longident.Lident cd.pcd_name.txt} + ~debug; + Ast_mapper.default_mapper.constructor_declaration mapper cd + in let mapper = { Ast_mapper.default_mapper with + constructor_declaration; expr; label_declaration; module_declaration; diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 40e8e109c..922e05aba 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -60,6 +60,9 @@ Lident: someOtherField (60,4) Property Type: string (60,20)->(60,26) Lident: theParam (61,4) Property Type: someEnum (64,7)->(64,15) +Lident: A (64,18) EnumMember +Lident: B (64,22) EnumMember +Lident: C (64,26) EnumMember Ldot: T (67,17) Namespace Lident: someField (67,19) Property Lident: x (67,15) Variable From 4428025bf8ed0f75809b68a1d04e7bb6249f9c37 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Mon, 14 Mar 2022 12:30:49 +0100 Subject: [PATCH 25/68] more samples --- .../src/syntax/sample-highlighting.res | 23 +++++++++++++++++-- .../src/syntax/sample-highlighting.tsx | 21 +++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.res b/analysis/examples/example-project/src/syntax/sample-highlighting.res index 814c7faed..35ea24dca 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.res +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.res @@ -18,6 +18,13 @@ type someEnum = | AnotherMember | SomeMemberWithPayload(someRecord) +type somePolyEnum = [ + | #someMember + | #AnotherMember + | #SomeMemberWithPayload(someRecord) + | #"fourth Member" +] + // Destructuring let destructuring = () => { let someVar = (1, 2, 3) @@ -32,12 +39,24 @@ let destructuring = () => { someField } +module SomeModule = { + type t = Some | Value | Here +} + // JSX module SomeComponent = { @react.component - let make = () => { + let make = ( + ~someProp: int, + ~otherProp: string, + ~thirdProp: SomeModule.t, + ~fourth: somePolyEnum=#"fourth member", + ) => { React.null } } -let jsx =
+let jsx = +
+ +
diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx index 1919f6fc4..004a33c46 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -32,13 +32,30 @@ let destructuring = () => { return someField; }; +namespace SomeModule { + export enum t { + Some, + Value, + Here, + } +} + // JSX -const SomeComponent = () => { +interface Props { + someProp: number; + otherProp: string; + thirdProp: SomeModule.t; +} +const SomeComponent = ({ someProp, otherProp, thirdProp }: Props) => { return null; }; let jsx = (
- +
); From ae086821292fdbc33d64bd4201e88f657b1f5374 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 15:01:01 +0100 Subject: [PATCH 26/68] Fix issue with function definitions that use labeled arguments. --- analysis/src/SemanticTokens.ml | 11 +++++------ analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 20 +++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index abed444e6..99bcff2a3 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -136,10 +136,8 @@ let emitLongident ?(backwards = false) ?(jsx = false) else loop pos segments let emitVariable ~id ~debug ~loc emitter = - emitter - |> emitLongident - ~pos:(Utils.tupleOfLexing loc.Location.loc_start) - ~lid:(Longident.Lident id) ~debug + if debug then Printf.printf "Variable: %s %s\n" id (locToString loc); + emitter |> emitFromLoc ~loc ~type_:Variable let emitJsxOpen ~lid ~debug ~loc emitter = emitter @@ -188,8 +186,9 @@ let parser ~debug ~emitter ~path = in let pat (mapper : Ast_mapper.mapper) (p : Parsetree.pattern) = match p.ppat_desc with - | Ppat_var {loc; txt = id} -> - if isLowercaseId id then emitter |> emitVariable ~id ~debug ~loc; + | Ppat_var {txt = id} -> + if isLowercaseId id then + emitter |> emitVariable ~id ~debug ~loc:p.ppat_loc; Ast_mapper.default_mapper.pat mapper p | Ppat_record (cases, _) -> cases diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 2014244e4..f7770e68e 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -66,3 +66,5 @@ module T = { } let foo = x => x.T.someField + +let add = (~hello, ~world) => hello + world diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 922e05aba..95a5d9712 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,15 +1,15 @@ Parse tests/src/Parser.res -structure items:17 diagnostics:0 +structure items:18 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace Lident: Component (4,10) Namespace -Lident: _c (4,4) Variable +Variable: _c (4,4)->(4,6) Ldot: M (6,11) Namespace Lident: C (6,13) Namespace -Lident: _mc (6,4) Variable +Variable: _mc (6,4)->(6,7) Lident: div (8,10) JsxTag -Lident: _d (8,4) Variable +Variable: _d (8,4)->(8,6) Lident: div (11,3) JsxTag Lident: div (16,4) JsxTag Ldot: React (12,5) Namespace @@ -22,7 +22,7 @@ Ldot: React (14,5) Namespace Lident: string (14,11) Variable Ldot: React (15,5) Namespace Lident: string (15,11) Variable -Lident: _d2 (10,4) Variable +Variable: _d2 (10,4)->(10,7) Type: pair (18,5)->(18,9) Type: looooooooooooooooooooooooooooooooooooooong_int (20,5)->(20,51) Type: int (20,54)->(20,57) @@ -66,6 +66,12 @@ Lident: C (64,26) EnumMember Ldot: T (67,17) Namespace Lident: someField (67,19) Property Lident: x (67,15) Variable -Lident: x (67,10) Variable -Lident: foo (67,4) Variable +Variable: x (67,10)->(67,11) +Variable: foo (67,4)->(67,7) +BinaryExp: (69,36)->(69,37) +Lident: hello (69,30) Variable +Lident: world (69,38) Variable +Variable: world (69,19)->(69,25) +Variable: hello (69,11)->(69,17) +Variable: add (69,4)->(69,7) From bcb131f74dd84f11f4320144cd09b19fff00ee2b Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 16:08:41 +0100 Subject: [PATCH 27/68] tweak example file --- analysis/tests/src/Parser.res | 6 +++++- analysis/tests/src/expected/Parser.res.txt | 14 ++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index f7770e68e..7a44217f4 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -67,4 +67,8 @@ module T = { let foo = x => x.T.someField -let add = (~hello, ~world) => hello + world +let add = (~hello as x, ~world) => x + world + +let _ = add(~hello=3) + +let _ =
diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 95a5d9712..63606f94a 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:18 diagnostics:0 +structure items:20 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -68,10 +68,12 @@ Lident: someField (67,19) Property Lident: x (67,15) Variable Variable: x (67,10)->(67,11) Variable: foo (67,4)->(67,7) -BinaryExp: (69,36)->(69,37) -Lident: hello (69,30) Variable -Lident: world (69,38) Variable -Variable: world (69,19)->(69,25) -Variable: hello (69,11)->(69,17) +BinaryExp: (69,37)->(69,38) +Lident: x (69,35) Variable +Lident: world (69,39) Variable +Variable: world (69,24)->(69,30) +Variable: x (69,21)->(69,22) Variable: add (69,4)->(69,7) +Lident: add (71,8) Variable +Lident: div (73,9) JsxTag From dcaa38c3ce3413a6546dadc1d342678b97ab5d47 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 14 Mar 2022 16:09:36 +0100 Subject: [PATCH 28/68] Fix recursion in jsx props and children. --- analysis/src/SemanticTokens.ml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 99bcff2a3..0c1164b24 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -232,11 +232,7 @@ let parser ~debug ~emitter ~path = |> emitJsxClose ~debug ~lid:lident.txt ~pos:(lineEndWhole, colEndWhole - 1)); (* only process again arguments, not the jsx label *) - let _ = - args - |> List.map (fun (_lbl, arg) -> - Ast_mapper.default_mapper.expr mapper arg) - in + let _ = args |> List.map (fun (_lbl, arg) -> mapper.expr mapper arg) in e | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e -> From 29c843ff81a5fed77ce920f2884d42947bf95acb Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Tue, 15 Mar 2022 17:43:12 +0100 Subject: [PATCH 29/68] Mark identifiers as `variable` by default. This handles the case of labels whose location is not recoverable just from the AST: - function definitions `(~label as name)` - function calls `~label = 10` - component props `prop = 42` --- grammars/rescript.tmLanguage.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 10c5b46cb..cd4ff79e6 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -206,6 +206,14 @@ } ] }, + "defaultIdIsVariable": { + "patterns": [ + { + "match": "[A-Za-z_][A-Za-z0-9]*", + "name": "variable" + } + ] + }, "number": { "patterns": [ { @@ -501,6 +509,9 @@ }, { "include": "#punctuations" + }, + { + "include": "#defaultIdIsVariable" } ] } From cf7b9b3b9f42deb1cb32e8bae6d07a54c54b5c08 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Tue, 15 Mar 2022 17:44:18 +0100 Subject: [PATCH 30/68] Update rescript.tmLanguage.json --- grammars/rescript.tmLanguage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index cd4ff79e6..4a25fc8f6 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -209,7 +209,7 @@ "defaultIdIsVariable": { "patterns": [ { - "match": "[A-Za-z_][A-Za-z0-9]*", + "match": "[A-Za-z][A-Za-z0-9]*", "name": "variable" } ] From dada9946440fc688cc0dab9b1efae2409e6ce85d Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Tue, 15 Mar 2022 17:50:44 +0100 Subject: [PATCH 31/68] Tweak example. --- analysis/tests/src/Parser.res | 2 +- analysis/tests/src/expected/Parser.res.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 7a44217f4..0bd74f449 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -71,4 +71,4 @@ let add = (~hello as x, ~world) => x + world let _ = add(~hello=3) -let _ =
+let _ =
diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 63606f94a..cf2b066ef 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -76,4 +76,6 @@ Variable: x (69,21)->(69,22) Variable: add (69,4)->(69,7) Lident: add (71,8) Variable Lident: div (73,9) JsxTag +Lident: div (73,36) JsxTag +Lident: div (73,27) JsxTag From 0cdd1545f9ead77256b5a166a7f221cdd4bcc8d3 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Tue, 15 Mar 2022 20:56:59 +0100 Subject: [PATCH 32/68] set explicit scope for true/false --- grammars/rescript.tmLanguage.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 4a25fc8f6..1ef1fce64 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -17,6 +17,10 @@ "name": "keyword.control", "match": "\\b(and|as|assert|constraint|downto|else|exception|external|false|for|if|in|include|lazy|module|mutable|of|open|rec|switch|to|true|try|type|when|while|with)\\b" }, + "RE_CONSTANTS_BOOL": { + "name": "constant.language.boolean", + "match": "\\b(false|true)\\b" + }, "RE_LET": { "name": "keyword", "match": "\\b(let)\\b" @@ -105,6 +109,9 @@ "patterns": [ { "include": "#RE_LITERAL" + }, + { + "include": "#RE_CONSTANTS_BOOL" } ] }, From 23a4ff4cff3f6327f2e8e19b82ab569f46ddf7e1 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Tue, 15 Mar 2022 20:59:44 +0100 Subject: [PATCH 33/68] add nested sample --- .../example-project/src/syntax/sample-highlighting.res | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.res b/analysis/examples/example-project/src/syntax/sample-highlighting.res index 35ea24dca..be45a21c0 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.res +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.res @@ -54,9 +54,17 @@ module SomeComponent = { ) => { React.null } + + module Nested = { + @react.component + let make = (~children) => { + <> {children} + } + } } let jsx =
+ {React.string("Nested")}
From f59d9600d196fd13f087394042fb5570ede787f6 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Tue, 15 Mar 2022 21:03:55 +0100 Subject: [PATCH 34/68] add nested sample to TS file as well --- .../example-project/src/syntax/sample-highlighting.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx index 004a33c46..92e654e96 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -1,6 +1,10 @@ // Bindings let numberBinding = 123; +const SomeComp = { + Nested: () => null, +}; + let someFunction = (param: number): number => { let innerBinding = param + 2; return innerBinding; From 5aeb00edae98cde44bc3e94f57d16c20da8c5a26 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Tue, 15 Mar 2022 21:11:03 +0100 Subject: [PATCH 35/68] samples for interpolated strings --- .../example-project/src/syntax/sample-highlighting.res | 3 +++ .../example-project/src/syntax/sample-highlighting.tsx | 3 +++ 2 files changed, 6 insertions(+) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.res b/analysis/examples/example-project/src/syntax/sample-highlighting.res index be45a21c0..b3fd724d0 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.res +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.res @@ -43,6 +43,9 @@ module SomeModule = { type t = Some | Value | Here } +// Strings +let interpolated = `${numberBinding} ${"123"}` + // JSX module SomeComponent = { @react.component diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx index 92e654e96..4aa092c8a 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -44,6 +44,9 @@ namespace SomeModule { } } +// Strings +let interpolated = `${numberBinding} ${"123"}`; + // JSX interface Props { someProp: number; From 411dce0a659d030bfe2c5bd656286baea5cbf498 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Tue, 15 Mar 2022 21:14:02 +0100 Subject: [PATCH 36/68] set template-expression scope for interpolated strings --- grammars/rescript.tmLanguage.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 1ef1fce64..8df12ebc9 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -161,13 +161,13 @@ "begin": "\\$\\{", "beginCaptures": { "0": { - "name": "punctuation.section.interpolation.begin" + "name": "punctuation.definition.template-expression.begin" } }, "end": "\\}", "endCaptures": { "0": { - "name": "punctuation.section.interpolation.end" + "name": "punctuation.definition.template-expression.end" } }, "patterns": [ From 0dbba2ac38e98ccaefdea515d3b4cd2e3282db2d Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 16 Mar 2022 10:05:02 +0100 Subject: [PATCH 37/68] Tweak test: nested. --- analysis/tests/src/Parser.res | 11 +++++++++++ analysis/tests/src/expected/Parser.res.txt | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 0bd74f449..c5f548da8 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -72,3 +72,14 @@ let add = (~hello as x, ~world) => x + world let _ = add(~hello=3) let _ =
+ +module SomeComponent = { + module Nested = { + @react.component + let make = (~children) => { + <> {children} + } + } +} + +let _ =
diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index cf2b066ef..057957748 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:20 diagnostics:0 +structure items:22 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -78,4 +78,14 @@ Lident: add (71,8) Variable Lident: div (73,9) JsxTag Lident: div (73,36) JsxTag Lident: div (73,27) JsxTag +Lident: SomeComponent (75,7) Namespace +Lident: Nested (76,9) Namespace +Lident: children (79,10) Variable +Variable: children (78,16)->(78,25) +Variable: make (78,8)->(78,12) +Ldot: SomeComponent (84,9) Namespace +Lident: Nested (84,23) Namespace +Ldot: Nested (84,41) Namespace +Lident: SomeComponent (84,48) Namespace +Lident: div (84,32) JsxTag From cda48661a1e199dbd1d47f9054e81158eaa78db5 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 16 Mar 2022 10:05:39 +0100 Subject: [PATCH 38/68] Fix issue with closing tag of nested components. --- analysis/src/SemanticTokens.ml | 1 - analysis/tests/src/expected/Parser.res.txt | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 0c1164b24..05519cdcb 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -128,7 +128,6 @@ let emitLongident ?(backwards = false) ?(jsx = false) | _ -> () in let segments = flatten [] lid in - let segments = if backwards then List.rev segments else segments in if backwards then ( let totalLength = segments |> String.concat "." |> String.length in if snd pos >= totalLength then diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 057957748..c1cfa3d6f 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -85,7 +85,7 @@ Variable: children (78,16)->(78,25) Variable: make (78,8)->(78,12) Ldot: SomeComponent (84,9) Namespace Lident: Nested (84,23) Namespace -Ldot: Nested (84,41) Namespace -Lident: SomeComponent (84,48) Namespace +Ldot: SomeComponent (84,41) Namespace +Lident: Nested (84,55) Namespace Lident: div (84,32) JsxTag From ff13a670bc151280116197e15b8afebefc7aa65e Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 16 Mar 2022 09:52:56 +0100 Subject: [PATCH 39/68] Semantic highlighting: show brackets in jsx like TypeScript does. This is done in two places: The textmate grammar handles the cases that can be figured out locally: - "<" in "
" - "<>" - "" Semantic highlighting handles the cases that cannot be figured out locally: - ">" in "
" - ">" in "
" These cases can't be figures out locally as on the lhs of ">" there could be pretty much anything. --- analysis/src/SemanticTokens.ml | 27 ++++++++++++++++--- analysis/tests/src/expected/Parser.res.txt | 26 ++++++++++++------- grammars/rescript.tmLanguage.json | 30 ++++++++++++++++++++-- package.json | 2 +- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 05519cdcb..a1c049257 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -99,8 +99,8 @@ let emitFromLoc ~loc ~type_ emitter = emitter |> emitFromPos posStart posEnd ~type_ let emitLongident ?(backwards = false) ?(jsx = false) - ?(lowerCaseToken = if jsx then Token.JsxTag else Variable) - ?(upperCaseToken = Token.Namespace) ~pos ~lid ~debug emitter = + ?(lowerCaseToken = Token.Variable) ?(upperCaseToken = Token.Namespace) ~pos + ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -147,6 +147,10 @@ let emitJsxOpen ~lid ~debug ~loc emitter = let emitJsxClose ~lid ~debug ~pos emitter = emitter |> emitLongident ~backwards:true ~pos ~lid ~jsx:true ~debug +let emitJsxTag ~debug ~pos emitter = + if debug then Printf.printf "JsxTag >: %s\n" (posToString pos); + emitter |> emitFromPos pos (fst pos, snd pos + 1) ~type_:Token.JsxTag + let emitType ~id ~debug ~loc emitter = if debug then Printf.printf "Type: %s %s\n" id (locToString loc); emitter |> emitFromLoc ~loc ~type_:Token.Type @@ -226,10 +230,25 @@ let parser ~debug ~emitter ~path = let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in let length = if lineStart = lineEnd then colEnd - colStart else 0 in let lineEndWhole, colEndWhole = Utils.tupleOfLexing e.pexp_loc.loc_end in - if length > 0 && colEndWhole > length then + if length > 0 && colEndWhole > length then ( emitter |> emitJsxClose ~debug ~lid:lident.txt - ~pos:(lineEndWhole, colEndWhole - 1)); + ~pos:(lineEndWhole, colEndWhole - 1); + + let rec emitGreatherthanAfterProps args = + match args with + | (Asttypes.Labelled "children", {Parsetree.pexp_loc = {loc_start}}) + :: _ -> + emitter |> emitJsxTag ~debug ~pos:(Utils.tupleOfLexing loc_start) + | _ :: args -> emitGreatherthanAfterProps args + | [] -> () + in + emitGreatherthanAfterProps args (* <-- *); + emitter (* ... <-- *) + |> emitJsxTag ~debug + ~pos: + (let pos = Utils.tupleOfLexing e.pexp_loc.loc_end in + (fst pos, snd pos - 1)))); (* only process again arguments, not the jsx label *) let _ = args |> List.map (fun (_lbl, arg) -> mapper.expr mapper arg) in e diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index c1cfa3d6f..9b19c2199 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -8,14 +8,18 @@ Variable: _c (4,4)->(4,6) Ldot: M (6,11) Namespace Lident: C (6,13) Namespace Variable: _mc (6,4)->(6,7) -Lident: div (8,10) JsxTag +Lident: div (8,10) Variable Variable: _d (8,4)->(8,6) -Lident: div (11,3) JsxTag -Lident: div (16,4) JsxTag +Lident: div (11,3) Variable +Lident: div (16,4) Variable +JsxTag >: (11,6) +JsxTag >: (16,7) Ldot: React (12,5) Namespace Lident: string (12,11) Variable -Lident: div (13,5) JsxTag -Lident: div (13,34) JsxTag +Lident: div (13,5) Variable +Lident: div (13,34) Variable +JsxTag >: (13,8) +JsxTag >: (13,37) Ldot: React (13,11) Namespace Lident: string (13,17) Variable Ldot: React (14,5) Namespace @@ -75,9 +79,11 @@ Variable: world (69,24)->(69,30) Variable: x (69,21)->(69,22) Variable: add (69,4)->(69,7) Lident: add (71,8) Variable -Lident: div (73,9) JsxTag -Lident: div (73,36) JsxTag -Lident: div (73,27) JsxTag +Lident: div (73,9) Variable +Lident: div (73,36) Variable +JsxTag >: (73,24) +JsxTag >: (73,39) +Lident: div (73,27) Variable Lident: SomeComponent (75,7) Namespace Lident: Nested (76,9) Namespace Lident: children (79,10) Variable @@ -87,5 +93,7 @@ Ldot: SomeComponent (84,9) Namespace Lident: Nested (84,23) Namespace Ldot: SomeComponent (84,41) Namespace Lident: Nested (84,55) Namespace -Lident: div (84,32) JsxTag +JsxTag >: (84,29) +JsxTag >: (84,61) +Lident: div (84,32) Variable diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 8df12ebc9..232ec7fa1 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -344,22 +344,48 @@ "jsx": { "patterns": [ { - "match": "<>||/>" + "match": "<>||/>", + "name": "punctuation.definition.tag" }, { "match": " Date: Wed, 16 Mar 2022 10:55:59 +0100 Subject: [PATCH 40/68] Handle < in
. --- analysis/src/SemanticTokens.ml | 14 ++++++++++---- analysis/tests/src/expected/Parser.res.txt | 9 +++++++++ grammars/rescript.tmLanguage.json | 11 ----------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index a1c049257..224f76f19 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -147,8 +147,8 @@ let emitJsxOpen ~lid ~debug ~loc emitter = let emitJsxClose ~lid ~debug ~pos emitter = emitter |> emitLongident ~backwards:true ~pos ~lid ~jsx:true ~debug -let emitJsxTag ~debug ~pos emitter = - if debug then Printf.printf "JsxTag >: %s\n" (posToString pos); +let emitJsxTag ~debug ~name ~pos emitter = + if debug then Printf.printf "JsxTag %s: %s\n" name (posToString pos); emitter |> emitFromPos pos (fst pos, snd pos + 1) ~type_:Token.JsxTag let emitType ~id ~debug ~loc emitter = @@ -224,6 +224,11 @@ let parser ~debug ~emitter ~path = true | _ :: rest -> isSelfClosing rest in + emitter + |> emitJsxTag ~debug ~name:"<" + ~pos: + (let pos = Utils.tupleOfLexing e.pexp_loc.loc_start in + (fst pos, snd pos - 1 (* the AST skips the loc of < somehow *))); emitter |> emitJsxOpen ~lid:lident.txt ~debug ~loc:pexp_loc; (if not (isSelfClosing args) then let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in @@ -239,13 +244,14 @@ let parser ~debug ~emitter ~path = match args with | (Asttypes.Labelled "children", {Parsetree.pexp_loc = {loc_start}}) :: _ -> - emitter |> emitJsxTag ~debug ~pos:(Utils.tupleOfLexing loc_start) + emitter + |> emitJsxTag ~debug ~name:">" ~pos:(Utils.tupleOfLexing loc_start) | _ :: args -> emitGreatherthanAfterProps args | [] -> () in emitGreatherthanAfterProps args (* <-- *); emitter (* ... <-- *) - |> emitJsxTag ~debug + |> emitJsxTag ~debug ~name:">" ~pos: (let pos = Utils.tupleOfLexing e.pexp_loc.loc_end in (fst pos, snd pos - 1)))); diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 9b19c2199..7f907af03 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -3,19 +3,24 @@ structure items:22 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace +JsxTag <: (4,9) Lident: Component (4,10) Namespace Variable: _c (4,4)->(4,6) +JsxTag <: (6,10) Ldot: M (6,11) Namespace Lident: C (6,13) Namespace Variable: _mc (6,4)->(6,7) +JsxTag <: (8,9) Lident: div (8,10) Variable Variable: _d (8,4)->(8,6) +JsxTag <: (11,2) Lident: div (11,3) Variable Lident: div (16,4) Variable JsxTag >: (11,6) JsxTag >: (16,7) Ldot: React (12,5) Namespace Lident: string (12,11) Variable +JsxTag <: (13,4) Lident: div (13,5) Variable Lident: div (13,34) Variable JsxTag >: (13,8) @@ -79,21 +84,25 @@ Variable: world (69,24)->(69,30) Variable: x (69,21)->(69,22) Variable: add (69,4)->(69,7) Lident: add (71,8) Variable +JsxTag <: (73,8) Lident: div (73,9) Variable Lident: div (73,36) Variable JsxTag >: (73,24) JsxTag >: (73,39) +JsxTag <: (73,26) Lident: div (73,27) Variable Lident: SomeComponent (75,7) Namespace Lident: Nested (76,9) Namespace Lident: children (79,10) Variable Variable: children (78,16)->(78,25) Variable: make (78,8)->(78,12) +JsxTag <: (84,8) Ldot: SomeComponent (84,9) Namespace Lident: Nested (84,23) Namespace Ldot: SomeComponent (84,41) Namespace Lident: Nested (84,55) Namespace JsxTag >: (84,29) JsxTag >: (84,61) +JsxTag <: (84,31) Lident: div (84,32) Variable diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 232ec7fa1..4eb3e853b 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -369,17 +369,6 @@ } } }, - { - "match": "<([a-z_][0-9a-zA-Z_]*)", - "captures": { - "0": { - "name": "punctuation.definition.tag" - }, - "1": { - "name": "variable" - } - } - }, { "match": "<([A-Z_][0-9a-zA-Z_]*)", "captures": { From ef69aed65757d442ac57408a1f6423dc4a554d16 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 16 Mar 2022 11:08:47 +0100 Subject: [PATCH 41/68] Emit binary operators < and > as keyword, just like other operators. --- analysis/src/SemanticTokens.ml | 11 ++++++++--- analysis/tests/src/expected/Parser.res.txt | 6 ++---- package.json | 3 ++- server/src/server.ts | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 224f76f19..aa0c59c72 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -258,9 +258,14 @@ let parser ~debug ~emitter ~path = (* only process again arguments, not the jsx label *) let _ = args |> List.map (fun (_lbl, arg) -> mapper.expr mapper arg) in e - | Pexp_apply ({pexp_loc}, _) when Res_parsetree_viewer.isBinaryExpression e - -> - if debug then Printf.printf "BinaryExp: %s\n" (locToString pexp_loc); + | Pexp_apply + ( { + pexp_desc = + Pexp_ident {txt = Longident.Lident (("<" | ">") as op); loc}; + }, + [_; _] ) -> + if debug then Printf.printf "Binary operator %s %s\n" op (locToString loc); + emitter |> emitFromLoc ~loc ~type_:Keyword; Ast_mapper.default_mapper.expr mapper e | Pexp_record (cases, _) -> cases diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 7f907af03..2952b2d6a 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -45,9 +45,8 @@ TypeArg: (26,4)->(26,50) TypeArg: (27,4)->(27,53) Type: looooooooooooooooooooooooooooooooooooooong_int (26,4)->(26,50) Type: looooooooooooooooooooooooooooooooooooooong_string (27,4)->(27,53) -BinaryExp: (31,14)->(31,16) -BinaryExp: (31,10)->(31,11) -BinaryExp: (31,19)->(31,20) +Binary operator < (31,10)->(31,11) +Binary operator > (31,19)->(31,20) Lident: MT (33,12) Type Lident: DDF (34,9) Namespace Lident: DDF (39,7) Namespace @@ -77,7 +76,6 @@ Lident: someField (67,19) Property Lident: x (67,15) Variable Variable: x (67,10)->(67,11) Variable: foo (67,4)->(67,7) -BinaryExp: (69,37)->(69,38) Lident: x (69,35) Variable Lident: world (69,39) Variable Variable: world (69,24)->(69,30) diff --git a/package.json b/package.json index 132f81f16..bdc9b6e58 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "semanticTokenScopes": [ { "scopes": { - "jsx-tag": ["punctuation.definition.tag"] + "jsx-tag": ["punctuation.definition.tag"], + "keyword-tag": ["keyword"] } } ], diff --git a/server/src/server.ts b/server/src/server.ts index 1d0656c10..540392e6e 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -761,7 +761,7 @@ function onMessage(msg: m.Message) { semanticTokensProvider: { legend: { tokenTypes: [ - "keyword", + "keyword-tag", "variable", "type", "jsx-tag", From fca46a36e3ec24ff3b8500d74a225056e06fd901 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 16 Mar 2022 11:15:20 +0100 Subject: [PATCH 42/68] Fix lowercase jsx. --- analysis/src/SemanticTokens.ml | 7 +++++-- analysis/tests/src/expected/Parser.res.txt | 18 +++++++++--------- package.json | 1 + server/src/server.ts | 1 + 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index aa0c59c72..c8909c88d 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -11,6 +11,7 @@ module Token = struct | Namespace | EnumMember | Property + | JsxLowercase type tokenModifiers = NoModifier @@ -22,6 +23,7 @@ module Token = struct | Namespace -> "4" | EnumMember -> "5" | Property -> "6" + | JsxLowercase -> "7" let tokenTypeDebug = function | Keyword -> "Keyword" @@ -31,6 +33,7 @@ module Token = struct | Namespace -> "Namespace" | EnumMember -> "EnumMember" | Property -> "Property" + | JsxLowercase -> "JsxLowercase" let tokenModifiersToString = function NoModifier -> "0" @@ -99,8 +102,8 @@ let emitFromLoc ~loc ~type_ emitter = emitter |> emitFromPos posStart posEnd ~type_ let emitLongident ?(backwards = false) ?(jsx = false) - ?(lowerCaseToken = Token.Variable) ?(upperCaseToken = Token.Namespace) ~pos - ~lid ~debug emitter = + ?(lowerCaseToken = if jsx then Token.JsxLowercase else Token.Variable) + ?(upperCaseToken = Token.Namespace) ~pos ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 2952b2d6a..053659ee0 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -11,18 +11,18 @@ Ldot: M (6,11) Namespace Lident: C (6,13) Namespace Variable: _mc (6,4)->(6,7) JsxTag <: (8,9) -Lident: div (8,10) Variable +Lident: div (8,10) JsxLowercase Variable: _d (8,4)->(8,6) JsxTag <: (11,2) -Lident: div (11,3) Variable -Lident: div (16,4) Variable +Lident: div (11,3) JsxLowercase +Lident: div (16,4) JsxLowercase JsxTag >: (11,6) JsxTag >: (16,7) Ldot: React (12,5) Namespace Lident: string (12,11) Variable JsxTag <: (13,4) -Lident: div (13,5) Variable -Lident: div (13,34) Variable +Lident: div (13,5) JsxLowercase +Lident: div (13,34) JsxLowercase JsxTag >: (13,8) JsxTag >: (13,37) Ldot: React (13,11) Namespace @@ -83,12 +83,12 @@ Variable: x (69,21)->(69,22) Variable: add (69,4)->(69,7) Lident: add (71,8) Variable JsxTag <: (73,8) -Lident: div (73,9) Variable -Lident: div (73,36) Variable +Lident: div (73,9) JsxLowercase +Lident: div (73,36) JsxLowercase JsxTag >: (73,24) JsxTag >: (73,39) JsxTag <: (73,26) -Lident: div (73,27) Variable +Lident: div (73,27) JsxLowercase Lident: SomeComponent (75,7) Namespace Lident: Nested (76,9) Namespace Lident: children (79,10) Variable @@ -102,5 +102,5 @@ Lident: Nested (84,55) Namespace JsxTag >: (84,29) JsxTag >: (84,61) JsxTag <: (84,31) -Lident: div (84,32) Variable +Lident: div (84,32) JsxLowercase diff --git a/package.json b/package.json index bdc9b6e58..4f3f47e5f 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "semanticTokenScopes": [ { "scopes": { + "jsx-lowercase": ["entity.name.tag"], "jsx-tag": ["punctuation.definition.tag"], "keyword-tag": ["keyword"] } diff --git a/server/src/server.ts b/server/src/server.ts index 540392e6e..fb1a9934a 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -768,6 +768,7 @@ function onMessage(msg: m.Message) { "namespace", "enumMember", "property", + "jsx-lowercase" ], tokenModifiers: [], }, From c5fb5914498bb4d44937e0d7921d8265580465eb Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Wed, 16 Mar 2022 12:49:00 +0100 Subject: [PATCH 43/68] add braces, and booleans to sample highlighting --- .../example-project/src/syntax/sample-highlighting.res | 2 ++ .../example-project/src/syntax/sample-highlighting.tsx | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.res b/analysis/examples/example-project/src/syntax/sample-highlighting.res index b3fd724d0..a458c55ea 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.res +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.res @@ -11,6 +11,7 @@ type someRecord<'typeParameter> = { someField: int, someOtherField: string, theParam: typeParameter, + another: bool, } type someEnum = @@ -33,6 +34,7 @@ let destructuring = () => { someField: 1, someOtherField: "hello", theParam: 2, + another: true, } let {someField, someOtherField, theParam} = someObj diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx index 4aa092c8a..e5a4be7ba 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -15,6 +15,7 @@ type someRecord = { someField: number; someOtherField: string; theParam: typeParameter; + another: boolean; }; enum someEnum { @@ -30,6 +31,7 @@ let destructuring = () => { someField: 1, someOtherField: "hello", theParam: 2, + another: true, }; let { someField, someOtherField, theParam } = someObj; @@ -64,5 +66,7 @@ let jsx = ( otherProp="hello" thirdProp={SomeModule.t.Value} /> + + {"Hello"}
); From 2941a60566cbbc1eeb3dd6c6db0166919bec7833 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Wed, 16 Mar 2022 18:16:36 +0100 Subject: [PATCH 44/68] color annotations like decorators in TS --- .../src/syntax/sample-highlighting.tsx | 20 +++++++++++++++++++ grammars/rescript.tmLanguage.json | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx index e5a4be7ba..4bfb916d9 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -46,6 +46,23 @@ namespace SomeModule { } } +// Decorators and classes +function someDecorator() { + return function ( + target: any, + propertyKey: string, + descriptor: PropertyDescriptor + ) { + console.log("first(): called"); + }; +} + +class SomeClass { + @someDecorator() doStuff() { + return 123; + } +} + // Strings let interpolated = `${numberBinding} ${"123"}`; @@ -70,3 +87,6 @@ let jsx = ( {"Hello"}
); +function Property() { + throw new Error("Function not implemented."); +} diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 4eb3e853b..dbdfb2733 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -308,7 +308,7 @@ "match": "(%%?|@@?)([A-Za-z_][A-Za-z0-9_\\.]*)", "captures": { "1": { - "name": "storage.modifier punctuation.definition.annotation" + "name": "punctuation.decorator" }, "2": { "patterns": [ @@ -327,13 +327,13 @@ "name": "invalid.deprecated" }, "2": { - "name": "variable.annotation" + "name": "entity.name.function" } } }, { "match": "[A-Za-z_][A-Za-z0-9_\\.]*", - "name": "variable.annotation" + "name": "entity.name.function" } ] } From 265e849e5592d0a74faf5cac22800960e79e1402 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 16 Mar 2022 18:30:17 +0100 Subject: [PATCH 45/68] Fix true/false. Don't emit semantic tokens for true/false. Remove redundant parts of grammar that were overriding booleans. --- analysis/src/SemanticTokens.ml | 3 +++ analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 2 +- grammars/rescript.tmLanguage.json | 9 +-------- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index c8909c88d..3499532e6 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -277,6 +277,9 @@ let parser ~debug ~emitter ~path = | Pexp_field (_, label) | Pexp_setfield (_, label, _) -> emitter |> emitRecordLabel ~label ~debug; Ast_mapper.default_mapper.expr mapper e + | Pexp_construct ({txt = Lident ("true" | "false")}, _) -> + (* Don't emit true or false *) + Ast_mapper.default_mapper.expr mapper e | Pexp_construct (name, _) -> emitter |> emitVariant ~name ~debug; Ast_mapper.default_mapper.expr mapper e diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index c5f548da8..90b53e108 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -83,3 +83,5 @@ module SomeComponent = { } let _ =
+ +let _ = true diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 053659ee0..bad3bb45e 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:22 diagnostics:0 +structure items:23 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index dbdfb2733..add55fdd9 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -15,7 +15,7 @@ }, "RE_KEYWORDS": { "name": "keyword.control", - "match": "\\b(and|as|assert|constraint|downto|else|exception|external|false|for|if|in|include|lazy|module|mutable|of|open|rec|switch|to|true|try|type|when|while|with)\\b" + "match": "\\b(and|as|assert|constraint|downto|else|exception|external|for|if|in|include|lazy|module|mutable|of|open|rec|switch|to|try|type|when|while|with)\\b" }, "RE_CONSTANTS_BOOL": { "name": "constant.language.boolean", @@ -25,10 +25,6 @@ "name": "keyword", "match": "\\b(let)\\b" }, - "RE_LITERAL": { - "name": "constant.language", - "match": "\\b(true|false)\\b" - }, "commentLine": { "match": "//.*", "name": "comment.line" @@ -107,9 +103,6 @@ }, "constant": { "patterns": [ - { - "include": "#RE_LITERAL" - }, { "include": "#RE_CONSTANTS_BOOL" } From b0349c02f0e76be06c1c071d343962d09f2112c8 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Wed, 16 Mar 2022 18:42:08 +0100 Subject: [PATCH 46/68] add to keyword as record prop in highlight examples --- .../examples/example-project/src/syntax/sample-highlighting.res | 2 ++ .../examples/example-project/src/syntax/sample-highlighting.tsx | 2 ++ 2 files changed, 4 insertions(+) diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.res b/analysis/examples/example-project/src/syntax/sample-highlighting.res index a458c55ea..3d8930c31 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.res +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.res @@ -12,6 +12,7 @@ type someRecord<'typeParameter> = { someOtherField: string, theParam: typeParameter, another: bool, + to: string, } type someEnum = @@ -35,6 +36,7 @@ let destructuring = () => { someOtherField: "hello", theParam: 2, another: true, + to: "123", } let {someField, someOtherField, theParam} = someObj diff --git a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx index 4bfb916d9..b646a4cfb 100644 --- a/analysis/examples/example-project/src/syntax/sample-highlighting.tsx +++ b/analysis/examples/example-project/src/syntax/sample-highlighting.tsx @@ -16,6 +16,7 @@ type someRecord = { someOtherField: string; theParam: typeParameter; another: boolean; + to: string; }; enum someEnum { @@ -32,6 +33,7 @@ let destructuring = () => { someOtherField: "hello", theParam: 2, another: true, + to: "123", }; let { someField, someOtherField, theParam } = someObj; From 71a7801248c22f44da086c595212c6772b50fed3 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 10:08:48 +0100 Subject: [PATCH 47/68] Don't crash when an id is the empty string. In case the user types `\""` --- analysis/src/SemanticTokens.ml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 3499532e6..d09328d95 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -82,10 +82,14 @@ let locToString (loc : Location.t) = Printf.sprintf "%s->%s" (posToString posStart) (posToString posEnd) let isLowercaseId id = + id <> "" + && let c = id.[0] in c == '_' || (c >= 'a' && c <= 'z') let isUppercaseId id = + id <> "" + && let c = id.[0] in c >= 'A' && c <= 'Z' From d371779723e6cfc2b0ebf8b5cb253d889b3fbaf6 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 10:51:46 +0100 Subject: [PATCH 48/68] Use the grammar to highlight when to/downto are used as labels. Labels are not handled by semantic highlighting (because there is no precise location information in the AST). But the grammar defines to/downto as keywords, in order to handle for loops. There are 3 cases where to/downto can be labels: 1) function definition `(~to as x) =>...` 2) function application `foo(~to=...)` 3) JSX props `
-let _ = true +// true/false +let _ = true || false + +// to/downto as label +let toAs = (~to as x) => x +let _toEquals = toAs(~to=10) + +let to = 1 +for _ in to + to to to + to { + () +} + +module ToAsProp = { + @react.component + let make = (~to) => { + <> {React.int(to)} + } +} +let _ = diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index bad3bb45e..61e0136a8 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:23 diagnostics:0 +structure items:29 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -103,4 +103,22 @@ JsxTag >: (84,29) JsxTag >: (84,61) JsxTag <: (84,31) Lident: div (84,32) JsxLowercase +Lident: x (90,25) Variable +Variable: x (90,19)->(90,20) +Variable: toAs (90,4)->(90,8) +Lident: toAs (91,16) Variable +Variable: _toEquals (91,4)->(91,13) +Variable: to (93,4)->(93,6) +Lident: to (94,20) Variable +Lident: to (94,25) Variable +Lident: to (94,9) Variable +Lident: to (94,14) Variable +Lident: ToAsProp (98,7) Namespace +Lident: to (101,18) Variable +Ldot: React (101,8) Namespace +Lident: int (101,14) Variable +Variable: to (100,14)->(100,17) +Variable: make (100,6)->(100,10) +JsxTag <: (104,8) +Lident: ToAsProp (104,9) Namespace diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index add55fdd9..ae038a319 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -17,6 +17,32 @@ "name": "keyword.control", "match": "\\b(and|as|assert|constraint|downto|else|exception|external|for|if|in|include|lazy|module|mutable|of|open|rec|switch|to|try|type|when|while|with)\\b" }, + "RE_TO_DOWNTO_AS_LABELS": { + "patterns": [ + { + "match": "(to|downto)\\s*(=)", + "captures": { + "1": { + "name": "variable" + }, + "2": { + "name": "keyword.operator keyword" + } + } + }, + { + "match": "(to|downto)\\s*(as)", + "captures": { + "1": { + "name": "variable" + }, + "2": { + "name": "keyword.control" + } + } + } + ] + }, "RE_CONSTANTS_BOOL": { "name": "constant.language.boolean", "match": "\\b(false|true)\\b" @@ -93,6 +119,9 @@ }, "keyword": { "patterns": [ + { + "include": "#RE_TO_DOWNTO_AS_LABELS" + }, { "include": "#RE_KEYWORDS" }, From 7c0b29779625f7702bb16e74440e2c881f0a658d Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 11:09:17 +0100 Subject: [PATCH 49/68] Clean up jsx grammar and comment jsx code. The jsx grammar only needs to take care of `<>` and `` and ``. The `<` and `>` cases are handled via semantic highlighting. And the ids are handled via semantic highlighting just like ids in any other context. --- analysis/src/SemanticTokens.ml | 12 +++++++-- grammars/rescript.tmLanguage.json | 41 ++----------------------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index d09328d95..52a8cdf4a 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -217,6 +217,14 @@ let parser ~debug ~emitter ~path = Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) when Res_parsetree_viewer.isJsxExpression e -> + (* + Angled brackets: + - These are handled in the grammar: <> + - Here we handle `<` and `>` + + Component names: + - handled like other Longitent.t, except lowercase id is marked Token.JsxLowercase + *) let rec isSelfClosing args = match args with | [] -> false @@ -231,7 +239,7 @@ let parser ~debug ~emitter ~path = true | _ :: rest -> isSelfClosing rest in - emitter + emitter (* --> emitJsxTag ~debug ~name:"<" ~pos: (let pos = Utils.tupleOfLexing e.pexp_loc.loc_start in @@ -262,7 +270,7 @@ let parser ~debug ~emitter ~path = ~pos: (let pos = Utils.tupleOfLexing e.pexp_loc.loc_end in (fst pos, snd pos - 1)))); - (* only process again arguments, not the jsx label *) + let _ = args |> List.map (fun (_lbl, arg) -> mapper.expr mapper arg) in e | Pexp_apply diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index ae038a319..e9ea0e9a5 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -364,45 +364,8 @@ ] }, "jsx": { - "patterns": [ - { - "match": "<>||/>", - "name": "punctuation.definition.tag" - }, - { - "match": "||", + "name": "punctuation.definition.tag" }, "openOrIncludeModule": { "patterns": [ From 1e076253fe0a692fbbc009d306b3d3f1ac35a8f8 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 11:56:24 +0100 Subject: [PATCH 50/68] More robust jsx self-closing test. Does not rely on the code to be formatted in order to determine if there are children. --- analysis/src/SemanticTokens.ml | 56 +++++++++++++++------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 52a8cdf4a..9691917b1 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -225,27 +225,33 @@ let parser ~debug ~emitter ~path = Component names: - handled like other Longitent.t, except lowercase id is marked Token.JsxLowercase *) - let rec isSelfClosing args = - match args with - | [] -> false - | [ - ( Asttypes.Labelled "children", - { - Parsetree.pexp_desc = - Pexp_construct ({txt = Longident.Lident "[]"}, None); - } ); - _; - ] -> - true - | _ :: rest -> isSelfClosing rest - in - emitter (* --> emitJsxTag ~debug ~name:"<" ~pos: (let pos = Utils.tupleOfLexing e.pexp_loc.loc_start in (fst pos, snd pos - 1 (* the AST skips the loc of < somehow *))); emitter |> emitJsxOpen ~lid:lident.txt ~debug ~loc:pexp_loc; - (if not (isSelfClosing args) then + + let posOfGreatherthanAfterProps = + let rec loop = function + | (Asttypes.Labelled "children", {Parsetree.pexp_loc = {loc_start}}) + :: _ -> + Utils.tupleOfLexing loc_start + | _ :: args -> loop args + | [] -> (-1, -1) + (* should not happen *) + in + loop args + in + let posOfFinalGreatherthan = + let pos = Utils.tupleOfLexing e.pexp_loc.loc_end in + (fst pos, snd pos - 1) + in + let selfClosing = + fst posOfGreatherthanAfterProps == fst posOfFinalGreatherthan + && snd posOfGreatherthanAfterProps + 1 == snd posOfFinalGreatherthan + in + (if not selfClosing then let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in let lineEnd, colEnd = Utils.tupleOfLexing pexp_loc.loc_end in let length = if lineStart = lineEnd then colEnd - colStart else 0 in @@ -254,22 +260,10 @@ let parser ~debug ~emitter ~path = emitter |> emitJsxClose ~debug ~lid:lident.txt ~pos:(lineEndWhole, colEndWhole - 1); - - let rec emitGreatherthanAfterProps args = - match args with - | (Asttypes.Labelled "children", {Parsetree.pexp_loc = {loc_start}}) - :: _ -> - emitter - |> emitJsxTag ~debug ~name:">" ~pos:(Utils.tupleOfLexing loc_start) - | _ :: args -> emitGreatherthanAfterProps args - | [] -> () - in - emitGreatherthanAfterProps args (* <-- *); + emitter (* <-- *) + |> emitJsxTag ~debug ~name:">" ~pos:posOfGreatherthanAfterProps; emitter (* ... <-- *) - |> emitJsxTag ~debug ~name:">" - ~pos: - (let pos = Utils.tupleOfLexing e.pexp_loc.loc_end in - (fst pos, snd pos - 1)))); + |> emitJsxTag ~debug ~name:">" ~pos:posOfFinalGreatherthan)); let _ = args |> List.map (fun (_lbl, arg) -> mapper.expr mapper arg) in e From 814a6568d06388a9371a6d62bc5a33ffe6f292e5 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 12:17:15 +0100 Subject: [PATCH 51/68] Handle quoted Pexp_ident. --- analysis/src/SemanticTokens.ml | 11 +++++++++++ analysis/tests/src/Parser.res | 4 ++++ analysis/tests/src/expected/Parser.res.txt | 4 +++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 9691917b1..fda865052 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -211,6 +211,17 @@ let parser ~debug ~emitter ~path = in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = match e.pexp_desc with + | Pexp_ident {txt = Lident id} + when snd (Utils.tupleOfLexing e.pexp_loc.loc_end) + - snd (Utils.tupleOfLexing e.pexp_loc.loc_start) + > String.length id (* /"stuff" *) -> + let type_ = Token.Variable in + if debug then + Printf.printf "QuotedIdent: %s %s %s\n" id + (posToString (Utils.tupleOfLexing e.pexp_loc.loc_start)) + (Token.tokenTypeDebug type_); + emitter |> emitFromLoc ~loc:e.pexp_loc ~type_; + Ast_mapper.default_mapper.expr mapper e | Pexp_ident {txt = lid; loc} -> emitter |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 1f5cc0fa6..8452eee50 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -103,3 +103,7 @@ module ToAsProp = { } } let _ = + +// quoted identifiers +let \"true" = 4 +let _ = \"true" diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 61e0136a8..8794ab84d 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:29 diagnostics:0 +structure items:31 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -121,4 +121,6 @@ Variable: to (100,14)->(100,17) Variable: make (100,6)->(100,10) JsxTag <: (104,8) Lident: ToAsProp (104,9) Namespace +Variable: true (107,4)->(107,11) +QuotedIdent: true (108,8) Variable From c1f574bd459712b73a70354345517708de9c3df5 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 14:29:05 +0100 Subject: [PATCH 52/68] True/false: pattern case was missing. --- analysis/src/SemanticTokens.ml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index fda865052..508d3e4ee 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -200,6 +200,9 @@ let parser ~debug ~emitter ~path = if isLowercaseId id then emitter |> emitVariable ~id ~debug ~loc:p.ppat_loc; Ast_mapper.default_mapper.pat mapper p + | Ppat_construct ({txt = Lident ("true" | "false")}, _) -> + (* Don't emit true or false *) + Ast_mapper.default_mapper.pat mapper p | Ppat_record (cases, _) -> cases |> List.iter (fun (label, _) -> emitter |> emitRecordLabel ~label ~debug); @@ -214,7 +217,8 @@ let parser ~debug ~emitter ~path = | Pexp_ident {txt = Lident id} when snd (Utils.tupleOfLexing e.pexp_loc.loc_end) - snd (Utils.tupleOfLexing e.pexp_loc.loc_start) - > String.length id (* /"stuff" *) -> + > String.length id + (* /"stuff" *) -> let type_ = Token.Variable in if debug then Printf.printf "QuotedIdent: %s %s %s\n" id From a321a9b4b7ddb25a87cf8711949816ab5be5e124 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 14:32:56 +0100 Subject: [PATCH 53/68] Example with enum in module. --- analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 8452eee50..e212cd905 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -107,3 +107,5 @@ let _ = // quoted identifiers let \"true" = 4 let _ = \"true" + +let enumInModule = T.A diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 8794ab84d..a2392980f 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:31 diagnostics:0 +structure items:32 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -123,4 +123,7 @@ JsxTag <: (104,8) Lident: ToAsProp (104,9) Namespace Variable: true (107,4)->(107,11) QuotedIdent: true (108,8) Variable +Ldot: T (110,19) EnumMember +Lident: A (110,21) EnumMember +Variable: enumInModule (110,4)->(110,16) From e3d93cecbe6752bf39a7c6fefa22bc1f5ff96845 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 14:43:40 +0100 Subject: [PATCH 54/68] Fix issue with enum in module. --- analysis/src/SemanticTokens.ml | 11 ++++++++--- analysis/tests/src/expected/Parser.res.txt | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 508d3e4ee..b8748692a 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -107,7 +107,8 @@ let emitFromLoc ~loc ~type_ emitter = let emitLongident ?(backwards = false) ?(jsx = false) ?(lowerCaseToken = if jsx then Token.JsxLowercase else Token.Variable) - ?(upperCaseToken = Token.Namespace) ~pos ~lid ~debug emitter = + ?(upperCaseToken = Token.Namespace) ?(lastToken = None) ~pos ~lid ~debug + emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -119,7 +120,11 @@ let emitLongident ?(backwards = false) ?(jsx = false) let rec loop pos segments = match segments with | [id] when isUppercaseId id || isLowercaseId id -> - let type_ = if isUppercaseId id then upperCaseToken else lowerCaseToken in + let type_ = + match lastToken with + | Some type_ -> type_ + | None -> if isUppercaseId id then upperCaseToken else lowerCaseToken + in if debug then Printf.printf "Lident: %s %s %s\n" id (posToString pos) (Token.tokenTypeDebug type_); @@ -170,7 +175,7 @@ let emitRecordLabel ~(label : Longident.t Location.loc) ~debug emitter = let emitVariant ~(name : Longident.t Location.loc) ~debug emitter = emitter - |> emitLongident ~upperCaseToken:Token.EnumMember + |> emitLongident ~lastToken:(Some Token.EnumMember) ~pos:(Utils.tupleOfLexing name.loc.loc_start) ~lid:name.txt ~debug diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index a2392980f..1c9688dce 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -123,7 +123,7 @@ JsxTag <: (104,8) Lident: ToAsProp (104,9) Namespace Variable: true (107,4)->(107,11) QuotedIdent: true (108,8) Variable -Ldot: T (110,19) EnumMember +Ldot: T (110,19) Namespace Lident: A (110,21) EnumMember Variable: enumInModule (110,4)->(110,16) From ccf81fc79fec499ef6b0471ff102af9610d926b2 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 15:05:25 +0100 Subject: [PATCH 55/68] Fix issue where type paths longer than 1 were not emitted. --- analysis/src/SemanticTokens.ml | 17 ++++----- analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 42 ++++++++++++---------- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index b8748692a..d269f3875 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -163,9 +163,11 @@ let emitJsxTag ~debug ~name ~pos emitter = if debug then Printf.printf "JsxTag %s: %s\n" name (posToString pos); emitter |> emitFromPos pos (fst pos, snd pos + 1) ~type_:Token.JsxTag -let emitType ~id ~debug ~loc emitter = - if debug then Printf.printf "Type: %s %s\n" id (locToString loc); - emitter |> emitFromLoc ~loc ~type_:Token.Type +let emitType ~lid ~debug ~loc emitter = + emitter + |> emitLongident ~lowerCaseToken:Token.Type + ~pos:(Utils.tupleOfLexing loc.Location.loc_start) + ~lid ~debug let emitRecordLabel ~(label : Longident.t Location.loc) ~debug emitter = emitter @@ -185,10 +187,8 @@ let parser ~debug ~emitter ~path = in let typ (mapper : Ast_mapper.mapper) (coreType : Parsetree.core_type) = match coreType.ptyp_desc with - | Ptyp_constr ({txt; loc}, args) -> - (match txt with - | Lident id -> emitter |> emitType ~id ~debug ~loc - | _ -> ()); + | Ptyp_constr ({txt = lid; loc}, args) -> + emitter |> emitType ~lid ~debug ~loc; args |> List.iter processTypeArg; Ast_mapper.default_mapper.typ mapper coreType | _ -> Ast_mapper.default_mapper.typ mapper coreType @@ -196,7 +196,8 @@ let parser ~debug ~emitter ~path = let type_declaration (mapper : Ast_mapper.mapper) (tydecl : Parsetree.type_declaration) = emitter - |> emitType ~id:tydecl.ptype_name.txt ~debug ~loc:tydecl.ptype_name.loc; + |> emitType ~lid:(Lident tydecl.ptype_name.txt) ~debug + ~loc:tydecl.ptype_name.loc; Ast_mapper.default_mapper.type_declaration mapper tydecl in let pat (mapper : Ast_mapper.mapper) (p : Parsetree.pattern) = diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index e212cd905..bb1db5659 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -109,3 +109,5 @@ let \"true" = 4 let _ = \"true" let enumInModule = T.A + +type typeInModule = XX.YY.t diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 1c9688dce..6a2722607 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:32 diagnostics:0 +structure items:33 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -32,19 +32,19 @@ Lident: string (14,11) Variable Ldot: React (15,5) Namespace Lident: string (15,11) Variable Variable: _d2 (10,4)->(10,7) -Type: pair (18,5)->(18,9) -Type: looooooooooooooooooooooooooooooooooooooong_int (20,5)->(20,51) -Type: int (20,54)->(20,57) -Type: looooooooooooooooooooooooooooooooooooooong_string (22,5)->(22,54) -Type: string (22,57)->(22,63) -Type: pairIntString (24,5)->(24,18) -Type: list (24,21)->(24,25) +Lident: pair (18,5) Type +Lident: looooooooooooooooooooooooooooooooooooooong_int (20,5) Type +Lident: int (20,54) Type +Lident: looooooooooooooooooooooooooooooooooooooong_string (22,5) Type +Lident: string (22,57) Type +Lident: pairIntString (24,5) Type +Lident: list (24,21) Type TypeArg: (25,2)->(28,3) -Type: pair (25,2)->(25,6) +Lident: pair (25,2) Type TypeArg: (26,4)->(26,50) TypeArg: (27,4)->(27,53) -Type: looooooooooooooooooooooooooooooooooooooong_int (26,4)->(26,50) -Type: looooooooooooooooooooooooooooooooooooooong_string (27,4)->(27,53) +Lident: looooooooooooooooooooooooooooooooooooooong_int (26,4) Type +Lident: looooooooooooooooooooooooooooooooooooooong_string (27,4) Type Binary operator < (31,10)->(31,11) Binary operator > (31,19)->(31,20) Lident: MT (33,12) Type @@ -54,20 +54,20 @@ Lident: MT (39,12) Type Lident: DDF (40,9) Namespace Lident: XX (45,7) Namespace Lident: YY (46,9) Namespace -Type: t (47,9)->(47,10) -Type: int (47,13)->(47,16) +Lident: t (47,9) Type +Lident: int (47,13) Type Ldot: XX (51,5) Namespace Lident: YY (51,8) Namespace -Type: tt (53,5)->(53,7) -Type: t (53,10)->(53,11) +Lident: tt (53,5) Type +Lident: t (53,10) Type Lident: T (57,7) Namespace -Type: someRecord (58,7)->(58,17) +Lident: someRecord (58,7) Type Lident: someField (59,4) Property -Type: int (59,15)->(59,18) +Lident: int (59,15) Type Lident: someOtherField (60,4) Property -Type: string (60,20)->(60,26) +Lident: string (60,20) Type Lident: theParam (61,4) Property -Type: someEnum (64,7)->(64,15) +Lident: someEnum (64,7) Type Lident: A (64,18) EnumMember Lident: B (64,22) EnumMember Lident: C (64,26) EnumMember @@ -126,4 +126,8 @@ QuotedIdent: true (108,8) Variable Ldot: T (110,19) Namespace Lident: A (110,21) EnumMember Variable: enumInModule (110,4)->(110,16) +Lident: typeInModule (112,5) Type +Ldot: XX (112,20) Namespace +Ldot: YY (112,23) Namespace +Lident: t (112,26) Type From ecda4c1c7d59a6fd3f57a1b5be342f8fb7129039 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Thu, 17 Mar 2022 16:09:40 +0100 Subject: [PATCH 56/68] Update grammar for poly variants, and semantic analysis for types #...someType. - #foo is marked as enum - space is allowed: # foo - #...foo. is a type --- analysis/src/SemanticTokens.ml | 3 +++ analysis/tests/src/Parser.res | 15 +++++++++++++++ analysis/tests/src/expected/Parser.res.txt | 11 ++++++++++- grammars/rescript.tmLanguage.json | 20 ++++++++++++++------ 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index d269f3875..f8ec57b96 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -216,6 +216,9 @@ let parser ~debug ~emitter ~path = | Ppat_construct (name, _) -> emitter |> emitVariant ~name ~debug; Ast_mapper.default_mapper.pat mapper p + | Ppat_type {txt = lid; loc} -> + emitter |> emitType ~lid ~debug ~loc; + Ast_mapper.default_mapper.pat mapper p | _ -> Ast_mapper.default_mapper.pat mapper p in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index bb1db5659..acbb133a3 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -111,3 +111,18 @@ let _ = \"true" let enumInModule = T.A type typeInModule = XX.YY.t + +module QQ = { + type somePolyEnumType = [ + | #someMember + | #AnotherMember + | #SomeMemberWithPayload(list) + | #"fourth Member" + ] +} + +let _ = x => + switch x { + | #stuff => 3 + | #...QQ.somePolyEnumType => 4 + } diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 6a2722607..a7f38f1c6 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:33 diagnostics:0 +structure items:35 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -130,4 +130,13 @@ Lident: typeInModule (112,5) Type Ldot: XX (112,20) Namespace Ldot: YY (112,23) Namespace Lident: t (112,26) Type +Lident: QQ (114,7) Namespace +Lident: somePolyEnumType (115,7) Type +Lident: list (118,29) Type +TypeArg: (118,34)->(118,37) +Lident: int (118,34) Type +Ldot: QQ (126,8) Namespace +Lident: somePolyEnumType (126,11) Type +Lident: x (124,9) Variable +Variable: x (123,8)->(123,9) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index e9ea0e9a5..449153f34 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -270,16 +270,24 @@ "name": "variable.function variable.other" }, { - "match": "(#)(\\.\\.\\.)?([a-zA-Z][0-9a-zA-Z_]*)\\b", + "match": "(#)\\s*([a-zA-Z][0-9a-zA-Z_]*)\\b", "captures": { "1": { - "name": "punctuation.definition.keyword" + "name": "variable.other.enummember" }, "2": { - "name": "punctuation.definition.keyword" + "name": "variable.other.enummember" + } + } + }, + { + "match": "(#)\\s*(\\.\\.\\.)\\b", + "captures": { + "1": { + "name": "variable.other.enummember" }, - "3": { - "name": "variable.function variable.other" + "2": { + "name": "variable.other.enummember" } } }, @@ -287,7 +295,7 @@ "match": "(#)", "captures": { "1": { - "name": "punctuation.definition.keyword" + "name": "variable.other.enummember" } } } From 30c922af2b531e3fef9470794e18deb55942b3f8 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 18 Mar 2022 08:54:38 +0100 Subject: [PATCH 57/68] Highlight operators. --- analysis/src/SemanticTokens.ml | 8 ++++---- grammars/rescript.tmLanguage.json | 2 +- package.json | 3 +-- server/src/server.ts | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index f8ec57b96..3ee55071d 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -4,7 +4,7 @@ module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) type tokenType = - | Keyword + | Operator | Variable | Type | JsxTag @@ -16,7 +16,7 @@ module Token = struct type tokenModifiers = NoModifier let tokenTypeToString = function - | Keyword -> "0" + | Operator -> "0" | Variable -> "1" | Type -> "2" | JsxTag -> "3" @@ -26,7 +26,7 @@ module Token = struct | JsxLowercase -> "7" let tokenTypeDebug = function - | Keyword -> "Keyword" + | Operator -> "Operator" | Variable -> "Variable" | Type -> "Type" | JsxTag -> "JsxTag" @@ -298,7 +298,7 @@ let parser ~debug ~emitter ~path = }, [_; _] ) -> if debug then Printf.printf "Binary operator %s %s\n" op (locToString loc); - emitter |> emitFromLoc ~loc ~type_:Keyword; + emitter |> emitFromLoc ~loc ~type_:Operator; Ast_mapper.default_mapper.expr mapper e | Pexp_record (cases, _) -> cases diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 449153f34..c790b3dad 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -255,7 +255,7 @@ "patterns": [ { "match": "->|\\|\\||&&|\\+\\+|\\*\\*|\\+\\.|\\+|-\\.|-|\\*\\.|\\*|/\\.|/|\\.\\.\\.|\\.\\.|===|==|\\^|:=|!|>=(?! *\\?)|<=|=", - "name": "keyword.operator keyword" + "name": "keyword.operator" }, { "match": "\\|>", diff --git a/package.json b/package.json index 4f3f47e5f..17563d644 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,7 @@ { "scopes": { "jsx-lowercase": ["entity.name.tag"], - "jsx-tag": ["punctuation.definition.tag"], - "keyword-tag": ["keyword"] + "jsx-tag": ["punctuation.definition.tag"] } } ], diff --git a/server/src/server.ts b/server/src/server.ts index fb1a9934a..3e4802ff3 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -761,7 +761,7 @@ function onMessage(msg: m.Message) { semanticTokensProvider: { legend: { tokenTypes: [ - "keyword-tag", + "operator", "variable", "type", "jsx-tag", From 373aeadee6e89aabf19554d7811d5521654dca31 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 18 Mar 2022 09:14:12 +0100 Subject: [PATCH 58/68] Fix: ! is represented internally as "not". --- analysis/src/SemanticTokens.ml | 5 +++-- analysis/tests/src/Parser.res | 2 +- analysis/tests/src/expected/Parser.res.txt | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 3ee55071d..82d68ab5a 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -236,8 +236,9 @@ let parser ~debug ~emitter ~path = emitter |> emitFromLoc ~loc:e.pexp_loc ~type_; Ast_mapper.default_mapper.expr mapper e | Pexp_ident {txt = lid; loc} -> - emitter - |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; + if lid <> Lident "not" then + emitter + |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) when Res_parsetree_viewer.isJsxExpression e -> diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index acbb133a3..92ae97e87 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -29,7 +29,7 @@ type pairIntString = list< >, > -let _ = 3 < 4 || 3 > 4 +let _ = !(3 < 4) || 3 > 4 module type MT = { module DDF: { diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index a7f38f1c6..9de3ddf0f 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -45,8 +45,8 @@ TypeArg: (26,4)->(26,50) TypeArg: (27,4)->(27,53) Lident: looooooooooooooooooooooooooooooooooooooong_int (26,4) Type Lident: looooooooooooooooooooooooooooooooooooooong_string (27,4) Type -Binary operator < (31,10)->(31,11) -Binary operator > (31,19)->(31,20) +Binary operator < (31,12)->(31,13) +Binary operator > (31,22)->(31,23) Lident: MT (33,12) Type Lident: DDF (34,9) Namespace Lident: DDF (39,7) Namespace From 1fdec924b4f91cfc96df45467f2b24faaac2340e Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 18 Mar 2022 09:21:41 +0100 Subject: [PATCH 59/68] Fix == and ===. --- analysis/src/SemanticTokens.ml | 7 ++++--- analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 82d68ab5a..77ee586c9 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -224,9 +224,10 @@ let parser ~debug ~emitter ~path = let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = match e.pexp_desc with | Pexp_ident {txt = Lident id} - when snd (Utils.tupleOfLexing e.pexp_loc.loc_end) - - snd (Utils.tupleOfLexing e.pexp_loc.loc_start) - > String.length id + when id <> "=" && id <> "==" + && snd (Utils.tupleOfLexing e.pexp_loc.loc_end) + - snd (Utils.tupleOfLexing e.pexp_loc.loc_start) + > String.length id (* /"stuff" *) -> let type_ = Token.Variable in if debug then diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 92ae97e87..e3a701ec5 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -126,3 +126,5 @@ let _ = x => | #stuff => 3 | #...QQ.somePolyEnumType => 4 } + +let _ = 3 == 3 || 3 === 3 diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 9de3ddf0f..3b3489299 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:35 diagnostics:0 +structure items:36 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace From 4001ffe750afe4911201384532c29a6353c7ee0d Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Sat, 19 Mar 2022 12:47:57 +0100 Subject: [PATCH 60/68] dummy commit to see if CI runs --- server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index 3e4802ff3..275bf6680 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -768,7 +768,7 @@ function onMessage(msg: m.Message) { "namespace", "enumMember", "property", - "jsx-lowercase" + "jsx-lowercase", ], tokenModifiers: [], }, From 2b96d8f86e192d4a38d1e913d24dca2236acbfe5 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 18 Mar 2022 09:44:24 +0100 Subject: [PATCH 61/68] Labels can have underscore. --- analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 2 +- grammars/rescript.tmLanguage.json | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index e3a701ec5..415194a1b 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -128,3 +128,5 @@ let _ = x => } let _ = 3 == 3 || 3 === 3 + +let _ = (~_type_ as _) => () diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 3b3489299..f02a53855 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:36 diagnostics:0 +structure items:37 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index c790b3dad..90c7925fe 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -238,7 +238,7 @@ "defaultIdIsVariable": { "patterns": [ { - "match": "[A-Za-z][A-Za-z0-9]*", + "match": "[A-Za-z_][A-Za-z0-9_]*", "name": "variable" } ] From d735fb1bade595a2c0f9342d44895f1d30dd1bce Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Fri, 18 Mar 2022 10:15:57 +0100 Subject: [PATCH 62/68] Added explanatory comments on top of SemanticTokens.ml. --- analysis/src/SemanticTokens.ml | 44 ++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 77ee586c9..e2f5029cc 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -1,17 +1,33 @@ -module Token = struct - type legend = {tokenTypes : string array; tokenModifiers : string array} +(* + Generally speaking, semantic highlighting here takes care of categorizing identifiers, + since the kind of an identifier is highly context-specific and hard to catch with a grammar. + + The big exception is labels, whose location is not represented in the AST + E.g. function definition such as (~foo as _) =>, application (~foo=3) and prop
. + Labels are handled in the grammar, not here. + Punned labels such as (~foo) => are both labels and identifiers. They are overridden here. + + There are 2 cases where the grammar and semantic highlighting work jointly. + The styles emitted in the grammar and here need to be kept in sync. + 1) For jsx angled brackets, the grammar handles basic cases such as /> + whose location is not in the AST. + Instead < and > are handled here. Those would be difficult to disambiguate in a grammar. + 2) Most operators are handled in the grammar. Except < and > are handled here. + The reason is again that < and > would be difficult do disambiguate in a grammar. +*) +module Token = struct (* This needs to stay synced with the same legend in `server.ts` *) (* See https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens *) type tokenType = - | Operator - | Variable - | Type - | JsxTag - | Namespace - | EnumMember - | Property - | JsxLowercase + | Operator (** < and > *) + | Variable (** let x = *) + | Type (** type t = *) + | JsxTag (** the < and > in
*) + | Namespace (** module M = *) + | EnumMember (** variant A or poly variant #A *) + | Property (** {x:...} *) + | JsxLowercase (** div in
*) type tokenModifiers = NoModifier @@ -47,7 +63,8 @@ module Token = struct let createEmitter () = {tokens = []; lastLine = 0; lastChar = 0} - let add ~line ~char ~length ~type_ ?(modifiers = NoModifier) e = + let add ~line ~char ~length ~type_ e = + let modifiers = NoModifier in e.tokens <- (line, char, length, type_, modifiers) :: e.tokens let emitToken buf (line, char, length, type_, modifiers) e = @@ -264,9 +281,9 @@ let parser ~debug ~emitter ~path = :: _ -> Utils.tupleOfLexing loc_start | _ :: args -> loop args - | [] -> (-1, -1) - (* should not happen *) + | [] -> (* should not happen *) (-1, -1) in + loop args in let posOfFinalGreatherthan = @@ -276,6 +293,7 @@ let parser ~debug ~emitter ~path = let selfClosing = fst posOfGreatherthanAfterProps == fst posOfFinalGreatherthan && snd posOfGreatherthanAfterProps + 1 == snd posOfFinalGreatherthan + (* there's an off-by one somehow in the AST *) in (if not selfClosing then let lineStart, colStart = Utils.tupleOfLexing pexp_loc.loc_start in From 2d7fcd11a3c3f26b5e5eb3d7e3649c95063c89d1 Mon Sep 17 00:00:00 2001 From: Gabriel Nordeborn Date: Sat, 19 Mar 2022 12:50:02 +0100 Subject: [PATCH 63/68] ci? --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index acdc84ea0..b7ebf7ff4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ analysis/tests/.bsb.lock analysis/_build analysis/tests/.merlin analysis/rescript-editor-analysis.exe + From ff026ef69b15ca201c66f6dea6592b1566e5d2c2 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Sun, 20 Mar 2022 09:32:16 +0100 Subject: [PATCH 64/68] Fix object fields (and clean uo quoted ids). --- analysis/src/SemanticTokens.ml | 36 +++++++++++----------- analysis/tests/src/Parser.res | 2 ++ analysis/tests/src/expected/Parser.res.txt | 5 +-- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index e2f5029cc..0fa9af5a6 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -124,8 +124,8 @@ let emitFromLoc ~loc ~type_ emitter = let emitLongident ?(backwards = false) ?(jsx = false) ?(lowerCaseToken = if jsx then Token.JsxLowercase else Token.Variable) - ?(upperCaseToken = Token.Namespace) ?(lastToken = None) ~pos ~lid ~debug - emitter = + ?(upperCaseToken = Token.Namespace) ?(lastToken = None) ?(posEnd = None) + ~pos ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc @@ -142,10 +142,19 @@ let emitLongident ?(backwards = false) ?(jsx = false) | Some type_ -> type_ | None -> if isUppercaseId id then upperCaseToken else lowerCaseToken in + let posAfter = (fst pos, snd pos + String.length id) in + let posEnd, lenMismatch = + (* There could be a length mismatch when ids are quoted + e.g. variable /"true" or object field {"x":...} *) + match posEnd with + | Some posEnd -> (posEnd, posEnd <> posAfter) + | None -> (posAfter, false) + in if debug then - Printf.printf "Lident: %s %s %s\n" id (posToString pos) + Printf.printf "Lident: %s %s%s %s\n" id (posToString pos) + (if lenMismatch then "->" ^ posToString posEnd else "") (Token.tokenTypeDebug type_); - emitter |> emitFromPos pos (fst pos, snd pos + String.length id) ~type_ + emitter |> emitFromPos pos posEnd ~type_ | id :: segments when isUppercaseId id || isLowercaseId id -> let type_ = if isUppercaseId id then upperCaseToken else lowerCaseToken in if debug then @@ -190,6 +199,7 @@ let emitRecordLabel ~(label : Longident.t Location.loc) ~debug emitter = emitter |> emitLongident ~lowerCaseToken:Token.Property ~pos:(Utils.tupleOfLexing label.loc.loc_start) + ~posEnd:(Some (Utils.tupleOfLexing label.loc.loc_end)) ~lid:label.txt ~debug let emitVariant ~(name : Longident.t Location.loc) ~debug emitter = @@ -240,23 +250,13 @@ let parser ~debug ~emitter ~path = in let expr (mapper : Ast_mapper.mapper) (e : Parsetree.expression) = match e.pexp_desc with - | Pexp_ident {txt = Lident id} - when id <> "=" && id <> "==" - && snd (Utils.tupleOfLexing e.pexp_loc.loc_end) - - snd (Utils.tupleOfLexing e.pexp_loc.loc_start) - > String.length id - (* /"stuff" *) -> - let type_ = Token.Variable in - if debug then - Printf.printf "QuotedIdent: %s %s %s\n" id - (posToString (Utils.tupleOfLexing e.pexp_loc.loc_start)) - (Token.tokenTypeDebug type_); - emitter |> emitFromLoc ~loc:e.pexp_loc ~type_; - Ast_mapper.default_mapper.expr mapper e | Pexp_ident {txt = lid; loc} -> if lid <> Lident "not" then emitter - |> emitLongident ~pos:(Utils.tupleOfLexing loc.loc_start) ~lid ~debug; + |> emitLongident + ~pos:(Utils.tupleOfLexing loc.loc_start) + ~posEnd:(Some (Utils.tupleOfLexing loc.loc_end)) + ~lid ~debug; Ast_mapper.default_mapper.expr mapper e | Pexp_apply ({pexp_desc = Pexp_ident lident; pexp_loc}, args) when Res_parsetree_viewer.isJsxExpression e -> diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Parser.res index 415194a1b..ec60ff131 100644 --- a/analysis/tests/src/Parser.res +++ b/analysis/tests/src/Parser.res @@ -130,3 +130,5 @@ let _ = x => let _ = 3 == 3 || 3 === 3 let _ = (~_type_ as _) => () + +let _ = {"abc": 34} diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index f02a53855..5748dd607 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,5 +1,5 @@ Parse tests/src/Parser.res -structure items:37 diagnostics:0 +structure items:38 diagnostics:0 Lident: M (0,7) Namespace Lident: C (1,9) Namespace Lident: Component (1,13) Namespace @@ -122,7 +122,7 @@ Variable: make (100,6)->(100,10) JsxTag <: (104,8) Lident: ToAsProp (104,9) Namespace Variable: true (107,4)->(107,11) -QuotedIdent: true (108,8) Variable +Lident: true (108,8)->(108,15) Variable Ldot: T (110,19) Namespace Lident: A (110,21) EnumMember Variable: enumInModule (110,4)->(110,16) @@ -139,4 +139,5 @@ Ldot: QQ (126,8) Namespace Lident: somePolyEnumType (126,11) Type Lident: x (124,9) Variable Variable: x (123,8)->(123,9) +Lident: abc (133,9)->(133,14) Property From 9bd8ba38a4fea6fad97f8c6ce65c29a5b518910f Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Tue, 22 Mar 2022 08:57:31 +0100 Subject: [PATCH 65/68] Clean up textmate grammar. Clean up keywords, and remove variant constructors and module access, which are handled at the semantic level. --- grammars/rescript.tmLanguage.json | 122 ++---------------------------- 1 file changed, 6 insertions(+), 116 deletions(-) diff --git a/grammars/rescript.tmLanguage.json b/grammars/rescript.tmLanguage.json index 90c7925fe..9ed6c4f42 100644 --- a/grammars/rescript.tmLanguage.json +++ b/grammars/rescript.tmLanguage.json @@ -3,19 +3,9 @@ "name": "ReScript", "scopeName": "source.rescript", "repository": { - "RE_IDENT": { - "match": "[a-z_][0-9a-zA-Z_]*" - }, - "RE_ATTRIBUTE": { - "match": "[A-Za-z_][A-Za-z0-9_\\.]*" - }, - "RE_MODULE_IDENT": { - "name": "entity.name.namespace", - "match": "[A-Z_][0-9a-zA-Z_]*" - }, - "RE_KEYWORDS": { + "RE_KEYWORD_CONTROL": { "name": "keyword.control", - "match": "\\b(and|as|assert|constraint|downto|else|exception|external|for|if|in|include|lazy|module|mutable|of|open|rec|switch|to|try|type|when|while|with)\\b" + "match": "\\b(and|as|assert|constraint|downto|else|exception|external|for|if|in|lazy|mutable|rec|switch|to|try|when|while|with)\\b" }, "RE_TO_DOWNTO_AS_LABELS": { "patterns": [ @@ -47,9 +37,9 @@ "name": "constant.language.boolean", "match": "\\b(false|true)\\b" }, - "RE_LET": { + "RE_KEYWORD": { "name": "keyword", - "match": "\\b(let)\\b" + "match": "\\b(include|let|module|of|open|type)\\b" }, "commentLine": { "match": "//.*", @@ -109,24 +99,16 @@ } ] }, - "storage": { - "patterns": [ - { - "match": "\\btype\\b", - "name": "storage.type" - } - ] - }, "keyword": { "patterns": [ { "include": "#RE_TO_DOWNTO_AS_LABELS" }, { - "include": "#RE_KEYWORDS" + "include": "#RE_KEYWORD_CONTROL" }, { - "include": "#RE_LET" + "include": "#RE_KEYWORD" } ] }, @@ -265,10 +247,6 @@ }, "constructor": { "patterns": [ - { - "match": "\\b[A-Z][0-9a-zA-Z_]*\\b", - "name": "variable.function variable.other" - }, { "match": "(#)\\s*([a-zA-Z][0-9a-zA-Z_]*)\\b", "captures": { @@ -374,94 +352,6 @@ "jsx": { "match": "<>||", "name": "punctuation.definition.tag" - }, - "openOrIncludeModule": { - "patterns": [ - { - "match": "\\b(open|include)\\s+([A-Z_][0-9a-zA-Z_]*((\\.)([A-Z_][0-9a-zA-Z_]*))*)", - "captures": { - "1": { - "name": "keyword" - }, - "2": { - "patterns": [ - { - "include": "#moduleAccessEndsWithModule" - } - ] - } - } - }, - { - "match": "\\b(open|include)\\s+", - "name": "keyword" - } - ] - }, - "moduleAccessEndsWithModule": { - "patterns": [ - { - "match": "[A-Z_][0-9a-zA-Z_]*", - "name": "entity.name.namespace" - }, - { - "match": "(\\.)([A-Z_][0-9a-zA-Z_]*)", - "captures": { - "1": { - "name": "punctuation.accessor" - }, - "2": { - "name": "entity.name.namespace" - } - } - } - ] - }, - "moduleAccess": { - "patterns": [ - { - "match": "\\b([A-Z_][0-9a-zA-Z_]*)(\\.)", - "captures": { - "1": { - "name": "entity.name.namespace" - }, - "2": { - "name": "punctuation.accessor" - } - } - } - ] - }, - "moduleDeclaration": { - "patterns": [ - { - "match": "\\b(module)\\s+(type\\s+)?(of\\s+)?([A-Z_][0-9a-zA-Z_]*)", - "captures": { - "1": { - "name": "keyword" - }, - "2": { - "name": "keyword" - }, - "3": { - "name": "keyword" - }, - "4": { - "name": "entity.name.namespace" - } - }, - "patterns": [ - { - "match": "\\s*:\\s*([A-Z_][0-9a-zA-Z_]*)", - "captures": { - "1": { - "name": "entity.name.namespace" - } - } - } - ] - } - ] } }, "patterns": [ From 55b91665e2a9d91f631f48f129074578527dfa3c Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Tue, 22 Mar 2022 19:47:18 +0100 Subject: [PATCH 66/68] Hipster themes are more into `class` than `namespace`. --- analysis/src/SemanticTokens.ml | 10 ++-- analysis/tests/src/expected/Parser.res.txt | 66 +++++++++++----------- server/src/server.ts | 2 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index 0fa9af5a6..c792d68df 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -24,7 +24,7 @@ module Token = struct | Variable (** let x = *) | Type (** type t = *) | JsxTag (** the < and > in
*) - | Namespace (** module M = *) + | Class (** module M = *) | EnumMember (** variant A or poly variant #A *) | Property (** {x:...} *) | JsxLowercase (** div in
*) @@ -36,7 +36,7 @@ module Token = struct | Variable -> "1" | Type -> "2" | JsxTag -> "3" - | Namespace -> "4" + | Class -> "4" | EnumMember -> "5" | Property -> "6" | JsxLowercase -> "7" @@ -46,7 +46,7 @@ module Token = struct | Variable -> "Variable" | Type -> "Type" | JsxTag -> "JsxTag" - | Namespace -> "Namespace" + | Class -> "Class" | EnumMember -> "EnumMember" | Property -> "Property" | JsxLowercase -> "JsxLowercase" @@ -124,8 +124,8 @@ let emitFromLoc ~loc ~type_ emitter = let emitLongident ?(backwards = false) ?(jsx = false) ?(lowerCaseToken = if jsx then Token.JsxLowercase else Token.Variable) - ?(upperCaseToken = Token.Namespace) ?(lastToken = None) ?(posEnd = None) - ~pos ~lid ~debug emitter = + ?(upperCaseToken = Token.Class) ?(lastToken = None) ?(posEnd = None) ~pos + ~lid ~debug emitter = let rec flatten acc lid = match lid with | Longident.Lident txt -> txt :: acc diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Parser.res.txt index 5748dd607..b24212b39 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Parser.res.txt @@ -1,14 +1,14 @@ Parse tests/src/Parser.res structure items:38 diagnostics:0 -Lident: M (0,7) Namespace -Lident: C (1,9) Namespace -Lident: Component (1,13) Namespace +Lident: M (0,7) Class +Lident: C (1,9) Class +Lident: Component (1,13) Class JsxTag <: (4,9) -Lident: Component (4,10) Namespace +Lident: Component (4,10) Class Variable: _c (4,4)->(4,6) JsxTag <: (6,10) -Ldot: M (6,11) Namespace -Lident: C (6,13) Namespace +Ldot: M (6,11) Class +Lident: C (6,13) Class Variable: _mc (6,4)->(6,7) JsxTag <: (8,9) Lident: div (8,10) JsxLowercase @@ -18,18 +18,18 @@ Lident: div (11,3) JsxLowercase Lident: div (16,4) JsxLowercase JsxTag >: (11,6) JsxTag >: (16,7) -Ldot: React (12,5) Namespace +Ldot: React (12,5) Class Lident: string (12,11) Variable JsxTag <: (13,4) Lident: div (13,5) JsxLowercase Lident: div (13,34) JsxLowercase JsxTag >: (13,8) JsxTag >: (13,37) -Ldot: React (13,11) Namespace +Ldot: React (13,11) Class Lident: string (13,17) Variable -Ldot: React (14,5) Namespace +Ldot: React (14,5) Class Lident: string (14,11) Variable -Ldot: React (15,5) Namespace +Ldot: React (15,5) Class Lident: string (15,11) Variable Variable: _d2 (10,4)->(10,7) Lident: pair (18,5) Type @@ -48,19 +48,19 @@ Lident: looooooooooooooooooooooooooooooooooooooong_string (27,4) Type Binary operator < (31,12)->(31,13) Binary operator > (31,22)->(31,23) Lident: MT (33,12) Type -Lident: DDF (34,9) Namespace -Lident: DDF (39,7) Namespace +Lident: DDF (34,9) Class +Lident: DDF (39,7) Class Lident: MT (39,12) Type -Lident: DDF (40,9) Namespace -Lident: XX (45,7) Namespace -Lident: YY (46,9) Namespace +Lident: DDF (40,9) Class +Lident: XX (45,7) Class +Lident: YY (46,9) Class Lident: t (47,9) Type Lident: int (47,13) Type -Ldot: XX (51,5) Namespace -Lident: YY (51,8) Namespace +Ldot: XX (51,5) Class +Lident: YY (51,8) Class Lident: tt (53,5) Type Lident: t (53,10) Type -Lident: T (57,7) Namespace +Lident: T (57,7) Class Lident: someRecord (58,7) Type Lident: someField (59,4) Property Lident: int (59,15) Type @@ -71,7 +71,7 @@ Lident: someEnum (64,7) Type Lident: A (64,18) EnumMember Lident: B (64,22) EnumMember Lident: C (64,26) EnumMember -Ldot: T (67,17) Namespace +Ldot: T (67,17) Class Lident: someField (67,19) Property Lident: x (67,15) Variable Variable: x (67,10)->(67,11) @@ -89,16 +89,16 @@ JsxTag >: (73,24) JsxTag >: (73,39) JsxTag <: (73,26) Lident: div (73,27) JsxLowercase -Lident: SomeComponent (75,7) Namespace -Lident: Nested (76,9) Namespace +Lident: SomeComponent (75,7) Class +Lident: Nested (76,9) Class Lident: children (79,10) Variable Variable: children (78,16)->(78,25) Variable: make (78,8)->(78,12) JsxTag <: (84,8) -Ldot: SomeComponent (84,9) Namespace -Lident: Nested (84,23) Namespace -Ldot: SomeComponent (84,41) Namespace -Lident: Nested (84,55) Namespace +Ldot: SomeComponent (84,9) Class +Lident: Nested (84,23) Class +Ldot: SomeComponent (84,41) Class +Lident: Nested (84,55) Class JsxTag >: (84,29) JsxTag >: (84,61) JsxTag <: (84,31) @@ -113,29 +113,29 @@ Lident: to (94,20) Variable Lident: to (94,25) Variable Lident: to (94,9) Variable Lident: to (94,14) Variable -Lident: ToAsProp (98,7) Namespace +Lident: ToAsProp (98,7) Class Lident: to (101,18) Variable -Ldot: React (101,8) Namespace +Ldot: React (101,8) Class Lident: int (101,14) Variable Variable: to (100,14)->(100,17) Variable: make (100,6)->(100,10) JsxTag <: (104,8) -Lident: ToAsProp (104,9) Namespace +Lident: ToAsProp (104,9) Class Variable: true (107,4)->(107,11) Lident: true (108,8)->(108,15) Variable -Ldot: T (110,19) Namespace +Ldot: T (110,19) Class Lident: A (110,21) EnumMember Variable: enumInModule (110,4)->(110,16) Lident: typeInModule (112,5) Type -Ldot: XX (112,20) Namespace -Ldot: YY (112,23) Namespace +Ldot: XX (112,20) Class +Ldot: YY (112,23) Class Lident: t (112,26) Type -Lident: QQ (114,7) Namespace +Lident: QQ (114,7) Class Lident: somePolyEnumType (115,7) Type Lident: list (118,29) Type TypeArg: (118,34)->(118,37) Lident: int (118,34) Type -Ldot: QQ (126,8) Namespace +Ldot: QQ (126,8) Class Lident: somePolyEnumType (126,11) Type Lident: x (124,9) Variable Variable: x (123,8)->(123,9) diff --git a/server/src/server.ts b/server/src/server.ts index 275bf6680..4b5c39049 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -765,7 +765,7 @@ function onMessage(msg: m.Message) { "variable", "type", "jsx-tag", - "namespace", + "class", "enumMember", "property", "jsx-lowercase", From 8282458c69744d43f885abb9aa65fcfb97b44b1c Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 23 Mar 2022 08:42:15 +0100 Subject: [PATCH 67/68] rename test --- analysis/tests/src/{Parser.res => Highlight.res} | 0 analysis/tests/src/expected/Debug.res.txt | 4 ++-- .../tests/src/expected/{Parser.res.txt => Highlight.res.txt} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename analysis/tests/src/{Parser.res => Highlight.res} (100%) rename analysis/tests/src/expected/{Parser.res.txt => Highlight.res.txt} (99%) diff --git a/analysis/tests/src/Parser.res b/analysis/tests/src/Highlight.res similarity index 100% rename from analysis/tests/src/Parser.res rename to analysis/tests/src/Highlight.res diff --git a/analysis/tests/src/expected/Debug.res.txt b/analysis/tests/src/expected/Debug.res.txt index 180b6861e..2897bf280 100644 --- a/analysis/tests/src/expected/Debug.res.txt +++ b/analysis/tests/src/expected/Debug.res.txt @@ -4,7 +4,7 @@ Dependencies: @rescript/react Source directories: tests/node_modules/@rescript/react/src tests/node_modules/@rescript/react/src/legacy Source files: tests/node_modules/@rescript/react/src/React.res tests/node_modules/@rescript/react/src/ReactDOM.res tests/node_modules/@rescript/react/src/ReactDOMServer.res tests/node_modules/@rescript/react/src/ReactDOMStyle.res tests/node_modules/@rescript/react/src/ReactEvent.res tests/node_modules/@rescript/react/src/ReactEvent.resi tests/node_modules/@rescript/react/src/ReactTestUtils.res tests/node_modules/@rescript/react/src/ReactTestUtils.resi tests/node_modules/@rescript/react/src/RescriptReactErrorBoundary.res tests/node_modules/@rescript/react/src/RescriptReactErrorBoundary.resi tests/node_modules/@rescript/react/src/RescriptReactRouter.res tests/node_modules/@rescript/react/src/RescriptReactRouter.resi tests/node_modules/@rescript/react/src/legacy/ReactDOMRe.res tests/node_modules/@rescript/react/src/legacy/ReasonReact.res Source directories: tests/src tests/src/expected -Source files: tests/src/Auto.res tests/src/CompletePrioritize1.res tests/src/CompletePrioritize2.res tests/src/Completion.res tests/src/Component.res tests/src/Component.resi tests/src/Cross.res tests/src/Debug.res tests/src/Definition.res tests/src/DefinitionWithInterface.res tests/src/DefinitionWithInterface.resi tests/src/Div.res tests/src/Fragment.res tests/src/Hover.res tests/src/Jsx.res tests/src/Jsx.resi tests/src/LongIdentTest.res tests/src/Obj.res tests/src/Parser.res tests/src/Patterns.res tests/src/RecModules.res tests/src/RecordCompletion.res tests/src/References.res tests/src/ReferencesWithInterface.res tests/src/ReferencesWithInterface.resi tests/src/Rename.res tests/src/RenameWithInterface.res tests/src/RenameWithInterface.resi tests/src/TableclothMap.ml tests/src/TableclothMap.mli tests/src/TypeDefinition.res +Source files: tests/src/Auto.res tests/src/CompletePrioritize1.res tests/src/CompletePrioritize2.res tests/src/Completion.res tests/src/Component.res tests/src/Component.resi tests/src/Cross.res tests/src/Debug.res tests/src/Definition.res tests/src/DefinitionWithInterface.res tests/src/DefinitionWithInterface.resi tests/src/Div.res tests/src/Fragment.res tests/src/Highlight.res tests/src/Hover.res tests/src/Jsx.res tests/src/Jsx.resi tests/src/LongIdentTest.res tests/src/Obj.res tests/src/Patterns.res tests/src/RecModules.res tests/src/RecordCompletion.res tests/src/References.res tests/src/ReferencesWithInterface.res tests/src/ReferencesWithInterface.resi tests/src/Rename.res tests/src/RenameWithInterface.res tests/src/RenameWithInterface.resi tests/src/TableclothMap.ml tests/src/TableclothMap.mli tests/src/TypeDefinition.res Impl cmt:tests/lib/bs/src/Auto.cmt res:tests/src/Auto.res Impl cmt:tests/lib/bs/src/CompletePrioritize1.cmt res:tests/src/CompletePrioritize1.res Impl cmt:tests/lib/bs/src/CompletePrioritize2.cmt res:tests/src/CompletePrioritize2.res @@ -16,11 +16,11 @@ Impl cmt:tests/lib/bs/src/Definition.cmt res:tests/src/Definition.res IntfAndImpl cmti:tests/lib/bs/src/DefinitionWithInterface.cmti resi:tests/src/DefinitionWithInterface.resi cmt:tests/lib/bs/src/DefinitionWithInterface.cmt res:tests/src/DefinitionWithInterface.res Impl cmt:tests/lib/bs/src/Div.cmt res:tests/src/Div.res Impl cmt:tests/lib/bs/src/Fragment.cmt res:tests/src/Fragment.res +Impl cmt:tests/lib/bs/src/Highlight.cmt res:tests/src/Highlight.res Impl cmt:tests/lib/bs/src/Hover.cmt res:tests/src/Hover.res IntfAndImpl cmti:tests/lib/bs/src/Jsx.cmti resi:tests/src/Jsx.resi cmt:tests/lib/bs/src/Jsx.cmt res:tests/src/Jsx.res Impl cmt:tests/lib/bs/src/LongIdentTest.cmt res:tests/src/LongIdentTest.res Impl cmt:tests/lib/bs/src/Obj.cmt res:tests/src/Obj.res -Impl cmt:tests/lib/bs/src/Parser.cmt res:tests/src/Parser.res Impl cmt:tests/lib/bs/src/Patterns.cmt res:tests/src/Patterns.res Impl cmt:tests/lib/bs/src/RecModules.cmt res:tests/src/RecModules.res Impl cmt:tests/lib/bs/src/RecordCompletion.cmt res:tests/src/RecordCompletion.res diff --git a/analysis/tests/src/expected/Parser.res.txt b/analysis/tests/src/expected/Highlight.res.txt similarity index 99% rename from analysis/tests/src/expected/Parser.res.txt rename to analysis/tests/src/expected/Highlight.res.txt index b24212b39..d7ab1f3d6 100644 --- a/analysis/tests/src/expected/Parser.res.txt +++ b/analysis/tests/src/expected/Highlight.res.txt @@ -1,4 +1,4 @@ -Parse tests/src/Parser.res +Parse tests/src/Highlight.res structure items:38 diagnostics:0 Lident: M (0,7) Class Lident: C (1,9) Class From c5d9f45c88a081a56246efdd56f20da6c94dc7af Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Wed, 23 Mar 2022 08:46:44 +0100 Subject: [PATCH 68/68] Clean up cli. --- analysis/src/Cli.ml | 2 +- analysis/src/Commands.ml | 6 +++--- analysis/src/SemanticTokens.ml | 7 +++---- analysis/tests/src/Highlight.res | 2 +- analysis/tests/src/expected/Highlight.res.txt | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/analysis/src/Cli.ml b/analysis/src/Cli.ml index cf56d3bbc..21c128b63 100644 --- a/analysis/src/Cli.ml +++ b/analysis/src/Cli.ml @@ -71,7 +71,7 @@ let main () = | _ :: "dump" :: files -> Commands.dump files | [_; "documentSymbol"; path] -> Commands.documentSymbol ~path | [_; "semanticTokens"; currentFile] -> - SemanticTokens.testCommand ~currentFile + SemanticTokens.semanticTokens ~currentFile | [_; "hover"; path; line; col] -> Commands.hover ~path ~line:(int_of_string line) ~col:(int_of_string col) | [_; "references"; path; line; col] -> diff --git a/analysis/src/Commands.ml b/analysis/src/Commands.ml index a9945d469..64b965347 100644 --- a/analysis/src/Commands.ml +++ b/analysis/src/Commands.ml @@ -347,9 +347,9 @@ let test ~path = close_out cout; completion ~path ~line ~col ~currentFile; Sys.remove currentFile - | "par" -> - print_endline ("Parse " ^ path); - SemanticTokens.parser ~debug:true + | "hig" -> + print_endline ("Highlight " ^ path); + SemanticTokens.command ~debug:true ~emitter:(SemanticTokens.Token.createEmitter ()) ~path | _ -> ()); diff --git a/analysis/src/SemanticTokens.ml b/analysis/src/SemanticTokens.ml index c792d68df..f8293813a 100644 --- a/analysis/src/SemanticTokens.ml +++ b/analysis/src/SemanticTokens.ml @@ -208,7 +208,7 @@ let emitVariant ~(name : Longident.t Location.loc) ~debug emitter = ~pos:(Utils.tupleOfLexing name.loc.loc_start) ~lid:name.txt ~debug -let parser ~debug ~emitter ~path = +let command ~debug ~emitter ~path = let processTypeArg (coreType : Parsetree.core_type) = if debug then Printf.printf "TypeArg: %s\n" (locToString coreType.ptyp_loc) in @@ -441,8 +441,7 @@ let parser ~debug ~emitter ~path = (List.length signature) (List.length diagnostics); mapper.signature mapper signature |> ignore -let testCommand ~currentFile = +let semanticTokens ~currentFile = let emitter = Token.createEmitter () in - parser ~emitter ~debug:false ~path:currentFile; - (* emitter |> Token.add ~line:0 ~char:0 ~length:3 ~type_:Token.Keyword; *) + command ~emitter ~debug:false ~path:currentFile; Printf.printf "{\"data\":[%s]}" (Token.emit emitter) diff --git a/analysis/tests/src/Highlight.res b/analysis/tests/src/Highlight.res index ec60ff131..c3eff1792 100644 --- a/analysis/tests/src/Highlight.res +++ b/analysis/tests/src/Highlight.res @@ -53,7 +53,7 @@ open XX.YY type tt = t -// ^par +// ^hig module T = { type someRecord<'typeParameter> = { diff --git a/analysis/tests/src/expected/Highlight.res.txt b/analysis/tests/src/expected/Highlight.res.txt index d7ab1f3d6..2c28ad8b0 100644 --- a/analysis/tests/src/expected/Highlight.res.txt +++ b/analysis/tests/src/expected/Highlight.res.txt @@ -1,4 +1,4 @@ -Parse tests/src/Highlight.res +Highlight tests/src/Highlight.res structure items:38 diagnostics:0 Lident: M (0,7) Class Lident: C (1,9) Class