Change LexBuffer type to uint16

Author: ncave

src/absil/illex.fsl

@@ -16,7 +16,7 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.AsciiParser
 open Microsoft.FSharp.Compiler.AbstractIL.Internal.AsciiConstants 
 
 
-let lexeme (lexbuf : LexBuffer<char>) = new System.String(lexbuf.Lexeme)
+let lexeme (lexbuf : LexBuffer<_>) = LexBuffer<_>.LexemeString (lexbuf)
 
 let unexpectedChar _lexbuf =
   raise Parsing.RecoverableParseError ;;

src/fsharp/UnicodeLexing.fs

@@ -12,13 +12,13 @@ open System.IO
 
 open Internal.Utilities.Text.Lexing
 
-type Lexbuf =  LexBuffer<char>
+type Lexbuf = LexBuffer<LexBufferChar>
 
-let StringAsLexbuf (s:string) : Lexbuf =
-    LexBuffer<_>.FromChars (s.ToCharArray())
+let StringAsLexbuf =
+    Lexbuf.FromString
 
-let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf =
-    LexBuffer<_>.FromFunction bufferFiller 
+let FunctionAsLexbuf =
+    Lexbuf.FromFunction
 
 #if !FABLE_COMPILER
 
@@ -67,7 +67,7 @@ let UnicodeFileAsLexbuf (filename,codePage : int option, retryLocked:bool) :  Le
                else 
                    reraise()
     let source = getSource 0
-    let lexbuf = LexBuffer<_>.FromChars(source.ToCharArray())  
+    let lexbuf = LexBuffer<_>.FromString (source)
     lexbuf
 
 #endif

src/fsharp/UnicodeLexing.fsi

@@ -5,9 +5,9 @@ module internal Microsoft.FSharp.Compiler.UnicodeLexing
 open Microsoft.FSharp.Text
 open Internal.Utilities.Text.Lexing
 
-type Lexbuf =  LexBuffer<char>
+type Lexbuf =  LexBuffer<LexBufferChar>
 val internal StringAsLexbuf : string -> Lexbuf
-val public FunctionAsLexbuf : (char [] * int * int -> int) -> Lexbuf
+val public FunctionAsLexbuf : (LexBufferChar[] * int * int -> int) -> Lexbuf
 
 #if !FABLE_COMPILER
 val public UnicodeFileAsLexbuf :string * int option * (*retryLocked*) bool -> Lexbuf

src/fsharp/fsi/fsi.fs

@@ -1735,7 +1735,7 @@ type internal FsiStdinLexerProvider
 
     let LexbufFromLineReader (fsiStdinSyphon: FsiStdinSyphon) readf = 
         UnicodeLexing.FunctionAsLexbuf 
-          (fun (buf: char[], start, len) -> 
+          (fun (buf, start, len) -> 
             //fprintf fsiConsoleOutput.Out "Calling ReadLine\n"
             let inputOption = try Some(readf()) with :? EndOfStreamException -> None
             inputOption |> Option.iter (fun t -> fsiStdinSyphon.Add (t + "\n"))
@@ -1749,7 +1749,11 @@ type internal FsiStdinLexerProvider
                 if ninput > len then fprintf fsiConsoleOutput.Error  "%s" (FSIstrings.SR.fsiLineTooLong())
                 let ntrimmed = min len ninput 
                 for i = 0 to ntrimmed-1 do
+#if FABLE_COMPILER
+                    buf.[i+start] <- uint16 input.[i]
+#else
                     buf.[i+start] <- input.[i]
+#endif
                 ntrimmed
         )
 

src/fsharp/lex.fsl

@@ -161,7 +161,7 @@ let shouldStartFile args lexbuf (m:range) err tok =
     else tok
 
 let evalIfDefExpression startPos args (lookup:string->bool) (lexed:string) =
-    let lexbuf          = LexBuffer<char>.FromChars (lexed.ToCharArray ())
+    let lexbuf          = LexBuffer<_>.FromString (lexed)
     lexbuf.StartPos     <- startPos
     lexbuf.EndPos       <- startPos
     let tokenStream     = Microsoft.FSharp.Compiler.PPLexer.tokenstream args

src/fsharp/service/ServiceLexing.fsi

@@ -3,6 +3,7 @@
 namespace Microsoft.FSharp.Compiler.SourceCodeServices
 
 open Microsoft.FSharp.Compiler 
+open Internal.Utilities.Text.Lexing
 
 type Position = int * int
 type Range = Position * Position
@@ -227,7 +228,7 @@ type FSharpLineTokenizer =
 type FSharpSourceTokenizer =
     new : conditionalDefines:string list * fileName:string option -> FSharpSourceTokenizer
     member CreateLineTokenizer : lineText:string -> FSharpLineTokenizer
-    member CreateBufferTokenizer : bufferFiller:(char[] * int * int -> int) -> FSharpLineTokenizer
+    member CreateBufferTokenizer : bufferFiller:(LexBufferChar[] * int * int -> int) -> FSharpLineTokenizer
 
 
 module internal TestExpose =     

src/utils/prim-lexing.fs

@@ -121,6 +121,7 @@ namespace Internal.Utilities.Text.Lexing
            and  set b =  endPos <- b
 
         member lexbuf.Lexeme         = Array.sub buffer bufferScanStart lexemeLength
+        member lexbuf.LexemeChar(n)  = buffer.[n+bufferScanStart]
 
         member lexbuf.BufferLocalStore = (context :> IDictionary<_,_>)
         member lexbuf.LexemeLength        with get() : int = lexemeLength    and set v = lexemeLength <- v
@@ -130,8 +131,14 @@ namespace Internal.Utilities.Text.Lexing
         member lexbuf.BufferScanStart     with get() : int = bufferScanStart     and set v = bufferScanStart <- v
         member lexbuf.BufferAcceptAction  with get() = bufferAcceptAction  and set v = bufferAcceptAction <- v
         member lexbuf.RefillBuffer () = filler lexbuf
-        static member LexemeString(lexbuf:LexBuffer<char>) = 
-            new System.String(lexbuf.Buffer,lexbuf.BufferScanStart,lexbuf.LexemeLength)
+
+        static member LexemeString (lexbuf: LexBuffer<LexBufferChar>) =
+#if FABLE_COMPILER
+            let chars = Array.init lexbuf.LexemeLength (lexbuf.LexemeChar >> char)
+            new System.String(chars)
+#else
+            new System.String(lexbuf.Buffer, lexbuf.BufferScanStart, lexbuf.LexemeLength)
+#endif
 
         member lexbuf.IsPastEndOfStream 
            with get() = eof
@@ -169,18 +176,30 @@ namespace Internal.Utilities.Text.Lexing
             let buffer = Array.copy s 
             LexBuffer<'Char>.FromArrayNoCopy buffer
 
-        // Important: This method takes ownership of the array
-        static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy arr 
+        static member FromString (s: string) =
+#if FABLE_COMPILER
+            let arr = Array.init s.Length (fun i -> uint16 s.[i])
+            LexBuffer.FromArrayNoCopy arr
+#else
+            LexBuffer.FromArrayNoCopy (s.ToCharArray())
+#endif
+
+#if FABLE_COMPILER
+    and internal LexBufferChar = uint16
+#else
+    and internal LexBufferChar = char
+#endif
+
 
-    module internal GenericImplFragments = 
-        let startInterpret(lexBuffer:LexBuffer<char>)= 
+    module GenericImplFragments = 
+        let startInterpret(lexBuffer:LexBuffer<LexBufferChar>) = 
             lexBuffer.BufferScanStart <- lexBuffer.BufferScanStart + lexBuffer.LexemeLength;
             lexBuffer.BufferMaxScanLength <- lexBuffer.BufferMaxScanLength - lexBuffer.LexemeLength;
             lexBuffer.BufferScanLength <- 0;
             lexBuffer.LexemeLength <- 0;
             lexBuffer.BufferAcceptAction <- -1;
 
-        let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer<char>,scanUntilSentinel,endOfScan,state,eofPos) = 
+        let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer<LexBufferChar>,scanUntilSentinel,endOfScan,state,eofPos) = 
             // end of file occurs if we couldn't extend the buffer 
             if lexBuffer.BufferScanLength = lexBuffer.BufferMaxScanLength then  
                 let snew = int trans.[state].[eofPos] // == EOF 
@@ -194,9 +213,9 @@ namespace Internal.Utilities.Text.Lexing
             else 
                 scanUntilSentinel lexBuffer state
 
-        let onAccept (lexBuffer:LexBuffer<char>,a) = 
-            lexBuffer.LexemeLength <- lexBuffer.BufferScanLength;
-            lexBuffer.BufferAcceptAction <- a;
+        let onAccept (lexBuffer:LexBuffer<LexBufferChar>, a) = 
+            lexBuffer.LexemeLength <- lexBuffer.BufferScanLength
+            lexBuffer.BufferAcceptAction <- a
 
     open GenericImplFragments
 
@@ -223,15 +242,15 @@ namespace Internal.Utilities.Text.Lexing
                         let baseForUnicodeCategories = numLowUnicodeChars+numSpecificUnicodeChars*2
                         let unicodeCategory = 
 #if FX_RESHAPED_GLOBALIZATION
-                            System.Globalization.CharUnicodeInfo.GetUnicodeCategory(inp)
+                            System.Globalization.CharUnicodeInfo.GetUnicodeCategory(char inp)
 #else
-                            System.Char.GetUnicodeCategory(inp)
+                            System.Char.GetUnicodeCategory(char inp)
 #endif
                         //System.Console.WriteLine("inp = {0}, unicodeCategory = {1}", [| box inp; box unicodeCategory |]);
                         int trans.[state].[baseForUnicodeCategories + int32 unicodeCategory]
                     else 
                         // This is the specific unicode character
-                        let c = char (int trans.[state].[baseForSpecificUnicodeChars+i*2])
+                        let c = trans.[state].[baseForSpecificUnicodeChars+i*2]
                         //System.Console.WriteLine("c = {0}, inp = {1}, i = {2}", [| box c; box inp; box i |]);
                         // OK, have we found the entry for a specific unicode character?
                         if c = inp
@@ -254,7 +273,7 @@ namespace Internal.Utilities.Text.Lexing
                 afterRefill (trans,sentinel,lexBuffer,scanUntilSentinel,lexBuffer.EndOfScan,state,eofPos)
             else
                 // read a character - end the scan if there are no further transitions 
-                let inp = lexBuffer.Buffer.[lexBuffer.BufferScanPos]
+                let inp = uint16 lexBuffer.Buffer.[lexBuffer.BufferScanPos]
 
                 // Find the new state
                 let snew = lookupUnicodeCharacters state inp
@@ -272,7 +291,7 @@ namespace Internal.Utilities.Text.Lexing
         //      30 entries, one for each UnicodeCategory
         //      1 entry for EOF
 
-        member tables.Interpret(initialState,lexBuffer : LexBuffer<char>) = 
+        member tables.Interpret(initialState, lexBuffer: LexBuffer<LexBufferChar>) = 
             startInterpret(lexBuffer)
             scanUntilSentinel lexBuffer initialState
 

src/utils/prim-lexing.fsi

@@ -56,24 +56,30 @@ type internal LexBuffer<'Char> =
     member Lexeme: 'Char []
 
     /// Fast helper to turn the matched characters into a string, avoiding an intermediate array.
-    static member LexemeString : LexBuffer<char> -> string
+    static member LexemeString : LexBuffer<LexBufferChar> -> string
 
     /// Dynamically typed, non-lexically scoped parameter table.
     member BufferLocalStore : IDictionary<string,obj>
 
     /// True if the refill of the buffer ever failed , or if explicitly set to True.
     member IsPastEndOfStream: bool with get,set
 
-    /// Create a lex buffer suitable for Unicode lexing that reads characters from the given array.
-    /// Important: does take ownership of the array.
-    static member FromChars: char[] -> LexBuffer<char>
+    /// Create a lex buffer suitable for Unicode lexing that reads characters from the given string.
+    static member FromString: string -> LexBuffer<LexBufferChar>
+
     /// Create a lex buffer that reads character or byte inputs by using the given function.
     static member FromFunction: ('Char[] * int * int -> int) -> LexBuffer<'Char>
 
+#if FABLE_COMPILER
+and internal LexBufferChar = uint16
+#else
+and internal LexBufferChar = char
+#endif
+
 /// The type of tables for an unicode lexer generated by <c>fslex.exe</c>. 
 [<Sealed>]
 type internal UnicodeTables =
     static member Create : uint16[][] * uint16[] -> UnicodeTables
     /// Interpret tables for a unicode lexer generated by <c>fslex.exe</c>. 
-    member Interpret:  initialState:int * LexBuffer<char> -> int
+    member Interpret:  initialState:int * LexBuffer<LexBufferChar> -> int
 

src/utils/prim-parsing.fs

@@ -11,7 +11,7 @@ exception RecoverableParseError
 exception Accept of obj
 
 [<Sealed>]
-type internal IParseState(ruleStartPoss:Position[],ruleEndPoss:Position[],lhsPos:Position[],ruleValues:obj[],lexbuf:LexBuffer<char>) = 
+type internal IParseState(ruleStartPoss:Position[],ruleEndPoss:Position[],lhsPos:Position[],ruleValues:obj[],lexbuf:LexBuffer<LexBufferChar>) = 
     member p.LexBuffer = lexbuf
     member p.InputRange n = ruleStartPoss.[n-1], ruleEndPoss.[n-1]; 
     member p.InputStartPosition n = ruleStartPoss.[n-1]

src/utils/prim-parsing.fsi

@@ -36,7 +36,7 @@ type internal IParseState =
     /// Raise an error in this parse context.
     member RaiseError<'b> : unit -> 'b 
     /// Return the LexBuffer for this parser instance.
-    member LexBuffer : LexBuffer<char>
+    member LexBuffer : LexBuffer<LexBufferChar>
 
 
 [<Sealed>]
@@ -96,7 +96,7 @@ type internal Tables<'tok> =
 
     /// Interpret the parser table taking input from the given lexer, using the given lex buffer, and the given start state.
     /// Returns an object indicating the final synthesized value for the parse.
-    member Interpret :  lexer:(LexBuffer<char> -> 'tok) * lexbuf:LexBuffer<char> * startState:int -> obj 
+    member Interpret :  lexer:(LexBuffer<LexBufferChar> -> 'tok) * lexbuf:LexBuffer<LexBufferChar> * startState:int -> obj 
 
 /// Indicates an accept action has occurred.
 exception internal Accept of obj