Skip to content

Update test framework - improve baseline comparison and failure diagnosis #16669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 9, 2024
Merged

Update test framework - improve baseline comparison and failure diagnosis #16669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30339f4
Update framework
KevinRansom Feb 8, 2024
a876992
feedback
KevinRansom Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 7 additions & 14 deletions tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/MethodResolution.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,26 +76,23 @@ extends [runtime]System.Object

}

.method assembly specialname static class [runtime]System.Tuple`2<bool,int32>
get_patternInput@9() cil managed
.method assembly specialname static class [runtime]System.Tuple`2<bool,int32> get_patternInput@9() cil managed
{

.maxstack 8
IL_0000: ldsfld class [runtime]System.Tuple`2<bool,int32> '<StartupCode$assembly>'.$OutOptionalTests::patternInput@9
IL_0005: ret
}

.method assembly specialname static int32
get_outArg@9() cil managed
.method assembly specialname static int32 get_outArg@9() cil managed
{

.maxstack 8
IL_0000: ldsfld int32 '<StartupCode$assembly>'.$OutOptionalTests::outArg@9
IL_0005: ret
}

.method assembly specialname static void
set_outArg@9(int32 'value') cil managed
.method assembly specialname static void set_outArg@9(int32 'value') cil managed
{

.maxstack 8
Expand All @@ -104,26 +101,23 @@ extends [runtime]System.Object
IL_0006: ret
}

.method assembly specialname static class [runtime]System.Tuple`2<bool,int32>
'get_patternInput@10-1'() cil managed
.method assembly specialname static class [runtime]System.Tuple`2<bool,int32> 'get_patternInput@10-1'() cil managed
{

.maxstack 8
IL_0000: ldsfld class [runtime]System.Tuple`2<bool,int32> '<StartupCode$assembly>'.$OutOptionalTests::'patternInput@10-1'
IL_0005: ret
}

.method assembly specialname static int32
'get_outArg@10-1'() cil managed
.method assembly specialname static int32 'get_outArg@10-1'() cil managed
{

.maxstack 8
IL_0000: ldsfld int32 '<StartupCode$assembly>'.$OutOptionalTests::'outArg@10-1'
IL_0005: ret
}

.method assembly specialname static void
'set_outArg@10-1'(int32 'value') cil managed
.method assembly specialname static void 'set_outArg@10-1'(int32 'value') cil managed
{

.maxstack 8
Expand Down Expand Up @@ -174,8 +168,7 @@ extends [runtime]System.Object
.custom instance void [runtime]System.Diagnostics.DebuggerBrowsableAttribute::.ctor(valuetype [runtime]System.Diagnostics.DebuggerBrowsableState) = ( 01 00 00 00 00 00 00 00 )
.custom instance void [runtime]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.custom instance void [runtime]System.Diagnostics.DebuggerNonUserCodeAttribute::.ctor() = ( 01 00 00 00 )
.method private specialname rtspecialname static
void .cctor() cil managed
.method private specialname rtspecialname static void .cctor() cil managed
{

.maxstack 4
Expand Down
3 changes: 1 addition & 2 deletions ...er.ComponentTests/Conformance/Types/TypeConstraints/IWSAMsAndSRTPs/IWSAMsAndSRTPsTests.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,8 +723,7 @@ let main _ =
|> shouldSucceed
|> verifyIL [
"""
.method public specialname static class [FSharp.Core]Microsoft.FSharp.Core.FSharpOption`1<class [FSharp.Core]Microsoft.FSharp.Core.Unit>
'|GoodPotato|_|'<(class [Potato]Potato.Lib/IPotato`1<!!T>) T>(!!T x) cil managed
.method public specialname static class [FSharp.Core]Microsoft.FSharp.Core.FSharpOption`1<class [FSharp.Core]Microsoft.FSharp.Core.Unit> '|GoodPotato|_|'<(class [Potato]Potato.Lib/IPotato`1<!!T>) T>(!!T x) cil managed
{

.maxstack 8
Expand Down
17 changes: 6 additions & 11 deletions tests/FSharp.Compiler.ComponentTests/EmittedIL/ByRefTests.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,7 @@ type C() =
extends [System.Runtime]System.Attribute
{
.custom instance void [netstandard]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.method public specialname rtspecialname
instance void .ctor() cil managed
.method public specialname rtspecialname instance void .ctor() cil managed
{
.custom instance void [netstandard]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.custom instance void [netstandard]System.Diagnostics.DebuggerNonUserCodeAttribute::.ctor() = ( 01 00 00 00 )
Expand Down Expand Up @@ -366,8 +365,7 @@ type C() =
.get instance int32& modreq([netstandard]System.Runtime.InteropServices.InAttribute) System.Runtime.CompilerServices.C::get_X()
}"""

let verifyMethod = """.method public hidebysig specialname instance int32& modreq([netstandard]System.Runtime.InteropServices.InAttribute)
get_X() cil managed
let verifyMethod = """.method public hidebysig specialname instance int32& modreq([netstandard]System.Runtime.InteropServices.InAttribute) get_X() cil managed
{
.param [0]
.custom instance void System.Runtime.CompilerServices.IsReadOnlyAttribute::.ctor() = ( 01 00 00 00 )"""
Expand All @@ -377,8 +375,7 @@ type C() =
extends [netstandard]System.Attribute
{
.custom instance void [FSharp.Core]Microsoft.FSharp.Core.CompilationMappingAttribute::.ctor(valuetype [FSharp.Core]Microsoft.FSharp.Core.SourceConstructFlags) = ( 01 00 03 00 00 00 00 00 )
.method public specialname rtspecialname
instance void .ctor() cil managed
.method public specialname rtspecialname instance void .ctor() cil managed
{

.maxstack 8
Expand Down Expand Up @@ -408,8 +405,7 @@ type C<'T>() =
member _.X<'U>(): inref<'T> = &x
"""

let verifyMethod = """.method public hidebysig instance !T& modreq([runtime]System.Runtime.InteropServices.InAttribute)
X<U>() cil managed
let verifyMethod = """.method public hidebysig instance !T& modreq([runtime]System.Runtime.InteropServices.InAttribute) X<U>() cil managed
{
.param [0]
.custom instance void [runtime]System.Runtime.CompilerServices.IsReadOnlyAttribute::.ctor() = ( 01 00 00 00 )"""
Expand All @@ -435,8 +431,7 @@ module Test =
member _.X<'U>(): inref<'T> = &x
"""

let verifyMethod = """.method public hidebysig instance !T& modreq([netstandard]System.Runtime.InteropServices.InAttribute)
X<U>() cil managed
let verifyMethod = """.method public hidebysig instance !T& modreq([netstandard]System.Runtime.InteropServices.InAttribute) X<U>() cil managed
{
.param [0]
.custom instance void System.Runtime.CompilerServices.IsReadOnlyAttribute::.ctor() = ( 01 00 00 00 )"""
Expand All @@ -463,7 +458,7 @@ type C<'T>() =
X<U>() cil managed
{
.param [0]
.custom instance void [runtime]System.Runtime.CompilerServices.IsReadOnlyAttribute::.ctor() = ( 01 00 00 00 ) """
.custom instance void [runtime]System.Runtime.CompilerServices.IsReadOnlyAttribute::.ctor() = ( 01 00 00 00 )"""

FSharp src
|> compile
Expand Down
14 changes: 5 additions & 9 deletions tests/FSharp.Compiler.ComponentTests/EmittedIL/CompilerGeneratedAttributeOnAccessors.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ module ``Auto-generated accessors have CompilerGenerated attribute`` =
|> compile
|> verifyIL [
"""
.method public hidebysig specialname
instance int32 get_Age() cil managed
.method public hidebysig specialname instance int32 get_Age() cil managed
{
.custom instance void [runtime]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.custom instance void [runtime]System.Diagnostics.DebuggerNonUserCodeAttribute::.ctor() = ( 01 00 00 00 )
Expand All @@ -53,8 +52,7 @@ module ``Auto-generated accessors have CompilerGenerated attribute`` =
IL_0006: ret
}

.method public hidebysig specialname
instance void set_Age(int32 v) cil managed
.method public hidebysig specialname instance void set_Age(int32 v) cil managed
{
.custom instance void [runtime]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.custom instance void [runtime]System.Diagnostics.DebuggerNonUserCodeAttribute::.ctor() = ( 01 00 00 00 )
Expand Down Expand Up @@ -84,8 +82,7 @@ module ``Auto-generated accessors have CompilerGenerated attribute`` =
|> compile
|> verifyIL [
"""
.method public specialname static int32
get_Age() cil managed
.method public specialname static int32 get_Age() cil managed
{
.custom instance void [runtime]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.custom instance void [runtime]System.Diagnostics.DebuggerNonUserCodeAttribute::.ctor() = ( 01 00 00 00 )
Expand All @@ -103,8 +100,7 @@ module ``Auto-generated accessors have CompilerGenerated attribute`` =
IL_0016: ret
}

.method public specialname static void
set_Age(int32 v) cil managed
.method public specialname static void set_Age(int32 v) cil managed
{
.custom instance void [runtime]System.Runtime.CompilerServices.CompilerGeneratedAttribute::.ctor() = ( 01 00 00 00 )
.custom instance void [runtime]System.Diagnostics.DebuggerNonUserCodeAttribute::.ctor() = ( 01 00 00 00 )
Expand Down Expand Up @@ -202,5 +198,5 @@ module ``Let bindings in classes shoulnd't have DebuggerNonUserCodeAttribute`` =
"""
|> compileAssembly
|> getType "Test+User"
|> getPrivateMethod "moo"
|> getPrivateInstanceMethod "moo"
|> shouldn't haveAttribute "DebuggerNonUserCodeAttribute"
15 changes: 5 additions & 10 deletions tests/FSharp.Compiler.ComponentTests/Interop/StaticsInInterfaces.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,7 @@ extends [runtime]System.Object
implements class [csLib]StaticsInInterfaces.IGetNext`1<class StaticsTesting/MyRepeatSequence>
{
.custom instance void [FSharp.Core]Microsoft.FSharp.Core.CompilationMappingAttribute::.ctor(valuetype [FSharp.Core]Microsoft.FSharp.Core.SourceConstructFlags) = ( 01 00 03 00 00 00 00 00 )
.method public specialname rtspecialname
instance void .ctor() cil managed
.method public specialname rtspecialname instance void .ctor() cil managed
{

.maxstack 8
Expand All @@ -205,8 +204,7 @@ instance void .ctor() cil managed
IL_0008: ret
}

.method public hidebysig static class StaticsTesting/MyRepeatSequence
'StaticsInInterfaces.IGetNext<StaticsTesting.MyRepeatSequence>.Next'(class StaticsTesting/MyRepeatSequence other) cil managed
.method public hidebysig static class StaticsTesting/MyRepeatSequence 'StaticsInInterfaces.IGetNext<StaticsTesting.MyRepeatSequence>.Next'(class StaticsTesting/MyRepeatSequence other) cil managed
{
.override method !0 class [csLib]StaticsInInterfaces.IGetNext`1<class StaticsTesting/MyRepeatSequence>::Next(!0)

Expand All @@ -222,8 +220,7 @@ extends [runtime]System.Object
implements class [csLib]StaticsInInterfaces.IGetNext`1<class StaticsTesting/MyRepeatSequence2>
{
.custom instance void [FSharp.Core]Microsoft.FSharp.Core.CompilationMappingAttribute::.ctor(valuetype [FSharp.Core]Microsoft.FSharp.Core.SourceConstructFlags) = ( 01 00 03 00 00 00 00 00 )
.method public specialname rtspecialname
instance void .ctor() cil managed
.method public specialname rtspecialname instance void .ctor() cil managed
{

.maxstack 8
Expand All @@ -234,17 +231,15 @@ instance void .ctor() cil managed
IL_0008: ret
}

.method public static class StaticsTesting/MyRepeatSequence2
Next(class StaticsTesting/MyRepeatSequence2 other) cil managed
.method public static class StaticsTesting/MyRepeatSequence2 Next(class StaticsTesting/MyRepeatSequence2 other) cil managed
{

.maxstack 8
IL_0000: ldarg.0
IL_0001: ret
}

.method public hidebysig static class StaticsTesting/MyRepeatSequence2
'StaticsInInterfaces.IGetNext<StaticsTesting.MyRepeatSequence2>.Next'(class StaticsTesting/MyRepeatSequence2 other) cil managed
.method public hidebysig static class StaticsTesting/MyRepeatSequence2 'StaticsInInterfaces.IGetNext<StaticsTesting.MyRepeatSequence2>.Next'(class StaticsTesting/MyRepeatSequence2 other) cil managed
{
.override method !0 class [csLib]StaticsInInterfaces.IGetNext`1<class StaticsTesting/MyRepeatSequence2>::Next(!0)

Expand Down
48 changes: 33 additions & 15 deletions tests/FSharp.Test.Utilities/Compiler.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ module rec Compiler =
| Arm = 5
| Arm64 = 6

let private defaultOptions : string list = []
let public defaultOptions : string list = []

let normalizePathSeparator (text:string) = text.Replace(@"\", "/")

Expand Down Expand Up @@ -1112,7 +1112,9 @@ Actual:

let private createBaselineErrors (baselineFile: BaselineFile) (actualErrors: string) : unit =
printfn $"creating baseline error file for convenience: {baselineFile.FilePath}, expected: {baselineFile.BslSource}"
FileSystem.OpenFileForWriteShim(baselineFile.FilePath).Write(actualErrors)
let file = FileSystem.OpenFileForWriteShim(baselineFile.FilePath)
file.SetLength(0)
file.WriteAllText(actualErrors)

let private verifyFSBaseline fs : unit =
match fs.Baseline with
Expand Down Expand Up @@ -1156,6 +1158,18 @@ Actual:
| Some p -> func p il
| CompilationResult.Failure f -> failwith $"Result should be \"Success\" in order to get IL. Failure: {Environment.NewLine}{f}"

let withILContains expected result : CompilationResult =
match result with
| CompilationResult.Success s ->
match s.OutputPath with
| None -> failwith "Operation didn't produce any output!"
| Some p ->
match ILChecker.verifyILAndReturnActual [] p expected with
| true, _, _ -> result
| false, errorMsg, _actualIL -> CompilationResult.Failure( {s with Output = Some (ExecutionOutput { StdOut = errorMsg; ExitCode = 0; StdErr = "" })} )

| CompilationResult.Failure f -> failwith $"Result should be \"Success\" in order to get IL. Failure: {Environment.NewLine}{f}"

let verifyIL = doILCheck ILChecker.checkIL

let verifyILNotPresent = doILCheck ILChecker.checkILNotPresent
Expand All @@ -1170,7 +1184,7 @@ Actual:
match baseline.ILBaseline.Content with
| Some b -> b
| None -> String.Empty
let success, errorMsg, actualIL = ILChecker.verifyILAndReturnActual p expectedIL
let success, errorMsg, actualIL = ILChecker.verifyILAndReturnActual [] p [expectedIL]

if not success then
// Failed try update baselines if required
Expand Down Expand Up @@ -1348,7 +1362,6 @@ Actual:
if documents <> expectedDocuments then
failwith $"Expected documents are different from PDB.\nExpected: %A{expectedDocuments}\nActual: %A{documents}"


let private verifyPdbOptions reader options =
for option in options do
match option with
Expand Down Expand Up @@ -1546,8 +1559,6 @@ Actual:
let withResult (expectedResult: SimpleErrorInfo ) (result: CompilationResult) : CompilationResult =
withResults [expectedResult] result



module TextBasedDiagnosticAsserts =
open FSharp.Compiler.Text.Range

Expand Down Expand Up @@ -1674,26 +1685,35 @@ Actual:
| _ -> failwith "Cannot check exit code on this run result."
result

let private checkOutput (category: string) (substring: string) (selector: ExecutionOutput -> string) (result: CompilationResult) : CompilationResult =
let private checkOutputInOrder (category: string) (substrings: string list) (selector: ExecutionOutput -> string) (result: CompilationResult) : CompilationResult =
match result.RunOutput with
| None -> failwith (sprintf "Execution output is missing cannot check \"%A\"" category)
| Some o ->
match o with
| ExecutionOutput e ->
let where = selector e
if not (where.Contains(substring)) then
failwith (sprintf "\nThe following substring:\n %A\nwas not found in the %A\nOutput:\n %A" substring category where)
let mutable searchPos = 0
for substring in substrings do
match where.IndexOf(substring, searchPos) with
| -1 -> failwith (sprintf "\nThe following substring:\n %A\nwas not found in the %A\nOutput:\n %A" substring category where)
| pos -> searchPos <- pos + substring.Length
| _ -> failwith "Cannot check output on this run result."
result

let withOutputContains (substring: string) (result: CompilationResult) : CompilationResult =
checkOutput "STDERR/STDOUT" substring (fun o -> o.StdOut + "\n" + o.StdErr) result
let withOutputContainsAllInOrder (substrings: string list) (result: CompilationResult) : CompilationResult =
checkOutputInOrder "STDERR/STDOUT" substrings (fun o -> o.StdOut + "\n" + o.StdErr) result

let withStdOutContains (substring: string) (result: CompilationResult) : CompilationResult =
checkOutput "STDOUT" substring (fun o -> o.StdOut) result
checkOutputInOrder "STDOUT" [substring] (fun o -> o.StdOut) result

let withStdOutContainsAllInOrder (substrings: string list) (result: CompilationResult) : CompilationResult =
checkOutputInOrder "STDOUT" substrings (fun o -> o.StdOut) result

let withStdErrContainsAllInOrder (substrings: string list) (result: CompilationResult) : CompilationResult =
checkOutputInOrder "STDERR" substrings (fun o -> o.StdErr) result

let withStdErrContains (substring: string) (result: CompilationResult) : CompilationResult =
checkOutput "STDERR" substring (fun o -> o.StdErr) result
checkOutputInOrder "STDERR" [substring] (fun o -> o.StdErr) result

let private assertEvalOutput (selector: FsiValue -> 'T) (value: 'T) (result: CompilationResult) : CompilationResult =
match result.RunOutput with
Expand Down Expand Up @@ -1752,5 +1772,3 @@ Actual:
match hash with
| Some h -> h
| None -> failwith "Implied signature hash returned 'None' which should not happen"


Loading