Add FSharpTestCheckerService to support ExpectErrorsAttribute in tests

ReSharper.FSharp/src/FSharp.Common/src/Checker/FSharpCheckerService.fs

@@ -58,7 +58,8 @@ type FSharpCheckerService
     member val OptionsProvider = Unchecked.defaultof<IFSharpProjectOptionsProvider> with get, set
     member x.Checker = checker.Value
 
-    member x.ParseFile(path: FileSystemPath, document: IDocument, parsingOptions: FSharpParsingOptions) =
+    abstract ParseFile : path: FileSystemPath * document: IDocument * parsingOptions: FSharpParsingOptions -> FSharpParseFileResults option
+    default x.ParseFile(path: FileSystemPath, document: IDocument, parsingOptions: FSharpParsingOptions) =
         let source = SourceText.ofString (document.GetText())
         try
             let parseResults = x.Checker.ParseFile(path.FullPath, source, parsingOptions).RunAsTask() 
@@ -74,8 +75,10 @@ type FSharpCheckerService
         let parsingOptions = x.OptionsProvider.GetParsingOptions(sourceFile)
         x.ParseFile(sourceFile.GetLocation(), sourceFile.Document, parsingOptions)
 
-    member x.ParseAndCheckFile([<NotNull>] file: IPsiSourceFile, opName,
-                               [<Optional; DefaultParameterValue(false)>] allowStaleResults) =
+    abstract ParseAndCheckFile : [<NotNull>] file: IPsiSourceFile * opName: string * [<Optional; DefaultParameterValue(false)>] allowStaleResults: bool -> FSharpParseAndCheckResults option
+
+    default x.ParseAndCheckFile([<NotNull>] file: IPsiSourceFile, opName,
+                                [<Optional; DefaultParameterValue(false)>] allowStaleResults) =
         match x.OptionsProvider.GetProjectOptions(file) with
         | None -> None
         | Some options ->

ReSharper.FSharp/test/data/features/daemon/inferredTypeCodeVision/Module functions and values.fs

@@ -4,6 +4,6 @@ let x = 1
 let f x = x
 let g = fun x -> x + 1
 let h f g x y = f x + g y
-let f (x, y) z (a, b, c) = x + y + a + string b + c
-let f (x, y) = x + y
-let f (x, y) z = x + y + z
+let i (x, y) z (a, b, c) = x + y + a + string b + c
+let j (x, y) = x + y
+let k (x, y) z = x + y + z

ReSharper.FSharp/test/data/features/daemon/inferredTypeCodeVision/Module functions and values.fs.gold

@@ -4,9 +4,9 @@ let |x|(0) = 1
 let |f|(1) x = x
 let |g|(2) = fun x -> x + 1
 let |h|(3) f g x y = f x + g y
-let |f|(4) (x, y) z (a, b, c) = x + y + a + string b + c
-let |f|(5) (x, y) = x + y
-let |f|(6) (x, y) z = x + y + z
+let |i|(4) (x, y) z (a, b, c) = x + y + a + string b + c
+let |j|(5) (x, y) = x + y
+let |k|(6) (x, y) z = x + y + z
 
 ---------------------------------------------------------
 (0): CodeInsights: int

ReSharper.FSharp/test/data/features/findUsages/occurenceKinds/Pattern - Record 02 - Nested.fs

@@ -1 +1,3 @@
-let { Foo = { Bar.Baz = 1 } } = ()
+type Bar = { Baz : int }
+type Record = { Foo : Bar }
+let { Foo = { Bar.Baz = 1 } } = failwith ""

ReSharper.FSharp/test/data/features/findUsages/occurenceKinds/Pattern - Record 02 - Nested.gold

@@ -1,11 +1,23 @@
-Pattern
+Field declaration
 ---
-TO: [O] let { |Foo| = { Bar.Baz = 1 } } = () RANGE: (6,9) @ Pattern - Record 02 - Nested.fs
-TO: [O] let { Foo = { Bar.|Baz| = 1 } } = () RANGE: (18,21) @ Pattern - Record 02 - Nested.fs
+TO: [O] type Bar = { Baz : |int| } RANGE: (19,22) @ Pattern - Record 02 - Nested.fs
+TO: [O] type Record = { Foo : |Bar| } RANGE: (48,51) @ Pattern - Record 02 - Nested.fs
+
+
+Pattern, Read access
+---
+TO: [R] let { |Foo| = { Bar.Baz = 1 } } = failwith "" RANGE: (61,64) @ Pattern - Record 02 - Nested.fs
+TO: [R] let { Foo = { Bar.|Baz| = 1 } } = failwith "" RANGE: (73,76) @ Pattern - Record 02 - Nested.fs
 
 
 Other
 ---
-TO: [O] let { Foo = { |Bar|.Baz = 1 } } = () RANGE: (14,17) @ Pattern - Record 02 - Nested.fs
+TO: [O] let { Foo = { |Bar|.Baz = 1 } } = failwith "" RANGE: (69,72) @ Pattern - Record 02 - Nested.fs
+TO: [O] let { Foo = { Bar.Baz = 1 } } = |failwith| "" RANGE: (87,95) @ Pattern - Record 02 - Nested.fs
+
+
+New instance creation
+---
+TO: [O] let { Foo = { Bar.Baz = 1 } } = |failwith| "" RANGE: (87,95) @ Pattern - Record 02 - Nested.fs
 
 

ReSharper.FSharp/test/data/features/naming/AppExpr 03 - Partial.fs

@@ -1,4 +1,4 @@
 module Module
 
-let getName _ _ = ""
+let getName _ (_: int) = ""
 let x{caret} = getName ()

ReSharper.FSharp/test/data/features/naming/AppExpr 03 - Partial.fs.gold

@@ -1,3 +1,3 @@
 name
-objectFunc
+int32Func
 func

ReSharper.FSharp/test/data/features/naming/Pipe - Left 02 - Nested.fs

@@ -1,5 +1,5 @@
 module Module
 
-let getName _ = ()
+let getName _ _ = ()
 let foo _ = ()
 let x{caret} = getName <| foo <| 1

ReSharper.FSharp/test/data/features/naming/Pipe - Left 02 - Nested.fs.gold

@@ -1,2 +1,2 @@
 name
-o
+unit

ReSharper.FSharp/test/src/FSharp.Tests/Actions/TypingAssistTest.fs

@@ -7,13 +7,15 @@ open JetBrains.ReSharper.FeaturesTestFramework.TypingAssist
 open JetBrains.ReSharper.Plugins.FSharp.Psi.Features.TypingAssist
 open JetBrains.ReSharper.Plugins.FSharp.Services.Formatter
 open JetBrains.ReSharper.Plugins.FSharp.Tests
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open JetBrains.ReSharper.Plugins.FSharp.Util
 open JetBrains.ReSharper.Psi.CachingLexers
 open JetBrains.ReSharper.TestFramework
 open JetBrains.TextControl
 open NUnit.Framework
 
 [<FSharpTest; TestSettingsKey(typeof<FSharpFormatSettingsKey>)>]
+[<ExpectErrors>]
 type FSharpTypingAssistTest() =
     inherit TypingAssistTestBase()
 
@@ -291,6 +293,7 @@ type LineIndentsTest() =
         writer.WriteLine(getLineIndent x.CachingLexerService textControl line)
 
 [<FSharpTest>]
+[<ExpectErrors>]
 type NestedIndentsTest() =
     inherit LineIndentsTestBase()
 

ReSharper.FSharp/test/src/FSharp.Tests/Common.fs

@@ -2,18 +2,23 @@
 module JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 
 open System
+open System.Collections.Concurrent
+open System.Reflection
+open System.Runtime.InteropServices
 open System.Threading
 open FSharp.Compiler.SourceCodeServices
+open JetBrains.Annotations
+open JetBrains.Application
 open JetBrains.Application.Components
 open JetBrains.Application.platforms
 open JetBrains.DataFlow
+open JetBrains.DocumentModel
 open JetBrains.Lifetimes
 open JetBrains.ProjectModel
 open JetBrains.ProjectModel.MSBuild
 open JetBrains.ProjectModel.Properties.Managed
 open JetBrains.ReSharper.Plugins.FSharp
 open JetBrains.ReSharper.Plugins.FSharp.Checker
-open JetBrains.ReSharper.Plugins.FSharp.Checker.ProjectOptions
 open JetBrains.ReSharper.Plugins.FSharp.ProjectModel.ProjectProperties
 open JetBrains.ReSharper.Psi
 open JetBrains.ReSharper.TestFramework
@@ -53,6 +58,90 @@ type FSharpScriptTestAttribute() =
     inherit FSharpTestAttribute(FSharpScriptProjectFileType.FsxExtension)
 
 
+[<AttributeUsage(AttributeTargets.Method ||| AttributeTargets.Class, AllowMultiple=false)>]
+type ExpectErrors ([<ParamArray>] errorCodes: int[]) =
+    inherit Attribute()
+
+    new () = ExpectErrors [||]
+
+    member __.ErrorCodes = errorCodes
+
+
+[<ShellComponent>]
+type FSharpTestCheckerService(lifetime, logger, onSolutionCloseNotifier, settingsStore, settingsSchema) =
+    inherit FSharpCheckerService(lifetime, logger, onSolutionCloseNotifier, settingsStore, settingsSchema)
+
+    let baseExpectedCodes =
+        set [
+            222 // FS0222: Files in libraries or multiple-file applications must begin with a namespace or module
+        ]
+
+    let expectedErrorCodeCache = ConcurrentDictionary<string, int[] option>()
+
+    let getExpectedErrorCodes () =
+        expectedErrorCodeCache.GetOrAdd(TestContext.CurrentContext.Test.FullName, fun _ ->
+            let assembly = Assembly.GetExecutingAssembly()
+            let typ = assembly.GetType(TestContext.CurrentContext.Test.ClassName, true)
+            let testMethod = typ.GetMethod(TestContext.CurrentContext.Test.MethodName)
+
+            testMethod.GetCustomAttributes<ExpectErrors>()
+            |> Seq.tryExactlyOne
+            |> Option.orElseWith (fun () -> typ.GetCustomAttributes<ExpectErrors>() |> Seq.tryExactlyOne)
+            |> Option.map (fun attr -> attr.ErrorCodes))
+
+    let validateErrors (errors: FSharpErrorInfo[]) =
+        match getExpectedErrorCodes() with
+        | Some [||] ->
+            // All errors are expected
+            ()
+        | extraExpectedCodes ->
+
+        let expectedErrorCodes =
+            baseExpectedCodes
+            |> Set.union (Set.ofArray (defaultArg extraExpectedCodes [||]))
+
+        let unexpectedErrors =
+            errors
+            |> Array.filter (fun err ->
+                err.Severity = FSharpErrorSeverity.Error &&
+                not (expectedErrorCodes |> Set.contains err.ErrorNumber))
+
+        if unexpectedErrors.Length = 0 then () else
+
+        unexpectedErrors
+        |> Array.map (fun err ->
+            sprintf
+                "(%d,%d)-(%d,%d): %s %d: %s"
+                err.Range.StartLine err.Range.StartColumn
+                err.Range.EndLine err.Range.EndColumn
+                err.Subcategory err.ErrorNumber
+                err.Message)
+        |> String.concat "\n"
+        |> sprintf "Unexpected compile errors:\n%s\n"
+        |> Assert.Fail
+
+    interface IHideImplementation<FSharpCheckerService>
+
+    override x.ParseFile(path: FileSystemPath, document: IDocument, parsingOptions: FSharpParsingOptions) =
+        let results = base.ParseFile(path, document, parsingOptions)
+
+        match results with
+        | None -> failwithf "ParseFile failed unexpectedly"
+        | Some results -> validateErrors results.Errors
+
+        results
+
+    override x.ParseAndCheckFile([<NotNull>] file: IPsiSourceFile, opName,
+                                 [<Optional; DefaultParameterValue(false)>] allowStaleResults) =
+        let results = base.ParseAndCheckFile(file, opName, allowStaleResults)
+
+        match results with
+        | None -> failwithf "ParseAndCheckFile failed unexpectedly"
+        | Some results -> validateErrors results.CheckResults.Errors
+
+        results
+
+
 [<SolutionComponent>]
 type FSharpTestProjectOptionsBuilder(checkerService, logger) =
     inherit FSharpProjectOptionsBuilder(checkerService, logger, Mock<_>().Object)

ReSharper.FSharp/test/src/FSharp.Tests/Daemon/ErrorsHighlightingTest.fs

@@ -1,9 +1,11 @@
 namespace JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 
 open JetBrains.ReSharper.Plugins.FSharp
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open JetBrains.ReSharper.TestFramework
 open NUnit.Framework
 
+[<ExpectErrors>]
 type ErrorsHighlightingTest() =
     inherit FSharpHighlightingTestBase()
 

ReSharper.FSharp/test/src/FSharp.Tests/Daemon/FormatSpecifiersHighlightingTest.fs

@@ -1,6 +1,7 @@
 namespace JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 
 open JetBrains.ReSharper.Daemon.Impl
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open NUnit.Framework
 
 type FormatSpecifiersHighlightingTest() =
@@ -16,7 +17,7 @@ type FormatSpecifiersHighlightingTest() =
     [<Test>] member x.``Escaped strings``() = x.DoNamedTest()
     [<Test>] member x.``Triple quoted strings``() = x.DoNamedTest()
     [<Test>] member x.``Multi line strings``() = x.DoNamedTest()
-    [<Test>] member x.``Malformed formatters``() = x.DoNamedTest()
+    [<Test; ExpectErrors 741>] member x.``Malformed formatters``() = x.DoNamedTest()
     [<Test>] member x.``kprintf bprintf fprintf``() = x.DoNamedTest()
     [<Test>] member x.``Plane strings``() = x.DoNamedTest()
     [<Test>] member x.``Extensions``() = x.DoNamedTest()

ReSharper.FSharp/test/src/FSharp.Tests/Daemon/InheritanceGutterMarksTest.fs

@@ -1,6 +1,7 @@
 namespace JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 
 open JetBrains.ReSharper.Feature.Services.Daemon
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open NUnit.Framework
 
 type InheritanceGutterMarks() =
@@ -13,5 +14,5 @@ type InheritanceGutterMarks() =
 
     [<Test>] member x.``Inherited gutter mark``() = x.DoNamedTest()
 
-    [<Test>] member x.``Module 01``() = x.DoNamedTest()
-    [<Test>] member x.``Struct 01``() = x.DoNamedTest()
+    [<Test; ExpectErrors 39>] member x.``Module 01``() = x.DoNamedTest()
+    [<Test; ExpectErrors 945>] member x.``Struct 01``() = x.DoNamedTest()

ReSharper.FSharp/test/src/FSharp.Tests/Daemon/PipeChainTypeHintStageTest.fs

@@ -3,6 +3,7 @@
 open JetBrains.ReSharper.FeaturesTestFramework.Daemon
 open JetBrains.ReSharper.Plugins.FSharp.Psi.Features.Daemon.Highlightings
 open JetBrains.ReSharper.Plugins.FSharp.Settings
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open JetBrains.ReSharper.TestFramework
 open NUnit.Framework
 
@@ -27,8 +28,8 @@ type PipeChainTypeHintStageTest() =
     [<TestSettings("{HideSameLine:All}")>]
     [<Test>] member x.``Multi line 03 - Same line``() = x.DoNamedTest()
 
-    [<Test>] member x.``Error 01 - Invalid reference``() = x.DoNamedTest()
-    [<Test>] member x.``Error 02 - Syntax error``() = x.DoNamedTest()
+    [<Test; ExpectErrors 39>] member x.``Error 01 - Invalid reference``() = x.DoNamedTest()
+    [<Test; ExpectErrors 10>] member x.``Error 02 - Syntax error``() = x.DoNamedTest()
 
     [<Test>] member x.``Skipped 01 - Other binary op``() = x.DoNamedTest()
     [<Test>] member x.``Skipped 02 - Shadowed pipe op``() = x.DoNamedTest()

ReSharper.FSharp/test/src/FSharp.Tests/Daemon/RedundantAttributeParensTest.fs

@@ -1,9 +1,11 @@
 namespace JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 
 open JetBrains.ReSharper.Plugins.FSharp.Psi.Features.Daemon.Highlightings
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 open NUnit.Framework
 
+[<ExpectErrors 39>]
 type RedundantAttributeParensTest() =
     inherit FSharpHighlightingTestBase()
 

ReSharper.FSharp/test/src/FSharp.Tests/Daemon/RedundantAttributeSuffixAnalyzerTest.fs

@@ -1,6 +1,7 @@
 namespace JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 
 open JetBrains.ReSharper.Plugins.FSharp.Psi.Features.Daemon.Highlightings
+open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open JetBrains.ReSharper.Plugins.FSharp.Tests.Features.Daemon
 open NUnit.Framework
 
@@ -16,4 +17,4 @@ type RedundantAttributeAnalyzerTest() =
     [<Test>] member x.``Single attribute 02 - Needed suffix``() = x.DoNamedTest()
     [<Test>] member x.``Single attribute 03 - With constructor``() = x.DoNamedTest()
     [<Test>] member x.``Single attribute 04 - With target``() = x.DoNamedTest()
-    [<Test>] member x.``Single attribute 05 - Name is just Attribute``() = x.DoNamedTest()
+    [<Test; ExpectErrors 509>] member x.``Single attribute 05 - Name is just Attribute``() = x.DoNamedTest()

ReSharper.FSharp/test/src/FSharp.Tests/Debugger/ExpressionEvaluationInfoTest.fs

@@ -22,7 +22,7 @@ type ExpressionEvaluationInfoTest() =
             writer.WriteLine(textToEvaluate)) |> ignore
 
     [<Test>] member x.``Id 01``() = x.DoNamedTest()
-    [<Test>] member x.``Id 02 - Qualifier``() = x.DoNamedTest()
+    [<Test; ExpectErrors 39>] member x.``Id 02 - Qualifier``() = x.DoNamedTest()
 
     [<Test>] member x.``SelfId 01 - Member``() = x.DoNamedTest()
     [<Test>] member x.``SelfId 02 - Type``() = x.DoNamedTest()

ReSharper.FSharp/test/src/FSharp.Tests/FSharpCompletionTest.fs

@@ -6,7 +6,7 @@ open JetBrains.ReSharper.Plugins.FSharp.Tests.Common
 open JetBrains.ReSharper.TestFramework
 open NUnit.Framework
 
-[<FSharpTest>]
+[<FSharpTest; ExpectErrors>]
 type FSharpCompletionTest() =
     inherit CodeCompletionTestBase()
 

ReSharper.FSharp/test/src/FSharp.Tests/FSharpOccurenceKindTest.fs

@@ -20,10 +20,10 @@ type FSharpOccurenceKindTest() =
     [<Test>] member x.``Unions 02 - Single empty case``() = x.DoNamedTest()
 
     [<Test>] member x.``Exception - Singleton 01``() = x.DoNamedTest()
-    [<Test>] member x.``Exception - Singleton 02 - Type specification``() = x.DoNamedTest()
+    [<Test; ExpectErrors 1>] member x.``Exception - Singleton 02 - Type specification``() = x.DoNamedTest()
     [<Test>] member x.``Exception - Singleton 03 - Members``() = x.DoNamedTest()
     [<Test>] member x.``Exception - Fields 01``() = x.DoNamedTest()
-    [<Test>] member x.``Exception - Fields 02 - Type specification``() = x.DoNamedTest()
+    [<Test; ExpectErrors 1>] member x.``Exception - Fields 02 - Type specification``() = x.DoNamedTest()
     [<Test>] member x.``Exception - Fields 03 - Members``() = x.DoNamedTest()
 
     [<Test>] member x.``Base Type 01``() = x.DoNamedTest()
@@ -35,19 +35,19 @@ type FSharpOccurenceKindTest() =
     [<Test>] member x.``Base Type - Object expressions 02 - Interface``() = x.DoNamedTest()
     [<Test>] member x.``Base Type - Object expressions 03 - Secondary interfaces``() = x.DoNamedTest()
 
-    [<Test>] member x.``Type Argument 01``() = x.DoNamedTest()
-    [<Test>] member x.``Type Argument 02 - Pattern``() = x.DoNamedTest()
-    [<Test>] member x.``Type Argument 03 - Tuple``() = x.DoNamedTest()
+    [<Test; ExpectErrors 39>] member x.``Type Argument 01``() = x.DoNamedTest()
+    [<Test; ExpectErrors 39>] member x.``Type Argument 02 - Pattern``() = x.DoNamedTest()
+    [<Test; ExpectErrors 39>] member x.``Type Argument 03 - Tuple``() = x.DoNamedTest()
     [<Test>] member x.``Type Argument 04 - Array``() = x.DoNamedTest()
-    [<Test>] member x.``Type Argument 05 - Return type``() = x.DoNamedTest()
-    [<Test>] member x.``Type Argument 06 - ML``() = x.DoNamedTest()
+    [<Test; ExpectErrors 1>] member x.``Type Argument 05 - Return type``() = x.DoNamedTest()
+    [<Test; ExpectErrors 1>] member x.``Type Argument 06 - ML``() = x.DoNamedTest()
     [<Test>] member x.``Type Argument 07 - Tuple abbreviation``() = x.DoNamedTest()
 
     [<Test>] member x.``Type Cast 01 - Upcast``() = x.DoNamedTest()
     [<Test>] member x.``Type Cast 02 - Downcast``() = x.DoNamedTest()
 
-    [<Test>] member x.``Type Test 01 - Expr``() = x.DoNamedTest()
-    [<Test>] member x.``Type Test 02 - Pattern``() = x.DoNamedTest()
+    [<Test; ExpectErrors 16>] member x.``Type Test 01 - Expr``() = x.DoNamedTest()
+    [<Test; ExpectErrors 16>] member x.``Type Test 02 - Pattern``() = x.DoNamedTest()
 
     [<Test>] member x.``Type abbreviation 01``() = x.DoNamedTest()