diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs index 966eb13863..aec6cdcd0a 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs @@ -5,6 +5,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; using Microsoft.VisualStudio.TestTools.UnitTesting; using ObjectModelUnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; @@ -113,7 +114,7 @@ public virtual TestResult Invoke(object?[]? arguments) watch.Start(); try { - result = IsTimeoutSet ? ExecuteInternalWithTimeout(arguments) : ExecuteInternal(arguments); + result = IsTimeoutSet ? ExecuteInternalWithTimeout(arguments) : ExecuteInternal(arguments, null); } finally { @@ -217,7 +218,7 @@ public virtual TestResult Invoke(object?[]? arguments) /// Arguments to be passed to the method. /// The result of the execution. [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] - private TestResult ExecuteInternal(object?[]? arguments) + private TestResult ExecuteInternal(object?[]? arguments, CancellationTokenSource? timeoutTokenSource) { DebugEx.Assert(TestMethod != null, "UnitTestExecuter.DefaultTestMethodInvoke: testMethod = null."); @@ -239,7 +240,7 @@ private TestResult ExecuteInternal(object?[]? arguments) // For any failure after this point, we must run TestCleanup _isTestContextSet = true; - if (RunTestInitializeMethod(_classInstance, result)) + if (RunTestInitializeMethod(_classInstance, result, timeoutTokenSource)) { hasTestInitializePassed = true; if (IsTimeoutSet) @@ -267,12 +268,21 @@ private TestResult ExecuteInternal(object?[]? arguments) // Expected Exception was thrown, so Pass the test result.Outcome = UTF.UnitTestOutcome.Passed; } - else if (realException is OperationCanceledException oce && oce.CancellationToken == TestMethodOptions.TestContext?.Context.CancellationTokenSource.Token) + else if (realException.IsOperationCanceledExceptionFromToken(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) { result.Outcome = UTF.UnitTestOutcome.Timeout; result.TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, - string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName)); + timeoutTokenSource?.Token.IsCancellationRequested == true + ? string.Format( + CultureInfo.InvariantCulture, + Resource.Execution_Test_Timeout, + TestMethodName, + TestMethodOptions.TimeoutInfo.Timeout) + : string.Format( + CultureInfo.InvariantCulture, + Resource.Execution_Test_Cancelled, + TestMethodName)); } else { @@ -321,7 +331,7 @@ private TestResult ExecuteInternal(object?[]? arguments) // Pulling it out so extension writers can abort custom cleanups if need be. Having this in a finally block // does not allow a thread abort exception to be raised within the block but throws one after finally is executed // crashing the process. This was blocking writing an extension for Dynamic Timeout in VSO. - RunTestCleanupMethod(result); + RunTestCleanupMethod(result, timeoutTokenSource); return testRunnerException != null ? throw testRunnerException : result; } @@ -466,7 +476,7 @@ private static TestFailedException HandleMethodException(Exception ex, Exception /// /// Instance of TestResult. [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] - private void RunTestCleanupMethod(TestResult result) + private void RunTestCleanupMethod(TestResult result, CancellationTokenSource? timeoutTokenSource) { DebugEx.Assert(result != null, "result != null"); @@ -482,16 +492,25 @@ private void RunTestCleanupMethod(TestResult result) { try { + // Reset the cancellation token source to avoid cancellation of cleanup methods because of the init or test method cancellation. + TestMethodOptions.TestContext!.Context.CancellationTokenSource = new CancellationTokenSource(); + + // If we are running with a method timeout, we need to cancel the cleanup when the overall timeout expires. If it already expired, nothing to do. + if (timeoutTokenSource is { IsCancellationRequested: false }) + { + timeoutTokenSource?.Token.Register(TestMethodOptions.TestContext.Context.CancellationTokenSource.Cancel); + } + // Test cleanups are called in the order of discovery // Current TestClass -> Parent -> Grandparent testCleanupException = testCleanupMethod is not null - ? InvokeCleanupMethod(testCleanupMethod, _classInstance, Parent.BaseTestCleanupMethodsQueue.Count) + ? InvokeCleanupMethod(testCleanupMethod, _classInstance, Parent.BaseTestCleanupMethodsQueue.Count, timeoutTokenSource) : null; var baseTestCleanupQueue = new Queue(Parent.BaseTestCleanupMethodsQueue); while (baseTestCleanupQueue.Count > 0 && testCleanupException is null) { testCleanupMethod = baseTestCleanupQueue.Dequeue(); - testCleanupException = InvokeCleanupMethod(testCleanupMethod, _classInstance, baseTestCleanupQueue.Count); + testCleanupException = InvokeCleanupMethod(testCleanupMethod, _classInstance, baseTestCleanupQueue.Count, timeoutTokenSource); } } finally @@ -515,9 +534,9 @@ private void RunTestCleanupMethod(TestResult result) } // If the exception is already a `TestFailedException` we throw it as-is - if (testCleanupException is TestFailedException) + if (testCleanupException is TestFailedException tfe) { - result.Outcome = UTF.UnitTestOutcome.Failed; + result.Outcome = tfe.Outcome.ToAdapterOutcome(); result.TestFailureException = testCleanupException; return; } @@ -593,7 +612,7 @@ private void RunTestCleanupMethod(TestResult result) /// Instance of TestResult. /// True if the TestInitialize method(s) did not throw an exception. [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] - private bool RunTestInitializeMethod(object classInstance, TestResult result) + private bool RunTestInitializeMethod(object classInstance, TestResult result, CancellationTokenSource? timeoutTokenSource) { DebugEx.Assert(classInstance != null, "classInstance != null"); DebugEx.Assert(result != null, "result != null"); @@ -609,7 +628,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) while (baseTestInitializeStack.Count > 0) { testInitializeMethod = baseTestInitializeStack.Pop(); - testInitializeException = testInitializeMethod is not null ? InvokeInitializeMethod(testInitializeMethod, classInstance) : null; + testInitializeException = testInitializeMethod is not null + ? InvokeInitializeMethod(testInitializeMethod, classInstance, timeoutTokenSource) + : null; if (testInitializeException is not null) { break; @@ -619,7 +640,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) if (testInitializeException == null) { testInitializeMethod = Parent.TestInitializeMethod; - testInitializeException = testInitializeMethod is not null ? InvokeInitializeMethod(testInitializeMethod, classInstance) : null; + testInitializeException = testInitializeMethod is not null + ? InvokeInitializeMethod(testInitializeMethod, classInstance, timeoutTokenSource) + : null; } } catch (Exception ex) @@ -634,9 +657,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) } // If the exception is already a `TestFailedException` we throw it as-is - if (testInitializeException is TestFailedException) + if (testInitializeException is TestFailedException tfe) { - result.Outcome = UTF.UnitTestOutcome.Failed; + result.Outcome = tfe.Outcome.ToAdapterOutcome(); result.TestFailureException = testInitializeException; return false; } @@ -655,7 +678,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) exceptionMessage); StackTraceInformation? stackTrace = realException.GetStackTraceInformation(); - result.Outcome = realException is AssertInconclusiveException ? UTF.UnitTestOutcome.Inconclusive : UTF.UnitTestOutcome.Failed; + result.Outcome = realException is AssertInconclusiveException + ? UTF.UnitTestOutcome.Inconclusive + : UTF.UnitTestOutcome.Failed; result.TestFailureException = new TestFailedException( result.Outcome.ToUnitTestOutcome(), errorMessage, @@ -665,7 +690,7 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) return false; } - private TestFailedException? InvokeInitializeMethod(MethodInfo methodInfo, object classInstance) + private TestFailedException? InvokeInitializeMethod(MethodInfo methodInfo, object classInstance, CancellationTokenSource? timeoutTokenSource) { TimeoutInfo? timeout = null; if (Parent.TestInitializeMethodTimeoutMilliseconds.TryGetValue(methodInfo, out TimeoutInfo localTimeout)) @@ -680,10 +705,13 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) methodInfo, new InstanceExecutionContextScope(classInstance, Parent.ClassType), Resource.TestInitializeWasCancelled, - Resource.TestInitializeTimedOut); + Resource.TestInitializeTimedOut, + timeoutTokenSource is null + ? null + : (timeoutTokenSource, TestMethodOptions.TimeoutInfo.Timeout)); } - private TestFailedException? InvokeCleanupMethod(MethodInfo methodInfo, object classInstance, int remainingCleanupCount) + private TestFailedException? InvokeCleanupMethod(MethodInfo methodInfo, object classInstance, int remainingCleanupCount, CancellationTokenSource? timeoutTokenSource) { TimeoutInfo? timeout = null; if (Parent.TestCleanupMethodTimeoutMilliseconds.TryGetValue(methodInfo, out TimeoutInfo localTimeout)) @@ -698,7 +726,10 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) methodInfo, new InstanceExecutionContextScope(classInstance, Parent.ClassType, remainingCleanupCount), Resource.TestCleanupWasCancelled, - Resource.TestCleanupTimedOut); + Resource.TestCleanupTimedOut, + timeoutTokenSource is null + ? null + : (timeoutTokenSource, TestMethodOptions.TimeoutInfo.Timeout)); } /// @@ -782,18 +813,27 @@ private bool SetTestContext(object classInstance, TestResult result) // In most cases, exception will be TargetInvocationException with real exception wrapped // in the InnerException; or user code throws an exception. // It also seems that in rare cases the ex can be null. - Exception actualException = ex.InnerException ?? ex; - string exceptionMessage = actualException.GetFormattedExceptionMessage(); - StackTraceInformation? stackTraceInfo = actualException.GetStackTraceInformation(); + Exception realException = ex.GetRealException(); - string errorMessage = string.Format( - CultureInfo.CurrentCulture, - Resource.UTA_InstanceCreationError, - TestClassName, - exceptionMessage); + if (realException.IsOperationCanceledExceptionFromToken(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) + { + result.Outcome = UTF.UnitTestOutcome.Timeout; + result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Timeout, string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout)); + } + else + { + string exceptionMessage = realException.GetFormattedExceptionMessage(); + StackTraceInformation? stackTraceInfo = realException.GetStackTraceInformation(); - result.Outcome = UTF.UnitTestOutcome.Failed; - result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Failed, errorMessage, stackTraceInfo); + string errorMessage = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_InstanceCreationError, + TestClassName, + exceptionMessage); + + result.Outcome = UTF.UnitTestOutcome.Failed; + result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Failed, errorMessage, stackTraceInfo); + } } return classInstance; @@ -823,13 +863,13 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) Outcome = UTF.UnitTestOutcome.Timeout, TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, - string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName)), + string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout)), }; } try { - return ExecuteInternal(arguments); + return ExecuteInternal(arguments, timeoutTokenSource); } catch (OperationCanceledException) { @@ -841,7 +881,7 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, timeoutTokenSource.Token.IsCancellationRequested - ? string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName) + ? string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout) : string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName)), }; } @@ -849,26 +889,14 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) finally { timeoutTokenSource?.Dispose(); + timeoutTokenSource = null; } } TestResult? result = null; Exception? failure = null; - void ExecuteAsyncAction() - { - try - { - result = ExecuteInternal(arguments); - } - catch (Exception ex) - { - failure = ex; - } - } - - CancellationToken cancelToken = TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token; - if (PlatformServiceProvider.Instance.ThreadOperations.Execute(ExecuteAsyncAction, TestMethodOptions.TimeoutInfo.Timeout, cancelToken)) + if (PlatformServiceProvider.Instance.ThreadOperations.Execute(ExecuteAsyncAction, TestMethodOptions.TimeoutInfo.Timeout, TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) { if (failure != null) { @@ -879,12 +907,12 @@ void ExecuteAsyncAction() // It's possible that some failures happened and that the cleanup wasn't executed, so we need to run it here. // The method already checks if the cleanup was already executed. - RunTestCleanupMethod(result); + RunTestCleanupMethod(result, null); return result; } // Timed out or canceled - string errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName); + string errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout); if (TestMethodOptions.TestContext.Context.CancellationTokenSource.IsCancellationRequested) { errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName); @@ -899,7 +927,20 @@ void ExecuteAsyncAction() // We don't know when the cancellation happened so it's possible that the cleanup wasn't executed, so we need to run it here. // The method already checks if the cleanup was already executed. - RunTestCleanupMethod(timeoutResult); + RunTestCleanupMethod(timeoutResult, null); return timeoutResult; + + // Local functions + void ExecuteAsyncAction() + { + try + { + result = ExecuteInternal(arguments, null); + } + catch (Exception ex) + { + failure = ex; + } + } } } diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs index 301984dbb1..ecdd6901b5 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs @@ -36,7 +36,9 @@ internal static UnitTestResult[] ToUnitTestResults(this IReadOnlyCollection UnitTestOutcome.Error, }; + internal static UTF.UnitTestOutcome ToAdapterOutcome(this UnitTestOutcome outcome) + => outcome switch + { + UnitTestOutcome.Failed => UTF.UnitTestOutcome.Failed, + UnitTestOutcome.Inconclusive => UTF.UnitTestOutcome.Inconclusive, + UnitTestOutcome.InProgress => UTF.UnitTestOutcome.InProgress, + UnitTestOutcome.Passed => UTF.UnitTestOutcome.Passed, + UnitTestOutcome.Timeout => UTF.UnitTestOutcome.Timeout, + UnitTestOutcome.NotRunnable => UTF.UnitTestOutcome.NotRunnable, + UnitTestOutcome.NotFound => UTF.UnitTestOutcome.NotFound, + _ => UTF.UnitTestOutcome.Error, + }; + /// /// Returns more important outcome of two. /// diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs index c7f5f4b126..79dd00a0aa 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs @@ -5,6 +5,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; using UnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; @@ -14,8 +15,18 @@ internal static class FixtureMethodRunner { internal static TestFailedException? RunWithTimeoutAndCancellation( Action action, CancellationTokenSource cancellationTokenSource, TimeoutInfo? timeoutInfo, MethodInfo methodInfo, - IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat) + IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat, + // When a test method is marked with [Timeout], this timeout is applied from ctor to destructor, so we need to take + // that into account when processing the OCE of the action. + (CancellationTokenSource TokenSource, int Timeout)? testTimeoutInfo = default) { + if (cancellationTokenSource.Token.IsCancellationRequested) + { + return new( + UnitTestOutcome.Timeout, + string.Format(CultureInfo.InvariantCulture, methodCanceledMessageFormat, methodInfo.DeclaringType!.FullName, methodInfo.Name)); + } + // If no timeout is specified, we can run the action directly. This avoids any overhead of creating a task/thread and // ensures that the execution context is preserved (as we run the action on the current thread). if (timeoutInfo is null) @@ -27,13 +38,22 @@ internal static class FixtureMethodRunner } catch (Exception ex) { - Exception realException = ex.GetRealException(); - - if (realException is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) + if (ex.GetRealException().IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { return new( UnitTestOutcome.Timeout, - string.Format(CultureInfo.InvariantCulture, methodCanceledMessageFormat, methodInfo.DeclaringType!.FullName, methodInfo.Name)); + testTimeoutInfo?.TokenSource.Token.IsCancellationRequested == true + ? string.Format( + CultureInfo.InvariantCulture, + methodTimedOutMessageFormat, + methodInfo.DeclaringType!.FullName, + methodInfo.Name, + testTimeoutInfo.Value.Timeout) + : string.Format( + CultureInfo.InvariantCulture, + methodCanceledMessageFormat, + methodInfo.DeclaringType!.FullName, + methodInfo.Name)); } throw; @@ -75,7 +95,7 @@ internal static class FixtureMethodRunner action(); return null; } - catch (OperationCanceledException) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { // Ideally we would like to check that the token of the exception matches cancellationTokenSource but TestContext // instances are not well defined so we have to handle the exception entirely. @@ -148,9 +168,7 @@ internal static class FixtureMethodRunner methodInfo.Name, timeout)); } - catch (Exception ex) when - (ex is OperationCanceledException - || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { return new( UnitTestOutcome.Timeout, @@ -168,7 +186,7 @@ internal static class FixtureMethodRunner } } - [System.Runtime.Versioning.SupportedOSPlatform("windows")] + [SupportedOSPlatform("windows")] private static TestFailedException? RunWithTimeoutAndCancellationWithSTAThread( Action action, CancellationTokenSource cancellationTokenSource, int timeout, MethodInfo methodInfo, IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat) @@ -212,12 +230,7 @@ internal static class FixtureMethodRunner methodInfo.Name, timeout)); } - catch (Exception ex) when - - // This exception occurs when the cancellation happens before the task is actually started. - ((ex is TaskCanceledException tce && tce.CancellationToken == cancellationTokenSource.Token) - || (ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) - || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { return new( UnitTestOutcome.Timeout, diff --git a/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs index 1614764330..d459e481e5 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs @@ -279,7 +279,7 @@ internal static string ExceptionsThrown { } /// - /// Looks up a localized string similar to Test '{0}' execution has been aborted.. + /// Looks up a localized string similar to Test '{0}' was canceled. /// internal static string Execution_Test_Cancelled { get { @@ -288,7 +288,7 @@ internal static string Execution_Test_Cancelled { } /// - /// Looks up a localized string similar to Test '{0}' exceeded execution timeout period.. + /// Looks up a localized string similar to Test '{0}' timed out after {1}ms. /// internal static string Execution_Test_Timeout { get { diff --git a/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx b/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx index 4a476afa7d..fba272ebdf 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx @@ -118,7 +118,7 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - Test '{0}' exceeded execution timeout period. + Test '{0}' timed out after {1}ms Running tests in any of the provided sources is not supported for the selected platform @@ -308,7 +308,7 @@ Error: {1} Cannot run test method '{0}.{1}': Method has parameters, but does not define any test source. Use '[DataRow]', '[DynamicData]', or a custom 'ITestDataSource' data source to provide test data. - Test '{0}' execution has been aborted. + Test '{0}' was canceled Exception occurred while enumerating IDataSource attribute on "{0}.{1}": {2} diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf index 75dc16e953..3184131b78 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf @@ -67,9 +67,9 @@ byl však přijat tento počet argumentů: {4} s typy {5}. - Test '{0}' exceeded execution timeout period. - Test {0} překročil časový limit spuštění. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Chyba: {1} - Test '{0}' execution has been aborted. - Spouštění testu {0} se přerušilo. - + Test '{0}' was canceled + Spouštění testu {0} se přerušilo. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf index 0e8d214715..f10f5c2787 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf @@ -67,9 +67,9 @@ aber empfing {4} Argument(e) mit den Typen „{5}“. - Test '{0}' exceeded execution timeout period. - Der Test "{0}" hat das Ausführungstimeout überschritten. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Fehler: {1} - Test '{0}' execution has been aborted. - Die Ausführung des Tests "{0}" wurde abgebrochen. - + Test '{0}' was canceled + Die Ausführung des Tests "{0}" wurde abgebrochen. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf index 395a71c5bd..fe8e4275a6 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf @@ -67,9 +67,9 @@ pero recibió {4} argumento(s), con los tipos "{5}". - Test '{0}' exceeded execution timeout period. - La prueba '{0}' superó el tiempo de espera de ejecución. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - Se anuló la ejecución de la prueba "{0}". - + Test '{0}' was canceled + Se anuló la ejecución de la prueba "{0}". + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf index b553cacfa8..a91f23e7cf 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf @@ -67,9 +67,9 @@ mais a reçu {4} argument(s), avec les types « {5} ». - Test '{0}' exceeded execution timeout period. - Le test '{0}' a dépassé le délai d'attente de l'exécution. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Erreur : {1} - Test '{0}' execution has been aborted. - L'exécution du test '{0}' a été abandonnée. - + Test '{0}' was canceled + L'exécution du test '{0}' a été abandonnée. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf index 6c5116a95b..917dfb783d 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf @@ -67,9 +67,9 @@ ma ha ricevuto {4} argomenti, con tipi "{5}". - Test '{0}' exceeded execution timeout period. - È stato superato il periodo di timeout per l'esecuzione del test '{0}'. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Errore: {1} - Test '{0}' execution has been aborted. - L'esecuzione del test '{0}' è stata interrotta. - + Test '{0}' was canceled + L'esecuzione del test '{0}' è stata interrotta. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf index 01bed386e8..85c9c7e0eb 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf @@ -68,9 +68,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - テスト '{0}' は実行タイムアウトを超えました。 - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -419,9 +419,9 @@ Error: {1} - Test '{0}' execution has been aborted. - テスト '{0}' の実行が中止されました。 - + Test '{0}' was canceled + テスト '{0}' の実行が中止されました。 + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf index f5710bb237..12e3c7bc3b 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - '{0}' 테스트가 실행 시간 제한을 초과했습니다. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - 테스트 '{0}' 실행이 중단되었습니다. - + Test '{0}' was canceled + 테스트 '{0}' 실행이 중단되었습니다. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf index 8e18340251..63394f6986 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf @@ -67,9 +67,9 @@ ale liczba odebranych argumentów to {4} z typami „{5}”. - Test '{0}' exceeded execution timeout period. - Test „{0}” przekroczył okres limitu czasu na wykonanie. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Błąd: {1} - Test '{0}' execution has been aborted. - Wykonanie testu „{0}” zostało przerwane. - + Test '{0}' was canceled + Wykonanie testu „{0}” zostało przerwane. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf index d8232e39e2..6982a4d066 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf @@ -67,9 +67,9 @@ mas {4} argumentos recebidos, com tipos '{5}'. - Test '{0}' exceeded execution timeout period. - Teste '{0}' ultrapassou o período de tempo limite de execução. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Erro: {1} - Test '{0}' execution has been aborted. - A execução do teste '{0}' foi anulada. - + Test '{0}' was canceled + A execução do teste '{0}' foi anulada. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf index ad1b9f9366..102c60d3bb 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - Превышено время ожидания выполнения теста "{0}". - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - Выполнение теста "{0}" было прервано. - + Test '{0}' was canceled + Выполнение теста "{0}" было прервано. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf index 6854ff4926..0c266c1565 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf @@ -67,9 +67,9 @@ ancak, '{5}' türüyle {4} argüman aldı. - Test '{0}' exceeded execution timeout period. - '{0}' testi yürütme zaman aşımı süresini aştı. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Hata: {1} - Test '{0}' execution has been aborted. - '{0}' testinin yürütülmesi iptal edildi. - + Test '{0}' was canceled + '{0}' testinin yürütülmesi iptal edildi. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf index 9f451c98c1..8d35d39c8f 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - 测试“{0}”的执行超时。 - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - 已中止测试“{0}”的执行。 - + Test '{0}' was canceled + 已中止测试“{0}”的执行。 + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf index 2dce2845f5..d9a3cf3089 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - 測試 '{0}' 超過執行逾時期限。 - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - 測試 '{0}' 執行已中止。 - + Test '{0}' was canceled + 測試 '{0}' 執行已中止。 + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs b/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs index 37b35a029c..04389c8555 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !WINDOWS_UWP - namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; /// @@ -32,5 +30,8 @@ internal static string GetExceptionMessage(this Exception? exception) return exceptionString; } + + internal static bool IsOperationCanceledExceptionFromToken(this Exception ex, CancellationToken cancellationToken) + => (ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) + || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any(oce => oce.CancellationToken == cancellationToken)); } -#endif diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs b/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs index d4c8b4ee1b..52ee8d5205 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; @@ -43,11 +44,7 @@ private static bool ExecuteWithThreadPool(Action action, int timeout, Cancellati // False means execution timed out. return executionTask.Wait(timeout, cancellationToken); } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) - - // This exception occurs when the cancellation happens before the task is actually started. - || (ex is TaskCanceledException tce && tce.CancellationToken == cancellationToken)) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationToken)) { // Task execution canceled. return false; @@ -81,11 +78,7 @@ private static bool ExecuteWithCustomThread(Action action, int timeout, Cancella // If the execution thread completes before the timeout, the task will return true, otherwise false. return executionTask.Result; } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) - - // This exception occurs when the cancellation happens before the task is actually started. - || (ex is TaskCanceledException tce && tce.CancellationToken == cancellationToken)) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationToken)) { // Task execution canceled. return false; diff --git a/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs b/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs index f0d1e3a689..7ed69424de 100644 --- a/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs +++ b/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs @@ -124,7 +124,7 @@ private static void CollectTestContextFieldsAssignedInConstructor( } if (operation is ISimpleAssignmentOperation assignmentOperation && - assignmentOperation.Target is IMemberReferenceOperation { Member: IFieldSymbol { } candidateField } targetMemberReference && + assignmentOperation.Target is IMemberReferenceOperation { Member: IFieldSymbol { } candidateField } && assignmentOperation.Value is IParameterReferenceOperation parameterReference && SymbolEqualityComparer.Default.Equals(parameterReference.Parameter, testContextParameter)) { diff --git a/src/TestFramework/TestFramework.Extensions/TestContext.cs b/src/TestFramework/TestFramework.Extensions/TestContext.cs index 418a7007e0..feb96117db 100644 --- a/src/TestFramework/TestFramework.Extensions/TestContext.cs +++ b/src/TestFramework/TestFramework.Extensions/TestContext.cs @@ -48,7 +48,7 @@ public abstract class TestContext /// /// Gets or sets the cancellation token source. This token source is canceled when test times out. Also when explicitly canceled the test will be aborted. /// - public virtual CancellationTokenSource CancellationTokenSource { get; protected set; } = new(); + public virtual CancellationTokenSource CancellationTokenSource { get; protected internal set; } = new(); public object?[]? TestData { get; protected set; } diff --git a/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj b/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj index a66f58bbf3..7aadfd3436 100644 --- a/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj +++ b/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj @@ -81,6 +81,10 @@ + + + + PreserveNewest diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs index 36124bf44a..e3ece73a25 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs @@ -110,7 +110,7 @@ public async Task WhenCancelingTestContextTokenInTestMethod_MessageIsAsExpected( // Assert testHostResult.AssertExitCodeIs(2); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); + testHostResult.AssertOutputContains("Test 'TestMethod' was canceled"); testHostResult.AssertOutputContains("Failed!"); } diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs deleted file mode 100644 index feccbd7bbd..0000000000 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs +++ /dev/null @@ -1,762 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Microsoft.Testing.Platform.Acceptance.IntegrationTests; -using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; - -namespace MSTest.Acceptance.IntegrationTests; - -[TestClass] -public class InitializeAndCleanupTimeoutTests : AcceptanceTestBase -{ - private static readonly Dictionary InfoByKind = new() - { - ["assemblyInit"] = ("TestClass.AssemblyInit", "Assembly initialize", "ASSEMBLYINIT", "AssemblyInitializeTimeout"), - ["assemblyCleanup"] = ("TestClass.AssemblyCleanupMethod", "Assembly cleanup", "ASSEMBLYCLEANUP", "AssemblyCleanupTimeout"), - ["classInit"] = ("TestClass.ClassInit", "Class initialize", "CLASSINIT", "ClassInitializeTimeout"), - ["baseClassInit"] = ("TestClassBase.ClassInitBase", "Class initialize", "BASE_CLASSINIT", "ClassInitializeTimeout"), - ["classCleanup"] = ("TestClass.ClassCleanupMethod", "Class cleanup", "CLASSCLEANUP", "ClassCleanupTimeout"), - ["baseClassCleanup"] = ("TestClassBase.ClassCleanupBase", "Class cleanup", "BASE_CLASSCLEANUP", "ClassCleanupTimeout"), - ["testInit"] = ("TestClass.TestInit", "Test initialize", "TESTINIT", "TestInitializeTimeout"), - ["testCleanup"] = ("TestClass.TestCleanupMethod", "Test cleanup", "TESTCLEANUP", "TestCleanupTimeout"), - }; - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInit_WhenTestContextCanceled_AssemblyInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, - tfm, "TESTCONTEXT_CANCEL_", "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInit_WhenTimeoutExpires_AssemblyInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, - tfm, "LONG_WAIT_", "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_AssemblyInitializeExits(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "TIMEOUT_", "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInit_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, - "TESTCONTEXT_CANCEL_", "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInit_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "TIMEOUT_", "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitBase_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, - "TESTCONTEXT_CANCEL_", "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitBase_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitBase_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "TIMEOUT_", "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInitialize_WhenTimeoutExpires_FromRunSettings_AssemblyInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInitialize_WhenTimeoutExpires_AssemblyInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanupBase_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "baseClassCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "classCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "assemblyCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyCleanup_WhenTimeoutExpires_FromRunSettings_AssemblyCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestInitialize_WhenTimeoutExpires_TestInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "testInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestInitialize_WhenTimeoutExpires_FromRunSettings_TestInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestInitialize_WhenTimeoutExpires_TestInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestCleanup_WhenTimeoutExpires_TestCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "testCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestCleanup_WhenTimeoutExpires_FromRunSettings_TestCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestCleanup_WhenTimeoutExpires_TestCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_ASSEMBLYINIT"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyInit started"); - testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("AssemblyInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_ASSEMBLYCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyCleanup started"); - testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("AssemblyCleanup Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassInitTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_CLASSINIT"] = "1" }); - - testHostResult.AssertOutputContains("ClassInit started"); - testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("ClassInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("ClassInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_CLASSCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("ClassCleanup started"); - testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("ClassCleanup Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestInitTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_TESTINIT"] = "1" }); - - testHostResult.AssertOutputContains("TestInit started"); - testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("TestInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_TESTCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("TestCleanup started"); - // TODO: We would expect to have the following line but that's not the case - // testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); - testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' was canceled"); - testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); - } - - [TestMethod] - [Ignore("Move to a different project")] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestMethodTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_TESTMETHOD"] = "1" }); - - testHostResult.AssertOutputContains("TestMethod started"); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); - testHostResult.AssertOutputDoesNotContain("TestMethod completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_ASSEMBLYINIT"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyInit started"); - testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); - testHostResult.AssertOutputContains("AssemblyInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_ASSEMBLYCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyCleanup started"); - testHostResult.AssertOutputContains("AssemblyCleanup Thread.Sleep completed"); - testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_CLASSINIT"] = "1" }); - - testHostResult.AssertOutputContains("ClassInit started"); - testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); - testHostResult.AssertOutputContains("ClassInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("ClassInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_CLASSCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("ClassCleanup started"); - testHostResult.AssertOutputContains("ClassCleanup Thread.Sleep completed"); - testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_TESTINIT"] = "1" }); - - testHostResult.AssertOutputContains("TestInit started"); - testHostResult.AssertOutputDoesNotContain("TestInit completed"); - testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_TESTCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("TestCleanup started"); - testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); - testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); - } - - [TestMethod] - [Ignore("Move to a different project")] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestMethodTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_TESTMETHOD"] = "1" }); - - testHostResult.AssertOutputContains("TestMethod started"); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); - } - - private async Task RunAndAssertTestWasCanceledAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) - { - var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); - testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' was canceled"); - } - - private async Task RunAndAssertTestTimedOutAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) - { - var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); - testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after 1000ms"); - } - - private async Task RunAndAssertWithRunSettingsAsync(string tfm, int timeoutValue, bool assertAttributePrecedence, string entryKind) - { - string runSettingsEntry = InfoByKind[entryKind].RunSettingsEntryName; - string runSettings = $""" - - - - - - <{runSettingsEntry}>{timeoutValue} - - -"""; - - // if assertAttributePrecedence is set we will use CodeWithOneSecTimeoutAssetPath - timeoutValue = assertAttributePrecedence ? 1000 : timeoutValue; - - TestHost testHost = assertAttributePrecedence - ? TestHost.LocateFrom(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm) - : TestHost.LocateFrom(AssetFixture.CodeWithNoTimeoutAssetPath, TestAssetFixture.CodeWithNoTimeout, tfm); - string runSettingsFilePath = Path.Combine(testHost.DirectoryName, $"{Guid.NewGuid():N}.runsettings"); - File.WriteAllText(runSettingsFilePath, runSettings); - - var stopwatch = Stopwatch.StartNew(); - TestHostResult testHostResult = await testHost.ExecuteAsync($"--settings {runSettingsFilePath}", environmentVariables: new() { { $"TIMEOUT_{InfoByKind[entryKind].EnvVarSuffix}", "1" } }); - stopwatch.Stop(); - - if (assertAttributePrecedence) - { - Assert.IsTrue(stopwatch.Elapsed.TotalSeconds < 25); - } - - testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after {timeoutValue}ms"); - } - - public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) - { - public const string CodeWithOneSecTimeout = nameof(CodeWithOneSecTimeout); - public const string CodeWithSixtySecTimeout = nameof(CodeWithSixtySecTimeout); - public const string CodeWithNoTimeout = nameof(CodeWithNoTimeout); - public const string CooperativeTimeout = nameof(CooperativeTimeout); - - private const string CooperativeTimeoutSourceCode = """ -#file $ProjectName$.csproj - - - - Exe - true - $TargetFrameworks$ - - - - - - - - - - PreserveNewest - - - - - -#file my.runsettings - - - false - - - -#file UnitTest1.cs -using System; -using System.Threading.Tasks; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -[TestClass] -public class TestClass -{ - [Timeout(100, CooperativeCancellation = true)] - [AssemblyInitialize] - public static async Task AssemblyInit(TestContext testContext) - => await DoWork("ASSEMBLYINIT", "AssemblyInit", testContext); - - [Timeout(100, CooperativeCancellation = true)] - [AssemblyCleanup] - public static async Task AssemblyCleanup(TestContext testContext) - => await DoWork("ASSEMBLYCLEANUP", "AssemblyCleanup", testContext); - - [Timeout(100, CooperativeCancellation = true)] - [ClassInitialize] - public static async Task ClassInit(TestContext testContext) - => await DoWork("CLASSINIT", "ClassInit", testContext); - - [Timeout(100, CooperativeCancellation = true)] - [ClassCleanup(ClassCleanupBehavior.EndOfClass)] - public static async Task ClassCleanup(TestContext testContext) - => await DoWork("CLASSCLEANUP", "ClassCleanup", testContext); - - public TestContext TestContext { get; set; } - - [Timeout(100, CooperativeCancellation = true)] - [TestInitialize] - public async Task TestInit() - => await DoWork("TESTINIT", "TestInit", TestContext); - - [Timeout(100, CooperativeCancellation = true)] - [TestCleanup] - public async Task TestCleanup() - => await DoWork("TESTCLEANUP", "TestCleanup", TestContext); - - [TestMethod] - public async Task TestMethod() - { - } - - private static async Task DoWork(string envVarSuffix, string stepName, TestContext testContext) - { - Console.WriteLine($"{stepName} started"); - - if (Environment.GetEnvironmentVariable($"TASKDELAY_{envVarSuffix}") == "1") - { - await Task.Delay(10_000, testContext.CancellationTokenSource.Token); - } - else - { - System.Threading.Thread.Sleep(200); - Console.WriteLine($"{stepName} Thread.Sleep completed"); - if (Environment.GetEnvironmentVariable($"CHECKTOKEN_{envVarSuffix}") == "1") - { - testContext.CancellationTokenSource.Token.ThrowIfCancellationRequested(); - } - - } - - Console.WriteLine($"{stepName} completed"); - } -} -"""; - - private const string SourceCode = """ -#file $ProjectName$.csproj - - - - Exe - true - $TargetFrameworks$ - - - - - - - - - -#file UnitTest1.cs - -using System; -using System.Threading.Tasks; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -public class TestClassBase -{ - $TimeoutAttribute$ - [ClassInitialize(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] - public static async Task ClassInitBase(TestContext testContext) - { - if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_BASE_CLASSINIT") == "1") - { - testContext.CancellationTokenSource.Cancel(); - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSINIT") == "1") - { - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSINIT") == "1") - { - await Task.Delay(10_000, testContext.CancellationTokenSource.Token); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [ClassCleanup(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] - public static async Task ClassCleanupBase() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - -} - -[TestClass] -public class TestClass : TestClassBase -{ - $TimeoutAttribute$ - [AssemblyInitialize] - public static async Task AssemblyInit(TestContext testContext) - { - if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_ASSEMBLYINIT") == "1") - { - testContext.CancellationTokenSource.Cancel(); - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYINIT") == "1") - { - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYINIT") == "1") - { - await Task.Delay(60_000, testContext.CancellationTokenSource.Token); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [AssemblyCleanup] - public static async Task AssemblyCleanupMethod() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [ClassInitialize] - public static async Task ClassInit(TestContext testContext) - { - if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_CLASSINIT") == "1") - { - testContext.CancellationTokenSource.Cancel(); - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSINIT") == "1") - { - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("TIMEOUT_CLASSINIT") == "1") - { - await Task.Delay(60_000, testContext.CancellationTokenSource.Token); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [ClassCleanup] - public static async Task ClassCleanupMethod() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_CLASSCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [TestInitialize] - public async Task TestInit() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTINIT") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [TestCleanup] - public async Task TestCleanupMethod() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - [TestMethod] - public Task Test1() => Task.CompletedTask; -} -"""; - - public string CodeWithOneSecTimeoutAssetPath => GetAssetPath(CodeWithOneSecTimeout); - - public string CodeWithSixtySecTimeoutAssetPath => GetAssetPath(CodeWithSixtySecTimeout); - - public string CodeWithNoTimeoutAssetPath => GetAssetPath(CodeWithNoTimeout); - - public string CooperativeTimeoutAssetPath => GetAssetPath(CooperativeTimeout); - - public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() - { - yield return (CodeWithNoTimeout, CodeWithNoTimeout, - SourceCode - .PatchCodeWithReplace("$TimeoutAttribute$", string.Empty) - .PatchCodeWithReplace("$ProjectName$", CodeWithNoTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - - yield return (CodeWithOneSecTimeout, CodeWithOneSecTimeout, - SourceCode - .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(1000)]") - .PatchCodeWithReplace("$ProjectName$", CodeWithOneSecTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - - yield return (CodeWithSixtySecTimeout, CodeWithSixtySecTimeout, - SourceCode - .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(60000)]") - .PatchCodeWithReplace("$ProjectName$", CodeWithSixtySecTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - - yield return (CooperativeTimeout, CooperativeTimeout, - CooperativeTimeoutSourceCode - .PatchCodeWithReplace("$ProjectName$", CooperativeTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - } - } -} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs index 59a05bfe2f..3e289d95de 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs @@ -10,6 +10,407 @@ namespace MSTest.Acceptance.IntegrationTests; [TestClass] public class TimeoutTests : AcceptanceTestBase { + private static readonly Dictionary InfoByKind = new() + { + ["assemblyInit"] = ("TestClass.AssemblyInit", "Assembly initialize", "ASSEMBLYINIT", "AssemblyInitializeTimeout"), + ["assemblyCleanup"] = ("TestClass.AssemblyCleanupMethod", "Assembly cleanup", "ASSEMBLYCLEANUP", "AssemblyCleanupTimeout"), + ["classInit"] = ("TestClass.ClassInit", "Class initialize", "CLASSINIT", "ClassInitializeTimeout"), + ["baseClassInit"] = ("TestClassBase.ClassInitBase", "Class initialize", "BASE_CLASSINIT", "ClassInitializeTimeout"), + ["classCleanup"] = ("TestClass.ClassCleanupMethod", "Class cleanup", "CLASSCLEANUP", "ClassCleanupTimeout"), + ["baseClassCleanup"] = ("TestClassBase.ClassCleanupBase", "Class cleanup", "BASE_CLASSCLEANUP", "ClassCleanupTimeout"), + ["testInit"] = ("TestClass.TestInit", "Test initialize", "TESTINIT", "TestInitializeTimeout"), + ["testCleanup"] = ("TestClass.TestCleanupMethod", "Test cleanup", "TESTCLEANUP", "TestCleanupTimeout"), + }; + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInit_WhenTestContextCanceled_AssemblyInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, + tfm, "TESTCONTEXT_CANCEL_", "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInit_WhenTimeoutExpires_AssemblyInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, + tfm, "LONG_WAIT_", "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_AssemblyInitializeExits(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "TIMEOUT_", "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInit_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, + "TESTCONTEXT_CANCEL_", "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInit_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "TIMEOUT_", "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitBase_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, + "TESTCONTEXT_CANCEL_", "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitBase_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitBase_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "TIMEOUT_", "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInitialize_WhenTimeoutExpires_FromRunSettings_AssemblyInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInitialize_WhenTimeoutExpires_AssemblyInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanupBase_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "baseClassCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "classCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "assemblyCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyCleanup_WhenTimeoutExpires_FromRunSettings_AssemblyCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestInitialize_WhenTimeoutExpires_TestInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "testInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestInitialize_WhenTimeoutExpires_FromRunSettings_TestInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestInitialize_WhenTimeoutExpires_TestInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestCleanup_WhenTimeoutExpires_TestCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "testCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestCleanup_WhenTimeoutExpires_FromRunSettings_TestCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestCleanup_WhenTimeoutExpires_TestCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_ASSEMBLYINIT"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyInit started"); + testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("AssemblyInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_ASSEMBLYCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyCleanup started"); + testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("AssemblyCleanup Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassInitTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_CLASSINIT"] = "1" }); + + testHostResult.AssertOutputContains("ClassInit started"); + testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("ClassInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("ClassInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_CLASSCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("ClassCleanup started"); + testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("ClassCleanup Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestInitTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_TESTINIT"] = "1" }); + + testHostResult.AssertOutputContains("TestInit started"); + testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("TestInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_TESTCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("TestCleanup started"); + testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_ASSEMBLYINIT"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyInit started"); + testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 1000ms"); + testHostResult.AssertOutputContains("AssemblyInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_ASSEMBLYCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyCleanup started"); + testHostResult.AssertOutputContains("AssemblyCleanup Thread.Sleep completed"); + testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_CLASSINIT"] = "1" }); + + testHostResult.AssertOutputContains("ClassInit started"); + testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 1000ms"); + testHostResult.AssertOutputContains("ClassInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("ClassInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_CLASSCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("ClassCleanup started"); + testHostResult.AssertOutputContains("ClassCleanup Thread.Sleep completed"); + testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 1000ms"); + testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_TESTINIT"] = "1" }); + + testHostResult.AssertOutputContains("TestInit started"); + testHostResult.AssertOutputDoesNotContain("TestInit completed"); + testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_TESTCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("TestCleanup started"); + testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); + testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 1000ms"); + } + + private async Task RunAndAssertTestWasCanceledAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) + { + var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); + testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' was canceled"); + } + + private async Task RunAndAssertTestTimedOutAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) + { + var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); + testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after 1000ms"); + } + + private async Task RunAndAssertWithRunSettingsAsync(string tfm, int timeoutValue, bool assertAttributePrecedence, string entryKind) + { + string runSettingsEntry = InfoByKind[entryKind].RunSettingsEntryName; + string runSettings = $""" + + + + + + <{runSettingsEntry}>{timeoutValue} + + +"""; + + // if assertAttributePrecedence is set we will use CodeWithOneSecTimeoutAssetPath + timeoutValue = assertAttributePrecedence ? 1000 : timeoutValue; + + TestHost testHost = assertAttributePrecedence + ? TestHost.LocateFrom(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm) + : TestHost.LocateFrom(AssetFixture.CodeWithNoTimeoutAssetPath, TestAssetFixture.CodeWithNoTimeout, tfm); + string runSettingsFilePath = Path.Combine(testHost.DirectoryName, $"{Guid.NewGuid():N}.runsettings"); + File.WriteAllText(runSettingsFilePath, runSettings); + + var stopwatch = Stopwatch.StartNew(); + TestHostResult testHostResult = await testHost.ExecuteAsync($"--settings {runSettingsFilePath}", environmentVariables: new() { { $"TIMEOUT_{InfoByKind[entryKind].EnvVarSuffix}", "1" } }); + stopwatch.Stop(); + + if (assertAttributePrecedence) + { + Assert.IsTrue(stopwatch.Elapsed.TotalSeconds < 25); + } + + testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after {timeoutValue}ms"); + } + [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task TimeoutWithInvalidArg_WithoutLetterSuffix_OutputInvalidMessage(string tfm) @@ -66,9 +467,127 @@ public async Task Timeout_WhenTimeoutValueGreaterThanTestDuration_OutputDoesNotC testHostResult.AssertOutputDoesNotContain("Canceling the test session"); } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInCtor_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_CTOR"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInTestInit_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTINIT"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInTestCleanup_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTCLEANUP"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInTestMethod_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TEST"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInCtor_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_CTOR"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestInit_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTINIT"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test initialize method 'TimeoutTest.UnitTest1.TestInit' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestCleanup_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTCLEANUP"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test cleanup method 'TimeoutTest.UnitTest1.TestCleanup' timed out after 1000ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestMethod_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TEST"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) { public const string AssetName = "TimeoutTest"; + public const string CodeWithOneSecTimeout = nameof(CodeWithOneSecTimeout); + public const string CodeWithSixtySecTimeout = nameof(CodeWithSixtySecTimeout); + public const string CodeWithNoTimeout = nameof(CodeWithNoTimeout); + public const string CooperativeTimeout = nameof(CooperativeTimeout); + public const string TestMethodTimeout = nameof(TestMethodTimeout); + public const string CooperativeTestMethodTimeout = nameof(CooperativeTestMethodTimeout); private const string TestCode = """ #file TimeoutTest.csproj @@ -102,16 +621,407 @@ public void TestA() Thread.Sleep(10000); } } +"""; + + private const string CooperativeTimeoutSourceCode = """ +#file $ProjectName$.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + + PreserveNewest + + + + + +#file my.runsettings + + + false + + + +#file UnitTest1.cs +using System; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class TestClass +{ + [Timeout(1000, CooperativeCancellation = true)] + [AssemblyInitialize] + public static async Task AssemblyInit(TestContext testContext) + => await DoWork("ASSEMBLYINIT", "AssemblyInit", testContext); + + [Timeout(1000, CooperativeCancellation = true)] + [AssemblyCleanup] + public static async Task AssemblyCleanup(TestContext testContext) + => await DoWork("ASSEMBLYCLEANUP", "AssemblyCleanup", testContext); + + [Timeout(1000, CooperativeCancellation = true)] + [ClassInitialize] + public static async Task ClassInit(TestContext testContext) + => await DoWork("CLASSINIT", "ClassInit", testContext); + + [Timeout(1000, CooperativeCancellation = true)] + [ClassCleanup(ClassCleanupBehavior.EndOfClass)] + public static async Task ClassCleanup(TestContext testContext) + => await DoWork("CLASSCLEANUP", "ClassCleanup", testContext); + + public TestContext TestContext { get; set; } + + [Timeout(1000, CooperativeCancellation = true)] + [TestInitialize] + public async Task TestInit() + => await DoWork("TESTINIT", "TestInit", TestContext); + + [Timeout(1000, CooperativeCancellation = true)] + [TestCleanup] + public async Task TestCleanup() + => await DoWork("TESTCLEANUP", "TestCleanup", TestContext); + + [TestMethod] + public void TestMethod() + { + } + + private static async Task DoWork(string envVarSuffix, string stepName, TestContext testContext) + { + Console.WriteLine($"{stepName} started"); + + if (Environment.GetEnvironmentVariable($"TASKDELAY_{envVarSuffix}") == "1") + { + await Task.Delay(10_000, testContext.CancellationTokenSource.Token); + } + else + { + // We want to wait more than the timeout value to ensure the timeout is hit + await Task.Delay(2_000); + Console.WriteLine($"{stepName} Thread.Sleep completed"); + if (Environment.GetEnvironmentVariable($"CHECKTOKEN_{envVarSuffix}") == "1") + { + testContext.CancellationTokenSource.Token.ThrowIfCancellationRequested(); + } + + } + + Console.WriteLine($"{stepName} completed"); + } +} +"""; + + private const string SourceCode = """ +#file $ProjectName$.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + +#file UnitTest1.cs + +using System; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +public class TestClassBase +{ + $TimeoutAttribute$ + [ClassInitialize(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] + public static async Task ClassInitBase(TestContext testContext) + { + if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_BASE_CLASSINIT") == "1") + { + testContext.CancellationTokenSource.Cancel(); + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSINIT") == "1") + { + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSINIT") == "1") + { + await Task.Delay(10_000, testContext.CancellationTokenSource.Token); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [ClassCleanup(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] + public static async Task ClassCleanupBase() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + +} + +[TestClass] +public class TestClass : TestClassBase +{ + $TimeoutAttribute$ + [AssemblyInitialize] + public static async Task AssemblyInit(TestContext testContext) + { + if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_ASSEMBLYINIT") == "1") + { + testContext.CancellationTokenSource.Cancel(); + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYINIT") == "1") + { + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYINIT") == "1") + { + await Task.Delay(60_000, testContext.CancellationTokenSource.Token); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [AssemblyCleanup] + public static async Task AssemblyCleanupMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [ClassInitialize] + public static async Task ClassInit(TestContext testContext) + { + if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_CLASSINIT") == "1") + { + testContext.CancellationTokenSource.Cancel(); + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSINIT") == "1") + { + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("TIMEOUT_CLASSINIT") == "1") + { + await Task.Delay(60_000, testContext.CancellationTokenSource.Token); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [ClassCleanup] + public static async Task ClassCleanupMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_CLASSCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [TestInitialize] + public async Task TestInit() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTINIT") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [TestCleanup] + public async Task TestCleanupMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + [TestMethod] + public Task Test1() => Task.CompletedTask; +} +"""; + + private const string TestMethodTimeoutCode = """ +#file $ProjectName$.csproj + + + $TargetFrameworks$ + true + Exe + enable + preview + + + + + + +#file UnitTest1.cs +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +namespace TimeoutTest; +[TestClass] +public class UnitTest1 +{ + private readonly TestContext _testContext; + + public UnitTest1(TestContext testContext) + { + _testContext = testContext; + if (Environment.GetEnvironmentVariable("LONG_WAIT_CTOR") == "1") + { + Task.Delay(10_000, _testContext.CancellationTokenSource.Token).Wait(); + } + } + + [TestInitialize] + public async Task TestInit() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1") + { + await Task.Delay(10_000, _testContext.CancellationTokenSource.Token); + } + } + + [TestCleanup] + public async Task TestCleanup() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1") + { + await Task.Delay(10_000, _testContext.CancellationTokenSource.Token); + } + } + + [TestMethod] + [Timeout(1000$TimeoutExtraArgs$)] + public async Task TestMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TEST") == "1") + { + await Task.Delay(10_000, _testContext.CancellationTokenSource.Token); + } + } +} """; public string NoExtensionTargetAssetPath => GetAssetPath(AssetName); + public string CodeWithOneSecTimeoutAssetPath => GetAssetPath(CodeWithOneSecTimeout); + + public string CodeWithSixtySecTimeoutAssetPath => GetAssetPath(CodeWithSixtySecTimeout); + + public string CodeWithNoTimeoutAssetPath => GetAssetPath(CodeWithNoTimeout); + + public string CooperativeTimeoutAssetPath => GetAssetPath(CooperativeTimeout); + + public string TestMethodTimeoutAssetPath => GetAssetPath(TestMethodTimeout); + + public string CooperativeTestMethodTimeoutAssetPath => GetAssetPath(CooperativeTestMethodTimeout); + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { yield return (AssetName, AssetName, TestCode .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CodeWithNoTimeout, CodeWithNoTimeout, + SourceCode + .PatchCodeWithReplace("$TimeoutAttribute$", string.Empty) + .PatchCodeWithReplace("$ProjectName$", CodeWithNoTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CodeWithOneSecTimeout, CodeWithOneSecTimeout, + SourceCode + .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(1000)]") + .PatchCodeWithReplace("$ProjectName$", CodeWithOneSecTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CodeWithSixtySecTimeout, CodeWithSixtySecTimeout, + SourceCode + .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(60000)]") + .PatchCodeWithReplace("$ProjectName$", CodeWithSixtySecTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CooperativeTimeout, CooperativeTimeout, + CooperativeTimeoutSourceCode + .PatchCodeWithReplace("$ProjectName$", CooperativeTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (TestMethodTimeout, TestMethodTimeout, + TestMethodTimeoutCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$ProjectName$", TestMethodTimeout) + .PatchCodeWithReplace("$TimeoutExtraArgs$", string.Empty) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CooperativeTestMethodTimeout, CooperativeTestMethodTimeout, + TestMethodTimeoutCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$ProjectName$", CooperativeTestMethodTimeout) + .PatchCodeWithReplace("$TimeoutExtraArgs$", ", CooperativeCancellation = true") .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); } } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs index d9b7d08a8e..84fcb95c57 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs @@ -1272,7 +1272,7 @@ public void TestMethodInfoInvokeShouldReturnTestFailureOnTimeout() UTF.TestResult result = method.Invoke(null); Verify(result.Outcome == UTF.UnitTestOutcome.Timeout); - Verify(result.TestFailureException.Message.Contains("exceeded execution timeout period")); + Verify(result.TestFailureException.Message.Equals("Test 'DummyTestMethod' timed out after 1ms", StringComparison.Ordinal)); }); } @@ -1299,7 +1299,7 @@ public void TestMethodInfoInvokeShouldCancelTokenSourceOnTimeout() UTF.TestResult result = method.Invoke(null); Verify(result.Outcome == UTF.UnitTestOutcome.Timeout); - Verify(result.TestFailureException.Message.Contains("exceeded execution timeout period")); + Verify(result.TestFailureException.Message.Equals("Test 'DummyTestMethod' timed out after 1ms", StringComparison.Ordinal)); Verify(_testContextImplementation.CancellationTokenSource.IsCancellationRequested, "Not canceled.."); }); } @@ -1329,7 +1329,7 @@ public void TestMethodInfoInvokeShouldFailOnTokenSourceCancellation() UTF.TestResult result = method.Invoke(null); Verify(result.Outcome == UTF.UnitTestOutcome.Timeout); - Verify(result.TestFailureException.Message.Contains("execution has been aborted")); + Verify(result.TestFailureException.Message.Equals("Test 'DummyTestMethod' was canceled", StringComparison.Ordinal)); Verify(_testContextImplementation.CancellationTokenSource.IsCancellationRequested, "Not canceled.."); }); }