From 52585b03d234cec4279f380d06edf0fe72af6455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 24 Dec 2024 17:23:26 +0100 Subject: [PATCH 1/8] Early return when CTS is canceled --- .../MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs index c7f5f4b126..0aea633c46 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs @@ -16,6 +16,13 @@ internal static class FixtureMethodRunner Action action, CancellationTokenSource cancellationTokenSource, TimeoutInfo? timeoutInfo, MethodInfo methodInfo, IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat) { + if (cancellationTokenSource.Token.IsCancellationRequested) + { + return new( + UnitTestOutcome.Timeout, + string.Format(CultureInfo.InvariantCulture, methodCanceledMessageFormat, methodInfo.DeclaringType!.FullName, methodInfo.Name)); + } + // If no timeout is specified, we can run the action directly. This avoids any overhead of creating a task/thread and // ensures that the execution context is preserved (as we run the action on the current thread). if (timeoutInfo is null) From 3da2c4fda8628a4df1561a2d2278915997657b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 24 Dec 2024 17:23:58 +0100 Subject: [PATCH 2/8] Standardize OCE and AggregateException handling --- .../MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs index 0aea633c46..a7b74c95d3 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs @@ -82,7 +82,9 @@ internal static class FixtureMethodRunner action(); return null; } - catch (OperationCanceledException) + catch (Exception ex) when + ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) + || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) { // Ideally we would like to check that the token of the exception matches cancellationTokenSource but TestContext // instances are not well defined so we have to handle the exception entirely. @@ -156,7 +158,7 @@ internal static class FixtureMethodRunner timeout)); } catch (Exception ex) when - (ex is OperationCanceledException + ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) { return new( @@ -220,10 +222,7 @@ internal static class FixtureMethodRunner timeout)); } catch (Exception ex) when - - // This exception occurs when the cancellation happens before the task is actually started. - ((ex is TaskCanceledException tce && tce.CancellationToken == cancellationTokenSource.Token) - || (ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) + ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) { return new( From b0e047588813e124a449fe163af800ad2b4cb6fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 26 Dec 2024 09:45:17 +0100 Subject: [PATCH 3/8] More improvements --- .../Execution/TestMethodInfo.cs | 59 +- .../Extensions/ExceptionExtensions.cs | 4 + .../Helpers/FixtureMethodRunner.cs | 18 +- .../TestContextShouldBeValidAnalyzer.cs | 2 +- .../TestFramework.Extensions/TestContext.cs | 2 +- .../TestFramework.Extensions.csproj | 4 + .../InitializeAndCleanupTimeoutTests.cs | 762 --------------- .../Properties/launchSettings.json | 2 +- .../TimeoutTests.cs | 911 +++++++++++++++++- 9 files changed, 966 insertions(+), 798 deletions(-) delete mode 100644 test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs index 966eb13863..51d57a575a 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs @@ -22,7 +22,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; [Obsolete(Constants.PublicTypeObsoleteMessage)] #endif #endif -public class TestMethodInfo : ITestMethod +public class TestMethodInfo : ITestMethod, IDisposable { /// /// Specifies the timeout when it is not set in a test case. @@ -32,6 +32,7 @@ public class TestMethodInfo : ITestMethod private object? _classInstance; private bool _isTestContextSet; private bool _isTestCleanupInvoked; + private CancellationTokenSource? _timeoutTokenSource; internal TestMethodInfo( MethodInfo testMethod, @@ -482,6 +483,15 @@ private void RunTestCleanupMethod(TestResult result) { try { + // Reset the cancellation token source to avoid cancellation of cleanup methods because of the init or test method cancellation. + TestMethodOptions.TestContext!.Context.CancellationTokenSource = new CancellationTokenSource(); + + // If we are running with a method timeout, we need to cancel the cleanup when the overall timeout expires. If it already expired, nothing to do. + if (_timeoutTokenSource is { IsCancellationRequested: false }) + { + _timeoutTokenSource?.Token.Register(TestMethodOptions.TestContext.Context.CancellationTokenSource.Cancel); + } + // Test cleanups are called in the order of discovery // Current TestClass -> Parent -> Grandparent testCleanupException = testCleanupMethod is not null @@ -782,18 +792,27 @@ private bool SetTestContext(object classInstance, TestResult result) // In most cases, exception will be TargetInvocationException with real exception wrapped // in the InnerException; or user code throws an exception. // It also seems that in rare cases the ex can be null. - Exception actualException = ex.InnerException ?? ex; - string exceptionMessage = actualException.GetFormattedExceptionMessage(); - StackTraceInformation? stackTraceInfo = actualException.GetStackTraceInformation(); + Exception realException = ex.GetRealException(); - string errorMessage = string.Format( - CultureInfo.CurrentCulture, - Resource.UTA_InstanceCreationError, - TestClassName, - exceptionMessage); + if (realException.IsOperationCanceledExceptionFromToken(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) + { + result.Outcome = UTF.UnitTestOutcome.Timeout; + result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Timeout, string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName)); + } + else + { + string exceptionMessage = realException.GetFormattedExceptionMessage(); + StackTraceInformation? stackTraceInfo = realException.GetStackTraceInformation(); - result.Outcome = UTF.UnitTestOutcome.Failed; - result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Failed, errorMessage, stackTraceInfo); + string errorMessage = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_InstanceCreationError, + TestClassName, + exceptionMessage); + + result.Outcome = UTF.UnitTestOutcome.Failed; + result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Failed, errorMessage, stackTraceInfo); + } } return classInstance; @@ -811,12 +830,11 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) if (TestMethodOptions.TimeoutInfo.CooperativeCancellation) { - CancellationTokenSource? timeoutTokenSource = null; try { - timeoutTokenSource = new(TestMethodOptions.TimeoutInfo.Timeout); - timeoutTokenSource.Token.Register(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Cancel); - if (timeoutTokenSource.Token.IsCancellationRequested) + _timeoutTokenSource = new(TestMethodOptions.TimeoutInfo.Timeout); + _timeoutTokenSource.Token.Register(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Cancel); + if (_timeoutTokenSource.Token.IsCancellationRequested) { return new() { @@ -840,7 +858,7 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) Outcome = UTF.UnitTestOutcome.Timeout, TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, - timeoutTokenSource.Token.IsCancellationRequested + _timeoutTokenSource.Token.IsCancellationRequested ? string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName) : string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName)), }; @@ -848,7 +866,8 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) } finally { - timeoutTokenSource?.Dispose(); + _timeoutTokenSource?.Dispose(); + _timeoutTokenSource = null; } } @@ -867,8 +886,7 @@ void ExecuteAsyncAction() } } - CancellationToken cancelToken = TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token; - if (PlatformServiceProvider.Instance.ThreadOperations.Execute(ExecuteAsyncAction, TestMethodOptions.TimeoutInfo.Timeout, cancelToken)) + if (PlatformServiceProvider.Instance.ThreadOperations.Execute(ExecuteAsyncAction, TestMethodOptions.TimeoutInfo.Timeout, TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) { if (failure != null) { @@ -902,4 +920,7 @@ void ExecuteAsyncAction() RunTestCleanupMethod(timeoutResult); return timeoutResult; } + + void IDisposable.Dispose() + => _timeoutTokenSource?.Dispose(); } diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs index 648fbc41db..dc3eff5037 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs @@ -32,6 +32,10 @@ internal static Exception GetRealException(this Exception exception) return exception; } + internal static bool IsOperationCanceledExceptionFromToken(this Exception ex, CancellationToken cancellationToken) + => (ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) + || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any(oce => oce.CancellationToken == cancellationToken)); + /// /// Get the exception message if available, empty otherwise. /// diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs index a7b74c95d3..a5bd34f201 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs @@ -34,9 +34,7 @@ internal static class FixtureMethodRunner } catch (Exception ex) { - Exception realException = ex.GetRealException(); - - if (realException is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) + if (ex.GetRealException().IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { return new( UnitTestOutcome.Timeout, @@ -82,9 +80,7 @@ internal static class FixtureMethodRunner action(); return null; } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) - || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { // Ideally we would like to check that the token of the exception matches cancellationTokenSource but TestContext // instances are not well defined so we have to handle the exception entirely. @@ -157,9 +153,7 @@ internal static class FixtureMethodRunner methodInfo.Name, timeout)); } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) - || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { return new( UnitTestOutcome.Timeout, @@ -177,7 +171,7 @@ internal static class FixtureMethodRunner } } - [System.Runtime.Versioning.SupportedOSPlatform("windows")] + [SupportedOSPlatform("windows")] private static TestFailedException? RunWithTimeoutAndCancellationWithSTAThread( Action action, CancellationTokenSource cancellationTokenSource, int timeout, MethodInfo methodInfo, IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat) @@ -221,9 +215,7 @@ internal static class FixtureMethodRunner methodInfo.Name, timeout)); } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationTokenSource.Token) - || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any())) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationTokenSource.Token)) { return new( UnitTestOutcome.Timeout, diff --git a/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs b/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs index f0d1e3a689..7ed69424de 100644 --- a/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs +++ b/src/Analyzers/MSTest.Analyzers/TestContextShouldBeValidAnalyzer.cs @@ -124,7 +124,7 @@ private static void CollectTestContextFieldsAssignedInConstructor( } if (operation is ISimpleAssignmentOperation assignmentOperation && - assignmentOperation.Target is IMemberReferenceOperation { Member: IFieldSymbol { } candidateField } targetMemberReference && + assignmentOperation.Target is IMemberReferenceOperation { Member: IFieldSymbol { } candidateField } && assignmentOperation.Value is IParameterReferenceOperation parameterReference && SymbolEqualityComparer.Default.Equals(parameterReference.Parameter, testContextParameter)) { diff --git a/src/TestFramework/TestFramework.Extensions/TestContext.cs b/src/TestFramework/TestFramework.Extensions/TestContext.cs index 418a7007e0..feb96117db 100644 --- a/src/TestFramework/TestFramework.Extensions/TestContext.cs +++ b/src/TestFramework/TestFramework.Extensions/TestContext.cs @@ -48,7 +48,7 @@ public abstract class TestContext /// /// Gets or sets the cancellation token source. This token source is canceled when test times out. Also when explicitly canceled the test will be aborted. /// - public virtual CancellationTokenSource CancellationTokenSource { get; protected set; } = new(); + public virtual CancellationTokenSource CancellationTokenSource { get; protected internal set; } = new(); public object?[]? TestData { get; protected set; } diff --git a/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj b/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj index a66f58bbf3..7aadfd3436 100644 --- a/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj +++ b/src/TestFramework/TestFramework.Extensions/TestFramework.Extensions.csproj @@ -81,6 +81,10 @@ + + + + PreserveNewest diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs deleted file mode 100644 index feccbd7bbd..0000000000 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InitializeAndCleanupTimeoutTests.cs +++ /dev/null @@ -1,762 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Microsoft.Testing.Platform.Acceptance.IntegrationTests; -using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; - -namespace MSTest.Acceptance.IntegrationTests; - -[TestClass] -public class InitializeAndCleanupTimeoutTests : AcceptanceTestBase -{ - private static readonly Dictionary InfoByKind = new() - { - ["assemblyInit"] = ("TestClass.AssemblyInit", "Assembly initialize", "ASSEMBLYINIT", "AssemblyInitializeTimeout"), - ["assemblyCleanup"] = ("TestClass.AssemblyCleanupMethod", "Assembly cleanup", "ASSEMBLYCLEANUP", "AssemblyCleanupTimeout"), - ["classInit"] = ("TestClass.ClassInit", "Class initialize", "CLASSINIT", "ClassInitializeTimeout"), - ["baseClassInit"] = ("TestClassBase.ClassInitBase", "Class initialize", "BASE_CLASSINIT", "ClassInitializeTimeout"), - ["classCleanup"] = ("TestClass.ClassCleanupMethod", "Class cleanup", "CLASSCLEANUP", "ClassCleanupTimeout"), - ["baseClassCleanup"] = ("TestClassBase.ClassCleanupBase", "Class cleanup", "BASE_CLASSCLEANUP", "ClassCleanupTimeout"), - ["testInit"] = ("TestClass.TestInit", "Test initialize", "TESTINIT", "TestInitializeTimeout"), - ["testCleanup"] = ("TestClass.TestCleanupMethod", "Test cleanup", "TESTCLEANUP", "TestCleanupTimeout"), - }; - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInit_WhenTestContextCanceled_AssemblyInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, - tfm, "TESTCONTEXT_CANCEL_", "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInit_WhenTimeoutExpires_AssemblyInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, - tfm, "LONG_WAIT_", "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_AssemblyInitializeExits(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "TIMEOUT_", "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInit_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, - "TESTCONTEXT_CANCEL_", "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInit_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "TIMEOUT_", "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitBase_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, - "TESTCONTEXT_CANCEL_", "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitBase_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitBase_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "TIMEOUT_", "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInitialize_WhenTimeoutExpires_FromRunSettings_AssemblyInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyInitialize_WhenTimeoutExpires_AssemblyInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanupBase_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "baseClassCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "classCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task BaseClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "assemblyCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyCleanup_WhenTimeoutExpires_FromRunSettings_AssemblyCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestInitialize_WhenTimeoutExpires_TestInitializeTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "testInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestInitialize_WhenTimeoutExpires_FromRunSettings_TestInitializeIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestInitialize_WhenTimeoutExpires_TestInitializeIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testInit"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestCleanup_WhenTimeoutExpires_TestCleanupTaskIsCanceled(string tfm) - => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, - "LONG_WAIT_", "testCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestCleanup_WhenTimeoutExpires_FromRunSettings_TestCleanupIsCanceled(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task TestCleanup_WhenTimeoutExpires_TestCleanupIsCanceled_AttributeTakesPrecedence(string tfm) - => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testCleanup"); - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_ASSEMBLYINIT"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyInit started"); - testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("AssemblyInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_ASSEMBLYCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyCleanup started"); - testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("AssemblyCleanup Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassInitTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_CLASSINIT"] = "1" }); - - testHostResult.AssertOutputContains("ClassInit started"); - testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("ClassInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("ClassInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_CLASSCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("ClassCleanup started"); - testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("ClassCleanup Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestInitTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_TESTINIT"] = "1" }); - - testHostResult.AssertOutputContains("TestInit started"); - testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("TestInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_TESTCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("TestCleanup started"); - // TODO: We would expect to have the following line but that's not the case - // testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); - testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' was canceled"); - testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); - } - - [TestMethod] - [Ignore("Move to a different project")] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestMethodTimeoutExpires_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["TASKDELAY_TESTMETHOD"] = "1" }); - - testHostResult.AssertOutputContains("TestMethod started"); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); - testHostResult.AssertOutputDoesNotContain("TestMethod completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_ASSEMBLYINIT"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyInit started"); - testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); - testHostResult.AssertOutputContains("AssemblyInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_ASSEMBLYCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("AssemblyCleanup started"); - testHostResult.AssertOutputContains("AssemblyCleanup Thread.Sleep completed"); - testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_CLASSINIT"] = "1" }); - - testHostResult.AssertOutputContains("ClassInit started"); - testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); - testHostResult.AssertOutputContains("ClassInit Thread.Sleep completed"); - testHostResult.AssertOutputDoesNotContain("ClassInit completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_CLASSCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("ClassCleanup started"); - testHostResult.AssertOutputContains("ClassCleanup Thread.Sleep completed"); - testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); - testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_TESTINIT"] = "1" }); - - testHostResult.AssertOutputContains("TestInit started"); - testHostResult.AssertOutputDoesNotContain("TestInit completed"); - testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); - } - - [TestMethod] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_TESTCLEANUP"] = "1" }); - - testHostResult.AssertOutputContains("TestCleanup started"); - testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); - testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); - } - - [TestMethod] - [Ignore("Move to a different project")] - [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] - public async Task CooperativeCancellation_WhenTestMethodTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) - { - var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--settings my.runsettings", - new() { ["CHECKTOKEN_TESTMETHOD"] = "1" }); - - testHostResult.AssertOutputContains("TestMethod started"); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); - } - - private async Task RunAndAssertTestWasCanceledAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) - { - var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); - testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' was canceled"); - } - - private async Task RunAndAssertTestTimedOutAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) - { - var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); - testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after 1000ms"); - } - - private async Task RunAndAssertWithRunSettingsAsync(string tfm, int timeoutValue, bool assertAttributePrecedence, string entryKind) - { - string runSettingsEntry = InfoByKind[entryKind].RunSettingsEntryName; - string runSettings = $""" - - - - - - <{runSettingsEntry}>{timeoutValue} - - -"""; - - // if assertAttributePrecedence is set we will use CodeWithOneSecTimeoutAssetPath - timeoutValue = assertAttributePrecedence ? 1000 : timeoutValue; - - TestHost testHost = assertAttributePrecedence - ? TestHost.LocateFrom(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm) - : TestHost.LocateFrom(AssetFixture.CodeWithNoTimeoutAssetPath, TestAssetFixture.CodeWithNoTimeout, tfm); - string runSettingsFilePath = Path.Combine(testHost.DirectoryName, $"{Guid.NewGuid():N}.runsettings"); - File.WriteAllText(runSettingsFilePath, runSettings); - - var stopwatch = Stopwatch.StartNew(); - TestHostResult testHostResult = await testHost.ExecuteAsync($"--settings {runSettingsFilePath}", environmentVariables: new() { { $"TIMEOUT_{InfoByKind[entryKind].EnvVarSuffix}", "1" } }); - stopwatch.Stop(); - - if (assertAttributePrecedence) - { - Assert.IsTrue(stopwatch.Elapsed.TotalSeconds < 25); - } - - testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after {timeoutValue}ms"); - } - - public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) - { - public const string CodeWithOneSecTimeout = nameof(CodeWithOneSecTimeout); - public const string CodeWithSixtySecTimeout = nameof(CodeWithSixtySecTimeout); - public const string CodeWithNoTimeout = nameof(CodeWithNoTimeout); - public const string CooperativeTimeout = nameof(CooperativeTimeout); - - private const string CooperativeTimeoutSourceCode = """ -#file $ProjectName$.csproj - - - - Exe - true - $TargetFrameworks$ - - - - - - - - - - PreserveNewest - - - - - -#file my.runsettings - - - false - - - -#file UnitTest1.cs -using System; -using System.Threading.Tasks; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -[TestClass] -public class TestClass -{ - [Timeout(100, CooperativeCancellation = true)] - [AssemblyInitialize] - public static async Task AssemblyInit(TestContext testContext) - => await DoWork("ASSEMBLYINIT", "AssemblyInit", testContext); - - [Timeout(100, CooperativeCancellation = true)] - [AssemblyCleanup] - public static async Task AssemblyCleanup(TestContext testContext) - => await DoWork("ASSEMBLYCLEANUP", "AssemblyCleanup", testContext); - - [Timeout(100, CooperativeCancellation = true)] - [ClassInitialize] - public static async Task ClassInit(TestContext testContext) - => await DoWork("CLASSINIT", "ClassInit", testContext); - - [Timeout(100, CooperativeCancellation = true)] - [ClassCleanup(ClassCleanupBehavior.EndOfClass)] - public static async Task ClassCleanup(TestContext testContext) - => await DoWork("CLASSCLEANUP", "ClassCleanup", testContext); - - public TestContext TestContext { get; set; } - - [Timeout(100, CooperativeCancellation = true)] - [TestInitialize] - public async Task TestInit() - => await DoWork("TESTINIT", "TestInit", TestContext); - - [Timeout(100, CooperativeCancellation = true)] - [TestCleanup] - public async Task TestCleanup() - => await DoWork("TESTCLEANUP", "TestCleanup", TestContext); - - [TestMethod] - public async Task TestMethod() - { - } - - private static async Task DoWork(string envVarSuffix, string stepName, TestContext testContext) - { - Console.WriteLine($"{stepName} started"); - - if (Environment.GetEnvironmentVariable($"TASKDELAY_{envVarSuffix}") == "1") - { - await Task.Delay(10_000, testContext.CancellationTokenSource.Token); - } - else - { - System.Threading.Thread.Sleep(200); - Console.WriteLine($"{stepName} Thread.Sleep completed"); - if (Environment.GetEnvironmentVariable($"CHECKTOKEN_{envVarSuffix}") == "1") - { - testContext.CancellationTokenSource.Token.ThrowIfCancellationRequested(); - } - - } - - Console.WriteLine($"{stepName} completed"); - } -} -"""; - - private const string SourceCode = """ -#file $ProjectName$.csproj - - - - Exe - true - $TargetFrameworks$ - - - - - - - - - -#file UnitTest1.cs - -using System; -using System.Threading.Tasks; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -public class TestClassBase -{ - $TimeoutAttribute$ - [ClassInitialize(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] - public static async Task ClassInitBase(TestContext testContext) - { - if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_BASE_CLASSINIT") == "1") - { - testContext.CancellationTokenSource.Cancel(); - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSINIT") == "1") - { - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSINIT") == "1") - { - await Task.Delay(10_000, testContext.CancellationTokenSource.Token); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [ClassCleanup(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] - public static async Task ClassCleanupBase() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - -} - -[TestClass] -public class TestClass : TestClassBase -{ - $TimeoutAttribute$ - [AssemblyInitialize] - public static async Task AssemblyInit(TestContext testContext) - { - if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_ASSEMBLYINIT") == "1") - { - testContext.CancellationTokenSource.Cancel(); - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYINIT") == "1") - { - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYINIT") == "1") - { - await Task.Delay(60_000, testContext.CancellationTokenSource.Token); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [AssemblyCleanup] - public static async Task AssemblyCleanupMethod() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [ClassInitialize] - public static async Task ClassInit(TestContext testContext) - { - if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_CLASSINIT") == "1") - { - testContext.CancellationTokenSource.Cancel(); - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSINIT") == "1") - { - await Task.Delay(10_000); - } - else if (Environment.GetEnvironmentVariable("TIMEOUT_CLASSINIT") == "1") - { - await Task.Delay(60_000, testContext.CancellationTokenSource.Token); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [ClassCleanup] - public static async Task ClassCleanupMethod() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_CLASSCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [TestInitialize] - public async Task TestInit() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTINIT") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - $TimeoutAttribute$ - [TestCleanup] - public async Task TestCleanupMethod() - { - if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTCLEANUP") == "1") - { - await Task.Delay(10_000); - } - else - { - await Task.CompletedTask; - } - } - - [TestMethod] - public Task Test1() => Task.CompletedTask; -} -"""; - - public string CodeWithOneSecTimeoutAssetPath => GetAssetPath(CodeWithOneSecTimeout); - - public string CodeWithSixtySecTimeoutAssetPath => GetAssetPath(CodeWithSixtySecTimeout); - - public string CodeWithNoTimeoutAssetPath => GetAssetPath(CodeWithNoTimeout); - - public string CooperativeTimeoutAssetPath => GetAssetPath(CooperativeTimeout); - - public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() - { - yield return (CodeWithNoTimeout, CodeWithNoTimeout, - SourceCode - .PatchCodeWithReplace("$TimeoutAttribute$", string.Empty) - .PatchCodeWithReplace("$ProjectName$", CodeWithNoTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - - yield return (CodeWithOneSecTimeout, CodeWithOneSecTimeout, - SourceCode - .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(1000)]") - .PatchCodeWithReplace("$ProjectName$", CodeWithOneSecTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - - yield return (CodeWithSixtySecTimeout, CodeWithSixtySecTimeout, - SourceCode - .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(60000)]") - .PatchCodeWithReplace("$ProjectName$", CodeWithSixtySecTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - - yield return (CooperativeTimeout, CooperativeTimeout, - CooperativeTimeoutSourceCode - .PatchCodeWithReplace("$ProjectName$", CooperativeTimeout) - .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - } - } -} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json index cdc273e704..569b9e378e 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "MSTest.Acceptance.IntegrationTests": { "commandName": "Project", - "commandLineArgs": "" + //"commandLineArgs": "--filter ClassName~TimeoutTests" } } } diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs index 59a05bfe2f..049114344c 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs @@ -10,6 +10,407 @@ namespace MSTest.Acceptance.IntegrationTests; [TestClass] public class TimeoutTests : AcceptanceTestBase { + private static readonly Dictionary InfoByKind = new() + { + ["assemblyInit"] = ("TestClass.AssemblyInit", "Assembly initialize", "ASSEMBLYINIT", "AssemblyInitializeTimeout"), + ["assemblyCleanup"] = ("TestClass.AssemblyCleanupMethod", "Assembly cleanup", "ASSEMBLYCLEANUP", "AssemblyCleanupTimeout"), + ["classInit"] = ("TestClass.ClassInit", "Class initialize", "CLASSINIT", "ClassInitializeTimeout"), + ["baseClassInit"] = ("TestClassBase.ClassInitBase", "Class initialize", "BASE_CLASSINIT", "ClassInitializeTimeout"), + ["classCleanup"] = ("TestClass.ClassCleanupMethod", "Class cleanup", "CLASSCLEANUP", "ClassCleanupTimeout"), + ["baseClassCleanup"] = ("TestClassBase.ClassCleanupBase", "Class cleanup", "BASE_CLASSCLEANUP", "ClassCleanupTimeout"), + ["testInit"] = ("TestClass.TestInit", "Test initialize", "TESTINIT", "TestInitializeTimeout"), + ["testCleanup"] = ("TestClass.TestCleanupMethod", "Test cleanup", "TESTCLEANUP", "TestCleanupTimeout"), + }; + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInit_WhenTestContextCanceled_AssemblyInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, + tfm, "TESTCONTEXT_CANCEL_", "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInit_WhenTimeoutExpires_AssemblyInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, + tfm, "LONG_WAIT_", "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_AssemblyInitializeExits(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "TIMEOUT_", "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInit_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, + "TESTCONTEXT_CANCEL_", "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInit_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInit_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "TIMEOUT_", "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitBase_WhenTestContextCanceled_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestWasCanceledAsync(AssetFixture.CodeWithSixtySecTimeoutAssetPath, TestAssetFixture.CodeWithSixtySecTimeout, tfm, + "TESTCONTEXT_CANCEL_", "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitBase_WhenTimeoutExpires_ClassInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitBase_WhenTimeoutExpiresAndTestContextTokenIsUsed_ClassInitializeExits(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "TIMEOUT_", "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInitialize_WhenTimeoutExpires_FromRunSettings_AssemblyInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassInitialize_WhenTimeoutExpires_FromRunSettings_ClassInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyInitialize_WhenTimeoutExpires_AssemblyInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassInitialize_WhenTimeoutExpires_ClassInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanupBase_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "baseClassCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "classCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "classCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassCleanup_WhenTimeoutExpires_FromRunSettings_ClassCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "baseClassCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "classCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task BaseClassCleanup_WhenTimeoutExpires_ClassCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "baseClassCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "assemblyCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyCleanup_WhenTimeoutExpires_FromRunSettings_AssemblyCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "assemblyCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task AssemblyCleanup_WhenTimeoutExpires_AssemblyCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "assemblyCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestInitialize_WhenTimeoutExpires_TestInitializeTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "testInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestInitialize_WhenTimeoutExpires_FromRunSettings_TestInitializeIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestInitialize_WhenTimeoutExpires_TestInitializeIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testInit"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestCleanup_WhenTimeoutExpires_TestCleanupTaskIsCanceled(string tfm) + => await RunAndAssertTestTimedOutAsync(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm, + "LONG_WAIT_", "testCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestCleanup_WhenTimeoutExpires_FromRunSettings_TestCleanupIsCanceled(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 300, false, "testCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TestCleanup_WhenTimeoutExpires_TestCleanupIsCanceled_AttributeTakesPrecedence(string tfm) + => await RunAndAssertWithRunSettingsAsync(tfm, 25000, true, "testCleanup"); + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_ASSEMBLYINIT"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyInit started"); + testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("AssemblyInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_ASSEMBLYCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyCleanup started"); + testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("AssemblyCleanup Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassInitTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_CLASSINIT"] = "1" }); + + testHostResult.AssertOutputContains("ClassInit started"); + testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("ClassInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("ClassInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_CLASSCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("ClassCleanup started"); + testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("ClassCleanup Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestInitTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_TESTINIT"] = "1" }); + + testHostResult.AssertOutputContains("TestInit started"); + testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("TestInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpires_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["TASKDELAY_TESTCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("TestCleanup started"); + testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_ASSEMBLYINIT"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyInit started"); + testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); + testHostResult.AssertOutputContains("AssemblyInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_ASSEMBLYCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("AssemblyCleanup started"); + testHostResult.AssertOutputContains("AssemblyCleanup Thread.Sleep completed"); + testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_CLASSINIT"] = "1" }); + + testHostResult.AssertOutputContains("ClassInit started"); + testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); + testHostResult.AssertOutputContains("ClassInit Thread.Sleep completed"); + testHostResult.AssertOutputDoesNotContain("ClassInit completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_CLASSCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("ClassCleanup started"); + testHostResult.AssertOutputContains("ClassCleanup Thread.Sleep completed"); + testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); + testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_TESTINIT"] = "1" }); + + testHostResult.AssertOutputContains("TestInit started"); + testHostResult.AssertOutputDoesNotContain("TestInit completed"); + testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTimeoutAssetPath, TestAssetFixture.CooperativeTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--settings my.runsettings", + new() { ["CHECKTOKEN_TESTCLEANUP"] = "1" }); + + testHostResult.AssertOutputContains("TestCleanup started"); + testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); + testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); + } + + private async Task RunAndAssertTestWasCanceledAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) + { + var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); + testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' was canceled"); + } + + private async Task RunAndAssertTestTimedOutAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) + { + var testHost = TestHost.LocateFrom(rootFolder, assetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() { { envVarPrefix + InfoByKind[entryKind].EnvVarSuffix, "1" } }); + testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after 1000ms"); + } + + private async Task RunAndAssertWithRunSettingsAsync(string tfm, int timeoutValue, bool assertAttributePrecedence, string entryKind) + { + string runSettingsEntry = InfoByKind[entryKind].RunSettingsEntryName; + string runSettings = $""" + + + + + + <{runSettingsEntry}>{timeoutValue} + + +"""; + + // if assertAttributePrecedence is set we will use CodeWithOneSecTimeoutAssetPath + timeoutValue = assertAttributePrecedence ? 1000 : timeoutValue; + + TestHost testHost = assertAttributePrecedence + ? TestHost.LocateFrom(AssetFixture.CodeWithOneSecTimeoutAssetPath, TestAssetFixture.CodeWithOneSecTimeout, tfm) + : TestHost.LocateFrom(AssetFixture.CodeWithNoTimeoutAssetPath, TestAssetFixture.CodeWithNoTimeout, tfm); + string runSettingsFilePath = Path.Combine(testHost.DirectoryName, $"{Guid.NewGuid():N}.runsettings"); + File.WriteAllText(runSettingsFilePath, runSettings); + + var stopwatch = Stopwatch.StartNew(); + TestHostResult testHostResult = await testHost.ExecuteAsync($"--settings {runSettingsFilePath}", environmentVariables: new() { { $"TIMEOUT_{InfoByKind[entryKind].EnvVarSuffix}", "1" } }); + stopwatch.Stop(); + + if (assertAttributePrecedence) + { + Assert.IsTrue(stopwatch.Elapsed.TotalSeconds < 25); + } + + testHostResult.AssertOutputContains($"{InfoByKind[entryKind].Prefix} method '{InfoByKind[entryKind].MethodFullName}' timed out after {timeoutValue}ms"); + } + [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task TimeoutWithInvalidArg_WithoutLetterSuffix_OutputInvalidMessage(string tfm) @@ -66,9 +467,127 @@ public async Task Timeout_WhenTimeoutValueGreaterThanTestDuration_OutputDoesNotC testHostResult.AssertOutputDoesNotContain("Canceling the test session"); } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInCtor_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_CTOR"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInTestInit_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTINIT"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInTestCleanup_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTCLEANUP"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task Timeout_WhenMethodTimeoutAndWaitInTestMethod_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TestMethodTimeoutAssetPath, TestAssetFixture.TestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TEST"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInCtor_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_CTOR"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestInit_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTINIT"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test initialize method 'TimeoutTest.UnitTest1.TestInit' was canceled"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestCleanup_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TESTCLEANUP"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test cleanup method 'TimeoutTest.UnitTest1.TestCleanup' was canceled"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestMethod_TestGetsCanceled(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.CooperativeTestMethodTimeoutAssetPath, TestAssetFixture.CooperativeTestMethodTimeout, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(environmentVariables: new() + { + ["LONG_WAIT_TEST"] = "1", + }); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) { public const string AssetName = "TimeoutTest"; + public const string CodeWithOneSecTimeout = nameof(CodeWithOneSecTimeout); + public const string CodeWithSixtySecTimeout = nameof(CodeWithSixtySecTimeout); + public const string CodeWithNoTimeout = nameof(CodeWithNoTimeout); + public const string CooperativeTimeout = nameof(CooperativeTimeout); + public const string TestMethodTimeout = nameof(TestMethodTimeout); + public const string CooperativeTestMethodTimeout = nameof(CooperativeTestMethodTimeout); private const string TestCode = """ #file TimeoutTest.csproj @@ -102,16 +621,406 @@ public void TestA() Thread.Sleep(10000); } } +"""; + + private const string CooperativeTimeoutSourceCode = """ +#file $ProjectName$.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + + PreserveNewest + + + + + +#file my.runsettings + + + false + + + +#file UnitTest1.cs +using System; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class TestClass +{ + [Timeout(100, CooperativeCancellation = true)] + [AssemblyInitialize] + public static async Task AssemblyInit(TestContext testContext) + => await DoWork("ASSEMBLYINIT", "AssemblyInit", testContext); + + [Timeout(100, CooperativeCancellation = true)] + [AssemblyCleanup] + public static async Task AssemblyCleanup(TestContext testContext) + => await DoWork("ASSEMBLYCLEANUP", "AssemblyCleanup", testContext); + + [Timeout(100, CooperativeCancellation = true)] + [ClassInitialize] + public static async Task ClassInit(TestContext testContext) + => await DoWork("CLASSINIT", "ClassInit", testContext); + + [Timeout(100, CooperativeCancellation = true)] + [ClassCleanup(ClassCleanupBehavior.EndOfClass)] + public static async Task ClassCleanup(TestContext testContext) + => await DoWork("CLASSCLEANUP", "ClassCleanup", testContext); + + public TestContext TestContext { get; set; } + + [Timeout(100, CooperativeCancellation = true)] + [TestInitialize] + public async Task TestInit() + => await DoWork("TESTINIT", "TestInit", TestContext); + + [Timeout(100, CooperativeCancellation = true)] + [TestCleanup] + public async Task TestCleanup() + => await DoWork("TESTCLEANUP", "TestCleanup", TestContext); + + [TestMethod] + public void TestMethod() + { + } + + private static async Task DoWork(string envVarSuffix, string stepName, TestContext testContext) + { + Console.WriteLine($"{stepName} started"); + + if (Environment.GetEnvironmentVariable($"TASKDELAY_{envVarSuffix}") == "1") + { + await Task.Delay(10_000, testContext.CancellationTokenSource.Token); + } + else + { + System.Threading.Thread.Sleep(200); + Console.WriteLine($"{stepName} Thread.Sleep completed"); + if (Environment.GetEnvironmentVariable($"CHECKTOKEN_{envVarSuffix}") == "1") + { + testContext.CancellationTokenSource.Token.ThrowIfCancellationRequested(); + } + + } + + Console.WriteLine($"{stepName} completed"); + } +} +"""; + + private const string SourceCode = """ +#file $ProjectName$.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + +#file UnitTest1.cs + +using System; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +public class TestClassBase +{ + $TimeoutAttribute$ + [ClassInitialize(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] + public static async Task ClassInitBase(TestContext testContext) + { + if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_BASE_CLASSINIT") == "1") + { + testContext.CancellationTokenSource.Cancel(); + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSINIT") == "1") + { + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSINIT") == "1") + { + await Task.Delay(10_000, testContext.CancellationTokenSource.Token); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [ClassCleanup(inheritanceBehavior: InheritanceBehavior.BeforeEachDerivedClass)] + public static async Task ClassCleanupBase() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_BASE_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_BASE_CLASSCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + +} + +[TestClass] +public class TestClass : TestClassBase +{ + $TimeoutAttribute$ + [AssemblyInitialize] + public static async Task AssemblyInit(TestContext testContext) + { + if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_ASSEMBLYINIT") == "1") + { + testContext.CancellationTokenSource.Cancel(); + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYINIT") == "1") + { + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYINIT") == "1") + { + await Task.Delay(60_000, testContext.CancellationTokenSource.Token); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [AssemblyCleanup] + public static async Task AssemblyCleanupMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_ASSEMBLYCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_ASSEMBLYCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [ClassInitialize] + public static async Task ClassInit(TestContext testContext) + { + if (Environment.GetEnvironmentVariable("TESTCONTEXT_CANCEL_CLASSINIT") == "1") + { + testContext.CancellationTokenSource.Cancel(); + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSINIT") == "1") + { + await Task.Delay(10_000); + } + else if (Environment.GetEnvironmentVariable("TIMEOUT_CLASSINIT") == "1") + { + await Task.Delay(60_000, testContext.CancellationTokenSource.Token); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [ClassCleanup] + public static async Task ClassCleanupMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_CLASSCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_CLASSCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [TestInitialize] + public async Task TestInit() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTINIT") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + $TimeoutAttribute$ + [TestCleanup] + public async Task TestCleanupMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1" || Environment.GetEnvironmentVariable("TIMEOUT_TESTCLEANUP") == "1") + { + await Task.Delay(10_000); + } + else + { + await Task.CompletedTask; + } + } + + [TestMethod] + public Task Test1() => Task.CompletedTask; +} +"""; + + private const string TestMethodTimeoutCode = """ +#file $ProjectName$.csproj + + + $TargetFrameworks$ + true + Exe + enable + preview + + + + + + +#file UnitTest1.cs +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +namespace TimeoutTest; +[TestClass] +public class UnitTest1 +{ + private TestContext _testContext; + + public UnitTest1(TestContext testContext) + { + _testContext = testContext; + if (Environment.GetEnvironmentVariable("LONG_WAIT_CTOR") == "1") + { + Task.Delay(10000, _testContext.CancellationTokenSource.Token).Wait(); + } + } + + [TestInitialize] + public async Task TestInit() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1") + { + await Task.Delay(10000, _testContext.CancellationTokenSource.Token); + } + } + + [TestCleanup] + public async Task TestCleanup() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1") + { + await Task.Delay(10000, _testContext.CancellationTokenSource.Token); + } + } + + [TestMethod] + [Timeout(1000$TimeoutExtraArgs$)] + public async Task TestMethod() + { + if (Environment.GetEnvironmentVariable("LONG_WAIT_TEST") == "1") + { + await Task.Delay(10000, _testContext.CancellationTokenSource.Token); + } + } +} """; public string NoExtensionTargetAssetPath => GetAssetPath(AssetName); + public string CodeWithOneSecTimeoutAssetPath => GetAssetPath(CodeWithOneSecTimeout); + + public string CodeWithSixtySecTimeoutAssetPath => GetAssetPath(CodeWithSixtySecTimeout); + + public string CodeWithNoTimeoutAssetPath => GetAssetPath(CodeWithNoTimeout); + + public string CooperativeTimeoutAssetPath => GetAssetPath(CooperativeTimeout); + + public string TestMethodTimeoutAssetPath => GetAssetPath(TestMethodTimeout); + + public string CooperativeTestMethodTimeoutAssetPath => GetAssetPath(CooperativeTestMethodTimeout); + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { yield return (AssetName, AssetName, TestCode .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CodeWithNoTimeout, CodeWithNoTimeout, + SourceCode + .PatchCodeWithReplace("$TimeoutAttribute$", string.Empty) + .PatchCodeWithReplace("$ProjectName$", CodeWithNoTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CodeWithOneSecTimeout, CodeWithOneSecTimeout, + SourceCode + .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(1000)]") + .PatchCodeWithReplace("$ProjectName$", CodeWithOneSecTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CodeWithSixtySecTimeout, CodeWithSixtySecTimeout, + SourceCode + .PatchCodeWithReplace("$TimeoutAttribute$", "[Timeout(60000)]") + .PatchCodeWithReplace("$ProjectName$", CodeWithSixtySecTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CooperativeTimeout, CooperativeTimeout, + CooperativeTimeoutSourceCode + .PatchCodeWithReplace("$ProjectName$", CooperativeTimeout) + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (TestMethodTimeout, TestMethodTimeout, + TestMethodTimeoutCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$ProjectName$", TestMethodTimeout) + .PatchCodeWithReplace("$TimeoutExtraArgs$", string.Empty) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + yield return (CooperativeTestMethodTimeout, CooperativeTestMethodTimeout, + TestMethodTimeoutCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$ProjectName$", CooperativeTestMethodTimeout) + .PatchCodeWithReplace("$TimeoutExtraArgs$", ", CooperativeCancellation = true") .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); } } From 055d57183d459ea4c19e6a2aaaeae86c5e752ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 26 Dec 2024 17:39:01 +0100 Subject: [PATCH 4/8] Best I can do --- .../Execution/TestMethodInfo.cs | 126 ++++++++++-------- .../Extensions/ExceptionExtensions.cs | 4 - .../Extensions/TestResultExtensions.cs | 4 +- .../Extensions/UnitTestOutcomeExtensions.cs | 13 ++ .../Helpers/FixtureMethodRunner.cs | 19 ++- .../Resources/Resource.Designer.cs | 4 +- .../Resources/Resource.resx | 4 +- .../Resources/xlf/Resource.cs.xlf | 12 +- .../Resources/xlf/Resource.de.xlf | 12 +- .../Resources/xlf/Resource.es.xlf | 12 +- .../Resources/xlf/Resource.fr.xlf | 12 +- .../Resources/xlf/Resource.it.xlf | 12 +- .../Resources/xlf/Resource.ja.xlf | 12 +- .../Resources/xlf/Resource.ko.xlf | 12 +- .../Resources/xlf/Resource.pl.xlf | 12 +- .../Resources/xlf/Resource.pt-BR.xlf | 12 +- .../Resources/xlf/Resource.ru.xlf | 12 +- .../Resources/xlf/Resource.tr.xlf | 12 +- .../Resources/xlf/Resource.zh-Hans.xlf | 12 +- .../Resources/xlf/Resource.zh-Hant.xlf | 12 +- .../Extensions/ExceptionExtensions.cs | 7 +- .../Services/ThreadOperations.cs | 13 +- .../Properties/launchSettings.json | 2 +- .../TimeoutTests.cs | 65 ++++----- 24 files changed, 229 insertions(+), 188 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs index 51d57a575a..aec6cdcd0a 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs @@ -5,6 +5,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; using Microsoft.VisualStudio.TestTools.UnitTesting; using ObjectModelUnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; @@ -22,7 +23,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; [Obsolete(Constants.PublicTypeObsoleteMessage)] #endif #endif -public class TestMethodInfo : ITestMethod, IDisposable +public class TestMethodInfo : ITestMethod { /// /// Specifies the timeout when it is not set in a test case. @@ -32,7 +33,6 @@ public class TestMethodInfo : ITestMethod, IDisposable private object? _classInstance; private bool _isTestContextSet; private bool _isTestCleanupInvoked; - private CancellationTokenSource? _timeoutTokenSource; internal TestMethodInfo( MethodInfo testMethod, @@ -114,7 +114,7 @@ public virtual TestResult Invoke(object?[]? arguments) watch.Start(); try { - result = IsTimeoutSet ? ExecuteInternalWithTimeout(arguments) : ExecuteInternal(arguments); + result = IsTimeoutSet ? ExecuteInternalWithTimeout(arguments) : ExecuteInternal(arguments, null); } finally { @@ -218,7 +218,7 @@ public virtual TestResult Invoke(object?[]? arguments) /// Arguments to be passed to the method. /// The result of the execution. [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] - private TestResult ExecuteInternal(object?[]? arguments) + private TestResult ExecuteInternal(object?[]? arguments, CancellationTokenSource? timeoutTokenSource) { DebugEx.Assert(TestMethod != null, "UnitTestExecuter.DefaultTestMethodInvoke: testMethod = null."); @@ -240,7 +240,7 @@ private TestResult ExecuteInternal(object?[]? arguments) // For any failure after this point, we must run TestCleanup _isTestContextSet = true; - if (RunTestInitializeMethod(_classInstance, result)) + if (RunTestInitializeMethod(_classInstance, result, timeoutTokenSource)) { hasTestInitializePassed = true; if (IsTimeoutSet) @@ -268,12 +268,21 @@ private TestResult ExecuteInternal(object?[]? arguments) // Expected Exception was thrown, so Pass the test result.Outcome = UTF.UnitTestOutcome.Passed; } - else if (realException is OperationCanceledException oce && oce.CancellationToken == TestMethodOptions.TestContext?.Context.CancellationTokenSource.Token) + else if (realException.IsOperationCanceledExceptionFromToken(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) { result.Outcome = UTF.UnitTestOutcome.Timeout; result.TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, - string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName)); + timeoutTokenSource?.Token.IsCancellationRequested == true + ? string.Format( + CultureInfo.InvariantCulture, + Resource.Execution_Test_Timeout, + TestMethodName, + TestMethodOptions.TimeoutInfo.Timeout) + : string.Format( + CultureInfo.InvariantCulture, + Resource.Execution_Test_Cancelled, + TestMethodName)); } else { @@ -322,7 +331,7 @@ private TestResult ExecuteInternal(object?[]? arguments) // Pulling it out so extension writers can abort custom cleanups if need be. Having this in a finally block // does not allow a thread abort exception to be raised within the block but throws one after finally is executed // crashing the process. This was blocking writing an extension for Dynamic Timeout in VSO. - RunTestCleanupMethod(result); + RunTestCleanupMethod(result, timeoutTokenSource); return testRunnerException != null ? throw testRunnerException : result; } @@ -467,7 +476,7 @@ private static TestFailedException HandleMethodException(Exception ex, Exception /// /// Instance of TestResult. [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] - private void RunTestCleanupMethod(TestResult result) + private void RunTestCleanupMethod(TestResult result, CancellationTokenSource? timeoutTokenSource) { DebugEx.Assert(result != null, "result != null"); @@ -487,21 +496,21 @@ private void RunTestCleanupMethod(TestResult result) TestMethodOptions.TestContext!.Context.CancellationTokenSource = new CancellationTokenSource(); // If we are running with a method timeout, we need to cancel the cleanup when the overall timeout expires. If it already expired, nothing to do. - if (_timeoutTokenSource is { IsCancellationRequested: false }) + if (timeoutTokenSource is { IsCancellationRequested: false }) { - _timeoutTokenSource?.Token.Register(TestMethodOptions.TestContext.Context.CancellationTokenSource.Cancel); + timeoutTokenSource?.Token.Register(TestMethodOptions.TestContext.Context.CancellationTokenSource.Cancel); } // Test cleanups are called in the order of discovery // Current TestClass -> Parent -> Grandparent testCleanupException = testCleanupMethod is not null - ? InvokeCleanupMethod(testCleanupMethod, _classInstance, Parent.BaseTestCleanupMethodsQueue.Count) + ? InvokeCleanupMethod(testCleanupMethod, _classInstance, Parent.BaseTestCleanupMethodsQueue.Count, timeoutTokenSource) : null; var baseTestCleanupQueue = new Queue(Parent.BaseTestCleanupMethodsQueue); while (baseTestCleanupQueue.Count > 0 && testCleanupException is null) { testCleanupMethod = baseTestCleanupQueue.Dequeue(); - testCleanupException = InvokeCleanupMethod(testCleanupMethod, _classInstance, baseTestCleanupQueue.Count); + testCleanupException = InvokeCleanupMethod(testCleanupMethod, _classInstance, baseTestCleanupQueue.Count, timeoutTokenSource); } } finally @@ -525,9 +534,9 @@ private void RunTestCleanupMethod(TestResult result) } // If the exception is already a `TestFailedException` we throw it as-is - if (testCleanupException is TestFailedException) + if (testCleanupException is TestFailedException tfe) { - result.Outcome = UTF.UnitTestOutcome.Failed; + result.Outcome = tfe.Outcome.ToAdapterOutcome(); result.TestFailureException = testCleanupException; return; } @@ -603,7 +612,7 @@ private void RunTestCleanupMethod(TestResult result) /// Instance of TestResult. /// True if the TestInitialize method(s) did not throw an exception. [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] - private bool RunTestInitializeMethod(object classInstance, TestResult result) + private bool RunTestInitializeMethod(object classInstance, TestResult result, CancellationTokenSource? timeoutTokenSource) { DebugEx.Assert(classInstance != null, "classInstance != null"); DebugEx.Assert(result != null, "result != null"); @@ -619,7 +628,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) while (baseTestInitializeStack.Count > 0) { testInitializeMethod = baseTestInitializeStack.Pop(); - testInitializeException = testInitializeMethod is not null ? InvokeInitializeMethod(testInitializeMethod, classInstance) : null; + testInitializeException = testInitializeMethod is not null + ? InvokeInitializeMethod(testInitializeMethod, classInstance, timeoutTokenSource) + : null; if (testInitializeException is not null) { break; @@ -629,7 +640,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) if (testInitializeException == null) { testInitializeMethod = Parent.TestInitializeMethod; - testInitializeException = testInitializeMethod is not null ? InvokeInitializeMethod(testInitializeMethod, classInstance) : null; + testInitializeException = testInitializeMethod is not null + ? InvokeInitializeMethod(testInitializeMethod, classInstance, timeoutTokenSource) + : null; } } catch (Exception ex) @@ -644,9 +657,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) } // If the exception is already a `TestFailedException` we throw it as-is - if (testInitializeException is TestFailedException) + if (testInitializeException is TestFailedException tfe) { - result.Outcome = UTF.UnitTestOutcome.Failed; + result.Outcome = tfe.Outcome.ToAdapterOutcome(); result.TestFailureException = testInitializeException; return false; } @@ -665,7 +678,9 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) exceptionMessage); StackTraceInformation? stackTrace = realException.GetStackTraceInformation(); - result.Outcome = realException is AssertInconclusiveException ? UTF.UnitTestOutcome.Inconclusive : UTF.UnitTestOutcome.Failed; + result.Outcome = realException is AssertInconclusiveException + ? UTF.UnitTestOutcome.Inconclusive + : UTF.UnitTestOutcome.Failed; result.TestFailureException = new TestFailedException( result.Outcome.ToUnitTestOutcome(), errorMessage, @@ -675,7 +690,7 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) return false; } - private TestFailedException? InvokeInitializeMethod(MethodInfo methodInfo, object classInstance) + private TestFailedException? InvokeInitializeMethod(MethodInfo methodInfo, object classInstance, CancellationTokenSource? timeoutTokenSource) { TimeoutInfo? timeout = null; if (Parent.TestInitializeMethodTimeoutMilliseconds.TryGetValue(methodInfo, out TimeoutInfo localTimeout)) @@ -690,10 +705,13 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) methodInfo, new InstanceExecutionContextScope(classInstance, Parent.ClassType), Resource.TestInitializeWasCancelled, - Resource.TestInitializeTimedOut); + Resource.TestInitializeTimedOut, + timeoutTokenSource is null + ? null + : (timeoutTokenSource, TestMethodOptions.TimeoutInfo.Timeout)); } - private TestFailedException? InvokeCleanupMethod(MethodInfo methodInfo, object classInstance, int remainingCleanupCount) + private TestFailedException? InvokeCleanupMethod(MethodInfo methodInfo, object classInstance, int remainingCleanupCount, CancellationTokenSource? timeoutTokenSource) { TimeoutInfo? timeout = null; if (Parent.TestCleanupMethodTimeoutMilliseconds.TryGetValue(methodInfo, out TimeoutInfo localTimeout)) @@ -708,7 +726,10 @@ private bool RunTestInitializeMethod(object classInstance, TestResult result) methodInfo, new InstanceExecutionContextScope(classInstance, Parent.ClassType, remainingCleanupCount), Resource.TestCleanupWasCancelled, - Resource.TestCleanupTimedOut); + Resource.TestCleanupTimedOut, + timeoutTokenSource is null + ? null + : (timeoutTokenSource, TestMethodOptions.TimeoutInfo.Timeout)); } /// @@ -797,7 +818,7 @@ private bool SetTestContext(object classInstance, TestResult result) if (realException.IsOperationCanceledExceptionFromToken(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) { result.Outcome = UTF.UnitTestOutcome.Timeout; - result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Timeout, string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName)); + result.TestFailureException = new TestFailedException(ObjectModelUnitTestOutcome.Timeout, string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout)); } else { @@ -830,24 +851,25 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) if (TestMethodOptions.TimeoutInfo.CooperativeCancellation) { + CancellationTokenSource? timeoutTokenSource = null; try { - _timeoutTokenSource = new(TestMethodOptions.TimeoutInfo.Timeout); - _timeoutTokenSource.Token.Register(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Cancel); - if (_timeoutTokenSource.Token.IsCancellationRequested) + timeoutTokenSource = new(TestMethodOptions.TimeoutInfo.Timeout); + timeoutTokenSource.Token.Register(TestMethodOptions.TestContext!.Context.CancellationTokenSource.Cancel); + if (timeoutTokenSource.Token.IsCancellationRequested) { return new() { Outcome = UTF.UnitTestOutcome.Timeout, TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, - string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName)), + string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout)), }; } try { - return ExecuteInternal(arguments); + return ExecuteInternal(arguments, timeoutTokenSource); } catch (OperationCanceledException) { @@ -858,34 +880,22 @@ private TestResult ExecuteInternalWithTimeout(object?[]? arguments) Outcome = UTF.UnitTestOutcome.Timeout, TestFailureException = new TestFailedException( ObjectModelUnitTestOutcome.Timeout, - _timeoutTokenSource.Token.IsCancellationRequested - ? string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName) + timeoutTokenSource.Token.IsCancellationRequested + ? string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout) : string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName)), }; } } finally { - _timeoutTokenSource?.Dispose(); - _timeoutTokenSource = null; + timeoutTokenSource?.Dispose(); + timeoutTokenSource = null; } } TestResult? result = null; Exception? failure = null; - void ExecuteAsyncAction() - { - try - { - result = ExecuteInternal(arguments); - } - catch (Exception ex) - { - failure = ex; - } - } - if (PlatformServiceProvider.Instance.ThreadOperations.Execute(ExecuteAsyncAction, TestMethodOptions.TimeoutInfo.Timeout, TestMethodOptions.TestContext!.Context.CancellationTokenSource.Token)) { if (failure != null) @@ -897,12 +907,12 @@ void ExecuteAsyncAction() // It's possible that some failures happened and that the cleanup wasn't executed, so we need to run it here. // The method already checks if the cleanup was already executed. - RunTestCleanupMethod(result); + RunTestCleanupMethod(result, null); return result; } // Timed out or canceled - string errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName); + string errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, TestMethodName, TestMethodOptions.TimeoutInfo.Timeout); if (TestMethodOptions.TestContext.Context.CancellationTokenSource.IsCancellationRequested) { errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, TestMethodName); @@ -917,10 +927,20 @@ void ExecuteAsyncAction() // We don't know when the cancellation happened so it's possible that the cleanup wasn't executed, so we need to run it here. // The method already checks if the cleanup was already executed. - RunTestCleanupMethod(timeoutResult); + RunTestCleanupMethod(timeoutResult, null); return timeoutResult; - } - void IDisposable.Dispose() - => _timeoutTokenSource?.Dispose(); + // Local functions + void ExecuteAsyncAction() + { + try + { + result = ExecuteInternal(arguments, null); + } + catch (Exception ex) + { + failure = ex; + } + } + } } diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs index dc3eff5037..648fbc41db 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/ExceptionExtensions.cs @@ -32,10 +32,6 @@ internal static Exception GetRealException(this Exception exception) return exception; } - internal static bool IsOperationCanceledExceptionFromToken(this Exception ex, CancellationToken cancellationToken) - => (ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) - || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any(oce => oce.CancellationToken == cancellationToken)); - /// /// Get the exception message if available, empty otherwise. /// diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs index 301984dbb1..ecdd6901b5 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs @@ -36,7 +36,9 @@ internal static UnitTestResult[] ToUnitTestResults(this IReadOnlyCollection UnitTestOutcome.Error, }; + internal static UTF.UnitTestOutcome ToAdapterOutcome(this UnitTestOutcome outcome) + => outcome switch + { + UnitTestOutcome.Failed => UTF.UnitTestOutcome.Failed, + UnitTestOutcome.Inconclusive => UTF.UnitTestOutcome.Inconclusive, + UnitTestOutcome.InProgress => UTF.UnitTestOutcome.InProgress, + UnitTestOutcome.Passed => UTF.UnitTestOutcome.Passed, + UnitTestOutcome.Timeout => UTF.UnitTestOutcome.Timeout, + UnitTestOutcome.NotRunnable => UTF.UnitTestOutcome.NotRunnable, + UnitTestOutcome.NotFound => UTF.UnitTestOutcome.NotFound, + _ => UTF.UnitTestOutcome.Error, + }; + /// /// Returns more important outcome of two. /// diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs index a5bd34f201..79dd00a0aa 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/FixtureMethodRunner.cs @@ -5,6 +5,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; using UnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; @@ -14,7 +15,10 @@ internal static class FixtureMethodRunner { internal static TestFailedException? RunWithTimeoutAndCancellation( Action action, CancellationTokenSource cancellationTokenSource, TimeoutInfo? timeoutInfo, MethodInfo methodInfo, - IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat) + IExecutionContextScope executionContextScope, string methodCanceledMessageFormat, string methodTimedOutMessageFormat, + // When a test method is marked with [Timeout], this timeout is applied from ctor to destructor, so we need to take + // that into account when processing the OCE of the action. + (CancellationTokenSource TokenSource, int Timeout)? testTimeoutInfo = default) { if (cancellationTokenSource.Token.IsCancellationRequested) { @@ -38,7 +42,18 @@ internal static class FixtureMethodRunner { return new( UnitTestOutcome.Timeout, - string.Format(CultureInfo.InvariantCulture, methodCanceledMessageFormat, methodInfo.DeclaringType!.FullName, methodInfo.Name)); + testTimeoutInfo?.TokenSource.Token.IsCancellationRequested == true + ? string.Format( + CultureInfo.InvariantCulture, + methodTimedOutMessageFormat, + methodInfo.DeclaringType!.FullName, + methodInfo.Name, + testTimeoutInfo.Value.Timeout) + : string.Format( + CultureInfo.InvariantCulture, + methodCanceledMessageFormat, + methodInfo.DeclaringType!.FullName, + methodInfo.Name)); } throw; diff --git a/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs index 1614764330..d459e481e5 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs @@ -279,7 +279,7 @@ internal static string ExceptionsThrown { } /// - /// Looks up a localized string similar to Test '{0}' execution has been aborted.. + /// Looks up a localized string similar to Test '{0}' was canceled. /// internal static string Execution_Test_Cancelled { get { @@ -288,7 +288,7 @@ internal static string Execution_Test_Cancelled { } /// - /// Looks up a localized string similar to Test '{0}' exceeded execution timeout period.. + /// Looks up a localized string similar to Test '{0}' timed out after {1}ms. /// internal static string Execution_Test_Timeout { get { diff --git a/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx b/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx index 4a476afa7d..fba272ebdf 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx @@ -118,7 +118,7 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - Test '{0}' exceeded execution timeout period. + Test '{0}' timed out after {1}ms Running tests in any of the provided sources is not supported for the selected platform @@ -308,7 +308,7 @@ Error: {1} Cannot run test method '{0}.{1}': Method has parameters, but does not define any test source. Use '[DataRow]', '[DynamicData]', or a custom 'ITestDataSource' data source to provide test data. - Test '{0}' execution has been aborted. + Test '{0}' was canceled Exception occurred while enumerating IDataSource attribute on "{0}.{1}": {2} diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf index 75dc16e953..3184131b78 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf @@ -67,9 +67,9 @@ byl však přijat tento počet argumentů: {4} s typy {5}. - Test '{0}' exceeded execution timeout period. - Test {0} překročil časový limit spuštění. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Chyba: {1} - Test '{0}' execution has been aborted. - Spouštění testu {0} se přerušilo. - + Test '{0}' was canceled + Spouštění testu {0} se přerušilo. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf index 0e8d214715..f10f5c2787 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf @@ -67,9 +67,9 @@ aber empfing {4} Argument(e) mit den Typen „{5}“. - Test '{0}' exceeded execution timeout period. - Der Test "{0}" hat das Ausführungstimeout überschritten. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Fehler: {1} - Test '{0}' execution has been aborted. - Die Ausführung des Tests "{0}" wurde abgebrochen. - + Test '{0}' was canceled + Die Ausführung des Tests "{0}" wurde abgebrochen. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf index 395a71c5bd..fe8e4275a6 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf @@ -67,9 +67,9 @@ pero recibió {4} argumento(s), con los tipos "{5}". - Test '{0}' exceeded execution timeout period. - La prueba '{0}' superó el tiempo de espera de ejecución. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - Se anuló la ejecución de la prueba "{0}". - + Test '{0}' was canceled + Se anuló la ejecución de la prueba "{0}". + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf index b553cacfa8..a91f23e7cf 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf @@ -67,9 +67,9 @@ mais a reçu {4} argument(s), avec les types « {5} ». - Test '{0}' exceeded execution timeout period. - Le test '{0}' a dépassé le délai d'attente de l'exécution. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Erreur : {1} - Test '{0}' execution has been aborted. - L'exécution du test '{0}' a été abandonnée. - + Test '{0}' was canceled + L'exécution du test '{0}' a été abandonnée. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf index 6c5116a95b..917dfb783d 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf @@ -67,9 +67,9 @@ ma ha ricevuto {4} argomenti, con tipi "{5}". - Test '{0}' exceeded execution timeout period. - È stato superato il periodo di timeout per l'esecuzione del test '{0}'. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Errore: {1} - Test '{0}' execution has been aborted. - L'esecuzione del test '{0}' è stata interrotta. - + Test '{0}' was canceled + L'esecuzione del test '{0}' è stata interrotta. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf index 01bed386e8..85c9c7e0eb 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf @@ -68,9 +68,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - テスト '{0}' は実行タイムアウトを超えました。 - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -419,9 +419,9 @@ Error: {1} - Test '{0}' execution has been aborted. - テスト '{0}' の実行が中止されました。 - + Test '{0}' was canceled + テスト '{0}' の実行が中止されました。 + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf index f5710bb237..12e3c7bc3b 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - '{0}' 테스트가 실행 시간 제한을 초과했습니다. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - 테스트 '{0}' 실행이 중단되었습니다. - + Test '{0}' was canceled + 테스트 '{0}' 실행이 중단되었습니다. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf index 8e18340251..63394f6986 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf @@ -67,9 +67,9 @@ ale liczba odebranych argumentów to {4} z typami „{5}”. - Test '{0}' exceeded execution timeout period. - Test „{0}” przekroczył okres limitu czasu na wykonanie. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Błąd: {1} - Test '{0}' execution has been aborted. - Wykonanie testu „{0}” zostało przerwane. - + Test '{0}' was canceled + Wykonanie testu „{0}” zostało przerwane. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf index d8232e39e2..6982a4d066 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf @@ -67,9 +67,9 @@ mas {4} argumentos recebidos, com tipos '{5}'. - Test '{0}' exceeded execution timeout period. - Teste '{0}' ultrapassou o período de tempo limite de execução. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Erro: {1} - Test '{0}' execution has been aborted. - A execução do teste '{0}' foi anulada. - + Test '{0}' was canceled + A execução do teste '{0}' foi anulada. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf index ad1b9f9366..102c60d3bb 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - Превышено время ожидания выполнения теста "{0}". - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - Выполнение теста "{0}" было прервано. - + Test '{0}' was canceled + Выполнение теста "{0}" было прервано. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf index 6854ff4926..0c266c1565 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf @@ -67,9 +67,9 @@ ancak, '{5}' türüyle {4} argüman aldı. - Test '{0}' exceeded execution timeout period. - '{0}' testi yürütme zaman aşımı süresini aştı. - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Hata: {1} - Test '{0}' execution has been aborted. - '{0}' testinin yürütülmesi iptal edildi. - + Test '{0}' was canceled + '{0}' testinin yürütülmesi iptal edildi. + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf index 9f451c98c1..8d35d39c8f 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - 测试“{0}”的执行超时。 - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - 已中止测试“{0}”的执行。 - + Test '{0}' was canceled + 已中止测试“{0}”的执行。 + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf index 2dce2845f5..d9a3cf3089 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf @@ -67,9 +67,9 @@ but received {4} argument(s), with types '{5}'. - Test '{0}' exceeded execution timeout period. - 測試 '{0}' 超過執行逾時期限。 - + Test '{0}' timed out after {1}ms + Test '{0}' timed out after {1}ms + The type of the generic parameter '{0}' could not be inferred. @@ -418,9 +418,9 @@ Error: {1} - Test '{0}' execution has been aborted. - 測試 '{0}' 執行已中止。 - + Test '{0}' was canceled + 測試 '{0}' 執行已中止。 + Invalid value '{0}' specified for 'ClassCleanupLifecycle'. Supported scopes are {1}. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs b/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs index 37b35a029c..04389c8555 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Extensions/ExceptionExtensions.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !WINDOWS_UWP - namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; /// @@ -32,5 +30,8 @@ internal static string GetExceptionMessage(this Exception? exception) return exceptionString; } + + internal static bool IsOperationCanceledExceptionFromToken(this Exception ex, CancellationToken cancellationToken) + => (ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) + || (ex is AggregateException aggregateEx && aggregateEx.InnerExceptions.OfType().Any(oce => oce.CancellationToken == cancellationToken)); } -#endif diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs b/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs index 06f5b3a7ef..8ae2cd7a3a 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Services/ThreadOperations.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; @@ -43,11 +44,7 @@ private static bool ExecuteWithThreadPool(Action action, int timeout, Cancellati // False means execution timed out. return executionTask.Wait(timeout, cancellationToken); } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) - - // This exception occurs when the cancellation happens before the task is actually started. - || (ex is TaskCanceledException tce && tce.CancellationToken == cancellationToken)) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationToken)) { // Task execution canceled. return false; @@ -81,11 +78,7 @@ private static bool ExecuteWithCustomThread(Action action, int timeout, Cancella // If the execution thread completes before the timeout, the task will return true, otherwise false. return executionTask.Result; } - catch (Exception ex) when - ((ex is OperationCanceledException oce && oce.CancellationToken == cancellationToken) - - // This exception occurs when the cancellation happens before the task is actually started. - || (ex is TaskCanceledException tce && tce.CancellationToken == cancellationToken)) + catch (Exception ex) when (ex.IsOperationCanceledExceptionFromToken(cancellationToken)) { // Task execution canceled. return false; diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json index 569b9e378e..cdc273e704 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "MSTest.Acceptance.IntegrationTests": { "commandName": "Project", - //"commandLineArgs": "--filter ClassName~TimeoutTests" + "commandLineArgs": "" } } } diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs index 049114344c..3e289d95de 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs @@ -196,7 +196,7 @@ public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpires_StepThr new() { ["TASKDELAY_ASSEMBLYINIT"] = "1" }); testHostResult.AssertOutputContains("AssemblyInit started"); - testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); + testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("AssemblyInit Thread.Sleep completed"); testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); } @@ -211,7 +211,7 @@ public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpires_Step new() { ["TASKDELAY_ASSEMBLYCLEANUP"] = "1" }); testHostResult.AssertOutputContains("AssemblyCleanup started"); - testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); + testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("AssemblyCleanup Thread.Sleep completed"); testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); } @@ -226,7 +226,7 @@ public async Task CooperativeCancellation_WhenClassInitTimeoutExpires_StepThrows new() { ["TASKDELAY_CLASSINIT"] = "1" }); testHostResult.AssertOutputContains("ClassInit started"); - testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); + testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("ClassInit Thread.Sleep completed"); testHostResult.AssertOutputDoesNotContain("ClassInit completed"); } @@ -241,7 +241,7 @@ public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpires_StepThr new() { ["TASKDELAY_CLASSCLEANUP"] = "1" }); testHostResult.AssertOutputContains("ClassCleanup started"); - testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); + testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("ClassCleanup Thread.Sleep completed"); testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); } @@ -256,7 +256,7 @@ public async Task CooperativeCancellation_WhenTestInitTimeoutExpires_StepThrows( new() { ["TASKDELAY_TESTINIT"] = "1" }); testHostResult.AssertOutputContains("TestInit started"); - testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); + testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("TestInit completed"); } @@ -270,7 +270,7 @@ public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpires_StepThro new() { ["TASKDELAY_TESTCLEANUP"] = "1" }); testHostResult.AssertOutputContains("TestCleanup started"); - testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); + testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); } @@ -284,7 +284,7 @@ public async Task CooperativeCancellation_WhenAssemblyInitTimeoutExpiresAndUserC new() { ["CHECKTOKEN_ASSEMBLYINIT"] = "1" }); testHostResult.AssertOutputContains("AssemblyInit started"); - testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 100ms"); + testHostResult.AssertOutputContains("Assembly initialize method 'TestClass.AssemblyInit' timed out after 1000ms"); testHostResult.AssertOutputContains("AssemblyInit Thread.Sleep completed"); testHostResult.AssertOutputDoesNotContain("AssemblyInit completed"); } @@ -300,7 +300,7 @@ public async Task CooperativeCancellation_WhenAssemblyCleanupTimeoutExpiresAndUs testHostResult.AssertOutputContains("AssemblyCleanup started"); testHostResult.AssertOutputContains("AssemblyCleanup Thread.Sleep completed"); - testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 100ms"); + testHostResult.AssertOutputContains("Assembly cleanup method 'TestClass.AssemblyCleanup' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("AssemblyCleanup completed"); } @@ -314,7 +314,7 @@ public async Task CooperativeCancellation_WhenClassInitTimeoutExpiresAndUserChec new() { ["CHECKTOKEN_CLASSINIT"] = "1" }); testHostResult.AssertOutputContains("ClassInit started"); - testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 100ms"); + testHostResult.AssertOutputContains("Class initialize method 'TestClass.ClassInit' timed out after 1000ms"); testHostResult.AssertOutputContains("ClassInit Thread.Sleep completed"); testHostResult.AssertOutputDoesNotContain("ClassInit completed"); } @@ -330,7 +330,7 @@ public async Task CooperativeCancellation_WhenClassCleanupTimeoutExpiresAndUserC testHostResult.AssertOutputContains("ClassCleanup started"); testHostResult.AssertOutputContains("ClassCleanup Thread.Sleep completed"); - testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 100ms"); + testHostResult.AssertOutputContains("Class cleanup method 'TestClass.ClassCleanup' timed out after 1000ms"); testHostResult.AssertOutputDoesNotContain("ClassCleanup completed"); } @@ -345,7 +345,7 @@ public async Task CooperativeCancellation_WhenTestInitTimeoutExpiresAndUserCheck testHostResult.AssertOutputContains("TestInit started"); testHostResult.AssertOutputDoesNotContain("TestInit completed"); - testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 100ms"); + testHostResult.AssertOutputContains("Test initialize method 'TestClass.TestInit' timed out after 1000ms"); } [TestMethod] @@ -359,7 +359,7 @@ public async Task CooperativeCancellation_WhenTestCleanupTimeoutExpiresAndUserCh testHostResult.AssertOutputContains("TestCleanup started"); testHostResult.AssertOutputDoesNotContain("TestCleanup completed"); - testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 100ms"); + testHostResult.AssertOutputContains("Test cleanup method 'TestClass.TestCleanup' timed out after 1000ms"); } private async Task RunAndAssertTestWasCanceledAsync(string rootFolder, string assetName, string tfm, string envVarPrefix, string entryKind) @@ -478,7 +478,7 @@ public async Task Timeout_WhenMethodTimeoutAndWaitInCtor_TestGetsCanceled(string }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); } [TestMethod] @@ -492,7 +492,7 @@ public async Task Timeout_WhenMethodTimeoutAndWaitInTestInit_TestGetsCanceled(st }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); } [TestMethod] @@ -506,7 +506,7 @@ public async Task Timeout_WhenMethodTimeoutAndWaitInTestCleanup_TestGetsCanceled }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); } [TestMethod] @@ -520,7 +520,7 @@ public async Task Timeout_WhenMethodTimeoutAndWaitInTestMethod_TestGetsCanceled( }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); } [TestMethod] @@ -534,7 +534,7 @@ public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInCtor_TestGetsCanc }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test 'TestMethod' exceeded execution timeout period."); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); } [TestMethod] @@ -548,7 +548,7 @@ public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestInit_TestGets }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test initialize method 'TimeoutTest.UnitTest1.TestInit' was canceled"); + testHostResult.AssertOutputContains("Test initialize method 'TimeoutTest.UnitTest1.TestInit' timed out after 1000ms"); } [TestMethod] @@ -562,7 +562,7 @@ public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestCleanup_TestG }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test cleanup method 'TimeoutTest.UnitTest1.TestCleanup' was canceled"); + testHostResult.AssertOutputContains("Test cleanup method 'TimeoutTest.UnitTest1.TestCleanup' timed out after 1000ms"); } [TestMethod] @@ -576,7 +576,7 @@ public async Task CooperativeTimeout_WhenMethodTimeoutAndWaitInTestMethod_TestGe }); testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); + testHostResult.AssertOutputContains("Test 'TestMethod' timed out after 1000ms"); } public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) @@ -661,34 +661,34 @@ public void TestA() [TestClass] public class TestClass { - [Timeout(100, CooperativeCancellation = true)] + [Timeout(1000, CooperativeCancellation = true)] [AssemblyInitialize] public static async Task AssemblyInit(TestContext testContext) => await DoWork("ASSEMBLYINIT", "AssemblyInit", testContext); - [Timeout(100, CooperativeCancellation = true)] + [Timeout(1000, CooperativeCancellation = true)] [AssemblyCleanup] public static async Task AssemblyCleanup(TestContext testContext) => await DoWork("ASSEMBLYCLEANUP", "AssemblyCleanup", testContext); - [Timeout(100, CooperativeCancellation = true)] + [Timeout(1000, CooperativeCancellation = true)] [ClassInitialize] public static async Task ClassInit(TestContext testContext) => await DoWork("CLASSINIT", "ClassInit", testContext); - [Timeout(100, CooperativeCancellation = true)] + [Timeout(1000, CooperativeCancellation = true)] [ClassCleanup(ClassCleanupBehavior.EndOfClass)] public static async Task ClassCleanup(TestContext testContext) => await DoWork("CLASSCLEANUP", "ClassCleanup", testContext); public TestContext TestContext { get; set; } - [Timeout(100, CooperativeCancellation = true)] + [Timeout(1000, CooperativeCancellation = true)] [TestInitialize] public async Task TestInit() => await DoWork("TESTINIT", "TestInit", TestContext); - [Timeout(100, CooperativeCancellation = true)] + [Timeout(1000, CooperativeCancellation = true)] [TestCleanup] public async Task TestCleanup() => await DoWork("TESTCLEANUP", "TestCleanup", TestContext); @@ -708,7 +708,8 @@ private static async Task DoWork(string envVarSuffix, string stepName, TestConte } else { - System.Threading.Thread.Sleep(200); + // We want to wait more than the timeout value to ensure the timeout is hit + await Task.Delay(2_000); Console.WriteLine($"{stepName} Thread.Sleep completed"); if (Environment.GetEnvironmentVariable($"CHECKTOKEN_{envVarSuffix}") == "1") { @@ -920,14 +921,14 @@ namespace TimeoutTest; [TestClass] public class UnitTest1 { - private TestContext _testContext; + private readonly TestContext _testContext; public UnitTest1(TestContext testContext) { _testContext = testContext; if (Environment.GetEnvironmentVariable("LONG_WAIT_CTOR") == "1") { - Task.Delay(10000, _testContext.CancellationTokenSource.Token).Wait(); + Task.Delay(10_000, _testContext.CancellationTokenSource.Token).Wait(); } } @@ -936,7 +937,7 @@ public async Task TestInit() { if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTINIT") == "1") { - await Task.Delay(10000, _testContext.CancellationTokenSource.Token); + await Task.Delay(10_000, _testContext.CancellationTokenSource.Token); } } @@ -945,7 +946,7 @@ public async Task TestCleanup() { if (Environment.GetEnvironmentVariable("LONG_WAIT_TESTCLEANUP") == "1") { - await Task.Delay(10000, _testContext.CancellationTokenSource.Token); + await Task.Delay(10_000, _testContext.CancellationTokenSource.Token); } } @@ -955,7 +956,7 @@ public async Task TestMethod() { if (Environment.GetEnvironmentVariable("LONG_WAIT_TEST") == "1") { - await Task.Delay(10000, _testContext.CancellationTokenSource.Token); + await Task.Delay(10_000, _testContext.CancellationTokenSource.Token); } } } From 9e6f33d52268e4c7f8667f3087788fa9d92c4787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 27 Dec 2024 15:41:52 +0100 Subject: [PATCH 5/8] Fix test --- .../MSTest.Acceptance.IntegrationTests/CancellationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs index 36124bf44a..e3ece73a25 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CancellationTests.cs @@ -110,7 +110,7 @@ public async Task WhenCancelingTestContextTokenInTestMethod_MessageIsAsExpected( // Assert testHostResult.AssertExitCodeIs(2); - testHostResult.AssertOutputContains("Test 'TestMethod' execution has been aborted."); + testHostResult.AssertOutputContains("Test 'TestMethod' was canceled"); testHostResult.AssertOutputContains("Failed!"); } From c59ef06647242b9e94407627daad6b5a426d2452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sat, 28 Dec 2024 14:26:47 +0100 Subject: [PATCH 6/8] Update UT --- .../MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs index d9b7d08a8e..218170b403 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs @@ -1272,7 +1272,7 @@ public void TestMethodInfoInvokeShouldReturnTestFailureOnTimeout() UTF.TestResult result = method.Invoke(null); Verify(result.Outcome == UTF.UnitTestOutcome.Timeout); - Verify(result.TestFailureException.Message.Contains("exceeded execution timeout period")); + Verify(result.TestFailureException.Message.Equals("Test 'DummyTestMethod' timed out after 1ms", StringComparison.Ordinal)); }); } From f60c84aa2f729c1cbcfcf0ae56bf1002cf1edb08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sat, 28 Dec 2024 14:31:38 +0100 Subject: [PATCH 7/8] Fix another UT --- .../MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs index 218170b403..2d70753036 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs @@ -1329,7 +1329,7 @@ public void TestMethodInfoInvokeShouldFailOnTokenSourceCancellation() UTF.TestResult result = method.Invoke(null); Verify(result.Outcome == UTF.UnitTestOutcome.Timeout); - Verify(result.TestFailureException.Message.Contains("execution has been aborted")); + Verify(result.TestFailureException.Message.Equals("Test 'DummyTestMethod' was canceled", StringComparison.Ordinal)); Verify(_testContextImplementation.CancellationTokenSource.IsCancellationRequested, "Not canceled.."); }); } From bdf31fc88ea2da2fe1b0bf6da9441fa1f77f37a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sat, 28 Dec 2024 14:56:51 +0100 Subject: [PATCH 8/8] Again --- .../MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs index 2d70753036..84fcb95c57 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs @@ -1299,7 +1299,7 @@ public void TestMethodInfoInvokeShouldCancelTokenSourceOnTimeout() UTF.TestResult result = method.Invoke(null); Verify(result.Outcome == UTF.UnitTestOutcome.Timeout); - Verify(result.TestFailureException.Message.Contains("exceeded execution timeout period")); + Verify(result.TestFailureException.Message.Equals("Test 'DummyTestMethod' timed out after 1ms", StringComparison.Ordinal)); Verify(_testContextImplementation.CancellationTokenSource.IsCancellationRequested, "Not canceled.."); }); }