From e5a572b2ebbb0e4947fc462022b542877c841650 Mon Sep 17 00:00:00 2001 From: Mike Curn Date: Tue, 2 Sep 2025 16:43:35 -0700 Subject: [PATCH 1/5] Fix for when an async block throws --- .../TestCaseTests/TestFailureTests.cs | 15 +++++++++++++++ .../TestData/TestBlocks/SingleDependencyBlocks.cs | 15 +++++++++++++++ IntelliTect.TestTools.TestFramework/TestCase.cs | 8 ++++++++ 3 files changed, 38 insertions(+) diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs index 1e9af68..96947e3 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs @@ -1,5 +1,6 @@ using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using Microsoft.Playwright; using System; using System.Threading.Tasks; using Xunit; @@ -41,5 +42,19 @@ public async Task DependencyWithMissingDependencyThrowsOriginalError() Assert.IsType(ex.InnerException); Assert.Contains("oops", ex.InnerException!.Message, StringComparison.InvariantCultureIgnoreCase); } + + [Fact] + public async Task CgiIssue() + { + TestCase tc = new TestBuilder() + .AddAsyncTestBlock() + .Build(); + + var ex = await Assert.ThrowsAsync(() => tc.ExecuteAsync()); + Assert.False(tc.Passed); + Assert.NotNull(ex.InnerException); + Assert.IsType(ex.InnerException); + Assert.Contains("oops", ex.InnerException!.Message, StringComparison.InvariantCultureIgnoreCase); + } } } diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs b/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs index b26c1c8..3ba064f 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs @@ -1,5 +1,7 @@ using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using Microsoft.Playwright; using System; +using System.Threading.Tasks; using Xunit; namespace IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks @@ -70,4 +72,17 @@ public bool Execute(SomeDependency dep) return true; } } + + public class PlaywrightError : TestBlock + { + public async Task Execute() + { + await Task.Delay(1); + throw new InvalidOperationException("Oops"); + //using IPlaywright pw = await Playwright.CreateAsync(); + //IAPIRequestContext context = await pw.APIRequest.NewContextAsync(); + //await context.GetAsync("http://bad.url.com"); + // Playwright exception thrown for ENOTFOUND + } + } } diff --git a/IntelliTect.TestTools.TestFramework/TestCase.cs b/IntelliTect.TestTools.TestFramework/TestCase.cs index 609cc94..1f613c4 100644 --- a/IntelliTect.TestTools.TestFramework/TestCase.cs +++ b/IntelliTect.TestTools.TestFramework/TestCase.cs @@ -422,6 +422,14 @@ private async Task TryRunBlock(Block block, object blockInstance, List FinallyBlockExceptions.Add(ex) ); } + catch (Exception ex) + { + HandleFinallyBlock( + block, + () => TestBlockException = ex, + () => FinallyBlockExceptions.Add(ex) + ); + } if (result) Log?.Debug($"Test block completed successfully."); return result; From dbe880af01467ddd66ae6230460584a18649754e Mon Sep 17 00:00:00 2001 From: Mike Curn Date: Tue, 2 Sep 2025 16:55:24 -0700 Subject: [PATCH 2/5] Fix the build (didn't need Playwright) --- .../TestCaseTests/TestFailureTests.cs | 1 - .../TestData/TestBlocks/SingleDependencyBlocks.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs index 96947e3..e8210d6 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs @@ -1,6 +1,5 @@ using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; -using Microsoft.Playwright; using System; using System.Threading.Tasks; using Xunit; diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs b/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs index 3ba064f..0ffc2e7 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs @@ -1,5 +1,4 @@ using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; -using Microsoft.Playwright; using System; using System.Threading.Tasks; using Xunit; From 97838c2df7f53f1aec9a8a94d21ab8540d2f17e5 Mon Sep 17 00:00:00 2001 From: Mike Curn Date: Wed, 3 Sep 2025 08:24:16 -0700 Subject: [PATCH 3/5] Some cleanup and fixes to deploy.yml --- .github/workflows/deploy.yml | 10 +++++----- .../TestCaseTests/TestFailureTests.cs | 2 +- .../TestBlocks/SingleDependencyBlocks.cs | 6 +----- IntelliTect.TestTools.TestFramework/TestCase.cs | 17 +++++------------ 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9033226..089fdfe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -41,13 +41,13 @@ jobs: run: dotnet test --filter FullyQualifiedName!~ExampleTests --no-build --configuration Release --verbosity normal - name: Pack if: startsWith(github.ref, 'refs/tags/v') - run: dotnet pack -p:PackageVersion=${{ env.nugetVersion }} --configuration Release -o ${{github.workspace}}/IntelliTect.SelenatePack --no-build + run: dotnet pack -p:PackageVersion=${{ env.nugetVersion }} --configuration Release -o ${{github.workspace}}/IntelliTect.TestToolsPack --no-build - name: Upload Artifacts if: startsWith(github.ref, 'refs/tags/v') uses: actions/upload-artifact@v4 with: name: NuGet - path: ${{github.workspace}}/IntelliTect.SelenatePack + path: ${{github.workspace}}/IntelliTect.TestToolsPack deploy: if: startsWith(github.ref, 'refs/tags/v') @@ -55,7 +55,7 @@ jobs: needs: build-and-test environment: name: 'Production' - url: 'https://www.nuget.org/packages/IntelliTect.TestTools.Selenate' + url: 'https://www.nuget.org/packages/IntelliTect.TestTools.TestFramework' steps: - name: Download artifact from build job uses: actions/download-artifact@v4 @@ -65,11 +65,11 @@ jobs: run: | $tagVersion = "${{ github.ref }}".substring(11) echo "::set-output name=TAG_VERSION::$tagVersion" - dotnet nuget push IntelliTect.Selenate.$tagVersion.nupkg --source https://api.nuget.org/v3/index.json -k ${{ secrets.NUGET_API_KEY }} --skip-duplicate + dotnet nuget push IntelliTect.TestTools.TestFramework.$tagVersion.nupkg --source https://api.nuget.org/v3/index.json -k ${{ secrets.NUGET_API_KEY }} --skip-duplicate id: tag-version - name: Upload nupkg to Releases uses: softprops/action-gh-release@v2 with: fail_on_unmatched_files: true generate_release_notes: true - files: IntelliTect.Selenate.${{ steps.tag-version.outputs.TAG_VERSION }}.nupkg + files: IntelliTect.TestTools.TestFramework.${{ steps.tag-version.outputs.TAG_VERSION }}.nupkg diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs index e8210d6..a5aac68 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs @@ -46,7 +46,7 @@ public async Task DependencyWithMissingDependencyThrowsOriginalError() public async Task CgiIssue() { TestCase tc = new TestBuilder() - .AddAsyncTestBlock() + .AddAsyncTestBlock() .Build(); var ex = await Assert.ThrowsAsync(() => tc.ExecuteAsync()); diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs b/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs index 0ffc2e7..c2563d6 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs @@ -72,16 +72,12 @@ public bool Execute(SomeDependency dep) } } - public class PlaywrightError : TestBlock + public class AsyncError : TestBlock { public async Task Execute() { await Task.Delay(1); throw new InvalidOperationException("Oops"); - //using IPlaywright pw = await Playwright.CreateAsync(); - //IAPIRequestContext context = await pw.APIRequest.NewContextAsync(); - //await context.GetAsync("http://bad.url.com"); - // Playwright exception thrown for ENOTFOUND } } } diff --git a/IntelliTect.TestTools.TestFramework/TestCase.cs b/IntelliTect.TestTools.TestFramework/TestCase.cs index 1f613c4..7b42dc8 100644 --- a/IntelliTect.TestTools.TestFramework/TestCase.cs +++ b/IntelliTect.TestTools.TestFramework/TestCase.cs @@ -94,12 +94,9 @@ public async Task ExecuteAsync() bool runSuccess = await TryRunBlock(tb, testBlockInstance, executeArgs); if (!runSuccess) { - if (TestBlockException is null) - { - TestBlockException = new( + TestBlockException ??= new( $"Unknown error occurred while running test block {tb}. " + "Please file an issue: https://github.com/IntelliTect/TestTools.TestFramework/issues"); - } break; } } @@ -122,6 +119,10 @@ public async Task ExecuteAsync() if (TestBlockException is null) { + // Note: This likely needs to be moved up above the finally blocks. + // If a test case "passes" i.e. finishes all of its test blocks, we probably need to know that + // in the finally blocks. + // Need to think about how to handle "passed" state if a finally block fails. Passed = true; Log?.Info("Test case finished successfully."); } @@ -405,14 +406,6 @@ private async Task TryRunBlock(Block block, object blockInstance, List FinallyBlockExceptions.Add(ex.InnerException) ); } - catch (ArgumentException ex) - { - HandleFinallyBlock( - block, - () => TestBlockException = ex, - () => FinallyBlockExceptions.Add(ex) - ); - } catch (TargetParameterCountException ex) { ex.Data.Add("AdditionalInfo", "Test block failed: Mismatched count between Execute method arguments and supplied dependencies."); From c79412c56c23b260397065826b0f890111f3abe5 Mon Sep 17 00:00:00 2001 From: Mike Curn Date: Wed, 3 Sep 2025 08:26:40 -0700 Subject: [PATCH 4/5] Rename test to what it actually does --- .../TestCaseTests/TestFailureTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs index a5aac68..21d27f4 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs @@ -43,7 +43,7 @@ public async Task DependencyWithMissingDependencyThrowsOriginalError() } [Fact] - public async Task CgiIssue() + public async Task AsyncBlockThrowsProperlyCatchAndLogException() { TestCase tc = new TestBuilder() .AddAsyncTestBlock() From f854ec01c31b703271f5e2c371b1ee016aa2970b Mon Sep 17 00:00:00 2001 From: Mike Curn Date: Wed, 17 Sep 2025 11:33:27 -0700 Subject: [PATCH 5/5] Save off variable from Assert.IsType Co-authored-by: Kevin B --- .../TestCaseTests/TestFailureTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs index 21d27f4..80702bf 100644 --- a/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs +++ b/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs @@ -51,9 +51,8 @@ public async Task AsyncBlockThrowsProperlyCatchAndLogException() var ex = await Assert.ThrowsAsync(() => tc.ExecuteAsync()); Assert.False(tc.Passed); - Assert.NotNull(ex.InnerException); - Assert.IsType(ex.InnerException); - Assert.Contains("oops", ex.InnerException!.Message, StringComparison.InvariantCultureIgnoreCase); + var invalidOp = Assert.IsType(ex.InnerException); + Assert.Contains("oops", invalidOp.Message, StringComparison.InvariantCultureIgnoreCase); } } }