Adding the ability to assert on security scan results

Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs

@@ -25,7 +25,7 @@ public Task SecurityScanShouldPass(Browser browser) =>
         ExecuteTestAfterSetupAsync(
             async context =>
             {
-                await context.RunBaselineSecurityScanAsync();
+                await context.RunAndAssertBaselineSecurityScanAsync();
             },
             browser);
 

Lombiq.Tests.UI/SecurityScanning/SecurityScanResult.cs

@@ -0,0 +1,15 @@
+using Microsoft.CodeAnalysis.Sarif;
+
+namespace Lombiq.Tests.UI.SecurityScanning;
+
+public class SecurityScanResult
+{
+    public string ReportsDirectoryPath { get; }
+    public SarifLog SarifLog { get; }
+
+    public SecurityScanResult(string reportsDirectoryPath, SarifLog sarifLog)
+    {
+        ReportsDirectoryPath = reportsDirectoryPath;
+        SarifLog = sarifLog;
+    }
+}

Lombiq.Tests.UI/SecurityScanning/SecurityScanningAssertionException.cs

@@ -0,0 +1,27 @@
+using System;
+
+namespace Lombiq.Tests.UI.Exceptions;
+
+public class SecurityScanningAssertionException : Exception
+{
+    public SecurityScanningAssertionException(Exception innerException)
+        : base(
+            "Asserting the security scan result failed. Check the security scan report in the failure dump for details.",
+            innerException)
+    {
+    }
+
+    public SecurityScanningAssertionException()
+    {
+    }
+
+    public SecurityScanningAssertionException(string message)
+        : base(message)
+    {
+    }
+
+    public SecurityScanningAssertionException(string message, Exception innerException)
+        : base(message, innerException)
+    {
+    }
+}

Lombiq.Tests.UI/SecurityScanning/SecurityScanningConfiguration.cs

@@ -0,0 +1,23 @@
+using Lombiq.Tests.UI.Services;
+using Microsoft.CodeAnalysis.Sarif;
+using Shouldly;
+using System;
+using System.Threading.Tasks;
+
+namespace Lombiq.Tests.UI.SecurityScanning;
+
+public class SecurityScanningConfiguration
+{
+    /// <summary>
+    /// Gets or sets a delegate that may modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// </summary>
+    public Func<UITestContext, object, Task> ZapAutomationFrameworkYamlModifier { get; set; }
+
+    /// <summary>
+    /// Gets or sets a delegate to run assertions on the <see cref="SarifLog"/> when security scanning happens.
+    /// </summary>
+    public Action<UITestContext, SarifLog> AssertSecurityScanResult { get; set; } = AssertSecurityScanHasNoFails;
+
+    public static readonly Action<UITestContext, SarifLog> AssertSecurityScanHasNoFails =
+        (_, sarifLog) => sarifLog.Runs[0].Results.ShouldNotContain(result => result.Kind == ResultKind.Fail);
+}

Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs

@@ -1,5 +1,7 @@
+using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Services;
+using Microsoft.CodeAnalysis.Sarif;
 using System;
 using System.Threading.Tasks;
 
@@ -9,71 +11,162 @@ public static class SecurityScanningUITestContextExtensions
 {
     /// <summary>
     /// Run a Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) security scan against an app with the Baseline
-    /// Automation Framework profile (see <see href="https://www.zaproxy.org/docs/docker/baseline-scan/"/> for the
-    /// official docs on the legacy version of this scan).
+    /// Automation Framework profile and runs assertions on the result (see <see
+    /// href="https://www.zaproxy.org/docs/docker/baseline-scan/"/> for the official docs on the legacy version of this
+    /// scan).
     /// </summary>
     /// <param name="startUri">
     /// The <see cref="Uri"/> under the app where to start the scan from. If not provided, defaults to the current URL.
     /// </param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
     /// </param>
-    public static Task RunBaselineSecurityScanAsync(
+    /// <param name="assertSecurityScanResult">
+    /// A delegate to run assertions on the <see cref="SarifLog"/> one the scan finishes.
+    /// </param>
+    public static Task RunAndAssertBaselineSecurityScanAsync(
         this UITestContext context,
         Uri startUri = null,
-        Func<object, Task> modifyYaml = null) =>
-        context.RunSecurityScanAsync(AutomationFrameworkYamlPaths.BaselineYamlPath, startUri, modifyYaml);
+        Func<object, Task> modifyYaml = null,
+        Action<SarifLog> assertSecurityScanResult = null) =>
+        context.RunAndAssertSecurityScanAsync(
+            AutomationFrameworkYamlPaths.BaselineYamlPath,
+            startUri,
+            modifyYaml,
+            assertSecurityScanResult);
 
     /// <summary>
     /// Run a Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) security scan against an app with the Full Scan
-    /// Automation Framework profile (see <see href="https://www.zaproxy.org/docs/docker/full-scan/"/> for the
-    /// official docs on the legacy version of this scan).
+    /// Automation Framework profile and runs assertions on the result (see <see
+    /// href="https://www.zaproxy.org/docs/docker/full-scan/"/> for the official docs on the legacy version of this
+    /// scan).
     /// </summary>
     /// <param name="startUri">
     /// The <see cref="Uri"/> under the app where to start the scan from. If not provided, defaults to the current URL.
     /// </param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
     /// </param>
-    public static Task RunFullSecurityScanAsync(
+    /// <param name="assertSecurityScanResult">
+    /// A delegate to run assertions on the <see cref="SarifLog"/> one the scan finishes.
+    /// </param>
+    public static Task RunAndAssertFullSecurityScanAsync(
         this UITestContext context,
         Uri startUri = null,
-        Func<object, Task> modifyYaml = null) =>
-        context.RunSecurityScanAsync(AutomationFrameworkYamlPaths.FullScanYamlPath, startUri, modifyYaml);
+        Func<object, Task> modifyYaml = null,
+        Action<SarifLog> assertSecurityScanResult = null) =>
+        context.RunAndAssertSecurityScanAsync(
+            AutomationFrameworkYamlPaths.FullScanYamlPath,
+            startUri,
+            modifyYaml,
+            assertSecurityScanResult);
 
     /// <summary>
     /// Run a Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) security scan against an app with the GraphQL
-    /// Automation Framework profile (see <see href="https://www.zaproxy.org/docs/desktop/addons/graphql-support/"/> for
-    /// the official docs on ZAP's GraphQL support).
+    /// Automation Framework profile and runs assertions on the result (see <see
+    /// href="https://www.zaproxy.org/docs/desktop/addons/graphql-support/"/> for the official docs on ZAP's GraphQL
+    /// support).
     /// </summary>
     /// <param name="startUri">
     /// The <see cref="Uri"/> under the app where to start the scan from. If not provided, defaults to the current URL.
     /// </param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
     /// </param>
-    public static Task RunGraphQLSecurityScanAsync(
+    /// <param name="assertSecurityScanResult">
+    /// A delegate to run assertions on the <see cref="SarifLog"/> one the scan finishes.
+    /// </param>
+    public static Task RunAndAssertGraphQLSecurityScanAsync(
         this UITestContext context,
         Uri startUri = null,
-        Func<object, Task> modifyYaml = null) =>
-        context.RunSecurityScanAsync(AutomationFrameworkYamlPaths.GraphQLYamlPath, startUri, modifyYaml);
+        Func<object, Task> modifyYaml = null,
+        Action<SarifLog> assertSecurityScanResult = null) =>
+        context.RunAndAssertSecurityScanAsync(
+            AutomationFrameworkYamlPaths.GraphQLYamlPath,
+            startUri,
+            modifyYaml,
+            assertSecurityScanResult);
 
     /// <summary>
     /// Run a Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) security scan against an app with the OpenAPI
-    /// Automation Framework profile (see <see href="https://www.zaproxy.org/docs/desktop/addons/openapi-support/"/> for
-    /// the official docs on ZAP's GraphQL support).
+    /// Automation Framework profile and runs assertions on the result (see <see
+    /// href="https://www.zaproxy.org/docs/desktop/addons/openapi-support/"/> for the official docs on ZAP's GraphQL
+    /// support).
     /// </summary>
     /// <param name="startUri">
     /// The <see cref="Uri"/> under the app where to start the scan from. If not provided, defaults to the current URL.
     /// </param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// </param>
+    /// <param name="assertSecurityScanResult">
+    /// A delegate to run assertions on the <see cref="SarifLog"/> one the scan finishes.
     /// </param>
-    public static Task RunOpenApiSecurityScanAsync(
+    public static Task RunAndAssertOpenApiSecurityScanAsync(
         this UITestContext context,
         Uri startUri = null,
-        Func<object, Task> modifyYaml = null) =>
-        context.RunSecurityScanAsync(AutomationFrameworkYamlPaths.BaselineYamlPath, startUri, modifyYaml);
+        Func<object, Task> modifyYaml = null,
+        Action<SarifLog> assertSecurityScanResult = null) =>
+        context.RunAndAssertSecurityScanAsync(
+            AutomationFrameworkYamlPaths.OpenAPIYamlPath,
+            startUri,
+            modifyYaml,
+            assertSecurityScanResult);
+
+    /// <summary>
+    /// Run a Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) security scan against an app and runs assertions on
+    /// the result.
+    /// </summary>
+    /// <param name="automationFrameworkYamlPath">
+    /// File system path to the YAML configuration file of ZAP's Automation Framework. See <see
+    /// href="https://www.zaproxy.org/docs/automate/automation-framework/"/> for details.
+    /// </param>
+    /// <param name="startUri">
+    /// The <see cref="Uri"/> under the app where to start the scan from. If not provided, defaults to the current URL.
+    /// </param>
+    /// <param name="modifyYaml">
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// </param>
+    /// <param name="assertSecurityScanResult">
+    /// A delegate to run assertions on the <see cref="SarifLog"/> one the scan finishes.
+    /// </param>
+    /// <returns>
+    /// A <see cref="SecurityScanResult"/> instance containing the SARIF (<see
+    /// href="https://sarifweb.azurewebsites.net/"/>) report of the scan.
+    /// </returns>
+    public static async Task RunAndAssertSecurityScanAsync(
+        this UITestContext context,
+        string automationFrameworkYamlPath,
+        Uri startUri = null,
+        Func<object, Task> modifyYaml = null,
+        Action<SarifLog> assertSecurityScanResult = null)
+    {
+        var securityScanningConfiguration = context.Configuration.SecurityScanningConfiguration;
+
+        async Task CompositeModifyYaml(object configuration)
+        {
+            if (securityScanningConfiguration.ZapAutomationFrameworkYamlModifier != null)
+            {
+                await securityScanningConfiguration.ZapAutomationFrameworkYamlModifier(context, configuration);
+            }
+
+            if (modifyYaml != null) await modifyYaml(configuration);
+        }
+
+        SecurityScanResult result = null;
+        try
+        {
+            result = await context.RunSecurityScanAsync(automationFrameworkYamlPath, startUri, CompositeModifyYaml);
+
+            if (assertSecurityScanResult != null) assertSecurityScanResult(result.SarifLog);
+            else securityScanningConfiguration.AssertSecurityScanResult(context, result.SarifLog);
+        }
+        catch (Exception ex)
+        {
+            if (result != null) context.AppendDirectoryToFailureDump(result.ReportsDirectoryPath);
+            throw new SecurityScanningAssertionException(ex);
+        }
+    }
 
     /// <summary>
     /// Run a Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) security scan against an app.
@@ -86,9 +179,13 @@ public static Task RunOpenApiSecurityScanAsync(
     /// The <see cref="Uri"/> under the app where to start the scan from. If not provided, defaults to the current URL.
     /// </param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
     /// </param>
-    public static Task RunSecurityScanAsync(
+    /// <returns>
+    /// A <see cref="SecurityScanResult"/> instance containing the SARIF (<see
+    /// href="https://sarifweb.azurewebsites.net/"/>) report of the scan.
+    /// </returns>
+    public static Task<SecurityScanResult> RunSecurityScanAsync(
         this UITestContext context,
         string automationFrameworkYamlPath,
         Uri startUri = null,

Lombiq.Tests.UI/SecurityScanning/ZapManager.cs

@@ -1,7 +1,6 @@
 using CliWrap;
 using Lombiq.HelpfulLibraries.Cli;
 using Lombiq.Tests.UI.Constants;
-using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Services;
 using Microsoft.CodeAnalysis.Sarif;
 using System;
@@ -50,10 +49,13 @@ public sealed class ZapManager : IAsyncDisposable
     /// </param>
     /// <param name="startUri">The <see cref="Uri"/> under the app where to start the scan from.</param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
     /// </param>
-    /// <returns>The SARIF (<see href="https://sarifweb.azurewebsites.net/"/>) report of the scan.</returns>
-    public Task<SarifLog> RunSecurityScanAsync(
+    /// <returns>
+    /// A <see cref="SecurityScanResult"/> instance containing the SARIF (<see
+    /// href="https://sarifweb.azurewebsites.net/"/>) report of the scan.
+    /// </returns>
+    public Task<SecurityScanResult> RunSecurityScanAsync(
         UITestContext context,
         string automationFrameworkYamlPath,
         Uri startUri,
@@ -72,13 +74,17 @@ public Task<SarifLog> RunSecurityScanAsync(
     /// </summary>
     /// <param name="context">The <see cref="UITestContext"/> of the currently executing test.</param>
     /// <param name="automationFrameworkYamlPath">
-    /// File system path to the YAML configuration file of ZAP's Automation Framework. See
-    /// <see href="https://www.zaproxy.org/docs/automate/automation-framework/"/> for details.
+    /// File system path to the YAML configuration file of ZAP's Automation Framework. See <see
+    /// href="https://www.zaproxy.org/docs/automate/automation-framework/"/> for details.
     /// </param>
     /// <param name="modifyYaml">
-    /// A delegate that may optionally modify the deserialized representation of the ZAP Automation Framework YAML.
+    /// A delegate to modify the deserialized representation of the ZAP Automation Framework YAML.
     /// </param>
-    public async Task<SarifLog> RunSecurityScanAsync(
+    /// <returns>
+    /// A <see cref="SecurityScanResult"/> instance containing the SARIF (<see
+    /// href="https://sarifweb.azurewebsites.net/"/>) report of the scan.
+    /// </returns>
+    public async Task<SecurityScanResult> RunSecurityScanAsync(
         UITestContext context,
         string automationFrameworkYamlPath,
         Func<object, Task> modifyYaml = null)
@@ -172,11 +178,7 @@ await _docker
                 "Check the test output for details.");
         }
 
-        context.AppendDirectoryToFailureDump(reportsDirectoryPath);
-
-        throw new Exception();
-        return SarifLog.Load(jsonReports[0]);
-        //log.Runs.First().Results.Any(result => result.Kind == ResultKind.Fail);
+        return new SecurityScanResult(reportsDirectoryPath, SarifLog.Load(jsonReports[0]));
     }
 
     public ValueTask DisposeAsync()

Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs

@@ -1,4 +1,5 @@
 using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.SecurityScanning;
 using Lombiq.Tests.UI.Services.GitHub;
 using OpenQA.Selenium;
 using Shouldly;
@@ -140,6 +141,8 @@ public class OrchardCoreUITestExecutorConfiguration
 
     public HtmlValidationConfiguration HtmlValidationConfiguration { get; set; } = new();
 
+    public SecurityScanningConfiguration SecurityScanningConfiguration { get; set; } = new();
+
     /// <summary>
     /// Gets or sets a value indicating whether the test should verify the Orchard Core logs and the browser logs for
     /// errors after every page load. When enabled and there is an error the test is failed immediately which prevents