ci: gate Fabric E2E on explicit cleanup preflight

AB#5628913

## Summary
Separate the existing Fabric cleanup from E2E execution into a named, fail-closed
pipeline task after authentication and build setup. Preserve cleanup reports
across the second SBT invocation and publish staged evidence on failure.

Defer Fabric workspace lookup, store allocation and notebook submission until
test execution. Cache per-suite cleanup and setup outcomes, preserve interrupts,
and retain bounded parallel notebook execution and tracked artifact cleanup.

## Prompting Intent
The engineer asked to make repository-owned test-item cleanup a pre-job task
before Fabric E2E rather than hiding it inside the notebook task. They approved
the explicit pipeline preflight plus lazy suite-level guards. Retain the strict
24-hour ownership policy in the separate cleanup PR, link their current-sprint
ADO task, and keep all review artifacts under reviews/pr-2728/.

## Linked Sources
- Pull request: #2728
- Tracking task: https://dev.azure.com/msdata/A365/_workitems/edit/5628913
- Operator guide: docs/Reference/Developer Setup.md
- Review findings and resolutions: reviews/pr-2728/task-5628913-attempt-2-review-*.md

## Rationale
A separate step exposes cleanup failures without introducing another job or
duplicating setup. Per-suite checks protect direct runs independently of CI.
Cached setup failures prevent later tests from retrying allocations. Laziness
allows test registration without connecting to Fabric while preserving the
existing parallel workload and public suite member types.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: ranadeepsingh

core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/FabricNotebookTests.scala

@@ -12,12 +12,47 @@ import java.util.concurrent.{ExecutorService, Executors, TimeUnit}
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, Future, blocking}
+import scala.util.{Failure, Try}
 import scala.util.control.NonFatal
 
 trait HasFabricNotebookTestConnection extends HasFabricOperationsConnection {
   fabricClientId = Some(FabricTestConstants.INTEGRATION_APP_ID)
   fabricRedirectUri = Some(FabricTestConstants.INTEGRATION_REDIRECT_URI)
-  fabricWorkspaceId = Some(FabricTestConstants.INTEGRATION_WORKSPACE_ID)
+
+  protected def integrationWorkspaceId: String = FabricTestConstants.INTEGRATION_WORKSPACE_ID
+
+  protected def cleanupStaleArtifacts(): Unit = {
+    val dryRun = sys.env.getOrElse("SYNAPSEML_FABRIC_CLEANUP_DRY_RUN", "false")
+    require(Set("true", "false")(dryRun), "SYNAPSEML_FABRIC_CLEANUP_DRY_RUN must be true or false")
+    fabricWorkspaceId = Some(integrationWorkspaceId)
+    fabric.cleanupTestArtifacts(dryRun.toBoolean)
+  }
+
+  protected final def captureFabricSetup[T](setup: => T): Try[T] = {
+    try {
+      Try(setup)
+    } catch {
+      case error: InterruptedException => Failure(error)
+    }
+  }
+
+  protected final def getFabricSetup[T](setup: Try[T]): T = setup match {
+    case Failure(error: InterruptedException) =>
+      Thread.currentThread().interrupt()
+      throw error
+    case _ => setup.get
+  }
+
+  private lazy val preflight = captureFabricSetup(cleanupStaleArtifacts())
+
+  protected final def ensureFabricPreflight(): Unit = getFabricSetup(preflight)
+
+  private lazy val storeSetup = captureFabricSetup {
+    ensureFabricPreflight()
+    createTrackedStore()
+  }
+
+  protected final def preparedStore: String = getFabricSetup(storeSetup)
 
   private val artifactTracker =
     new FabricTestArtifactTracker(artifactId => fabric.deleteArtifact(artifactId))
@@ -28,13 +63,13 @@ trait HasFabricNotebookTestConnection extends HasFabricOperationsConnection {
     artifactTracker.withArtifact(artifactId)(use)
 
   protected def cleanupTrackedArtifacts(): Unit = artifactTracker.cleanup()
+
+  protected def createTrackedStore(): String = trackArtifact(fabric.createStoreArtifact())
 }
 
 class FabricTestCleanup extends TestBase with HasFabricNotebookTestConnection {
   test("Clean up owned Fabric test artifacts older than 24 hours") {
-    val dryRun = sys.env.getOrElse("SYNAPSEML_FABRIC_CLEANUP_DRY_RUN", "false")
-    require(Set("true", "false")(dryRun), "SYNAPSEML_FABRIC_CLEANUP_DRY_RUN must be true or false")
-    fabric.cleanupTestArtifacts(dryRun.toBoolean)
+    ensureFabricPreflight()
   }
 }
 
@@ -63,13 +98,18 @@ class FabricSmokeTests extends TestBase with HasFabricNotebookTestConnection {
     f
   }
 
-  val storeArtifactId: String = trackArtifact(fabric.createStoreArtifact())
+  lazy val storeArtifactId: String = preparedStore
 
   test("OnePlusOne") {
+    ensureFabricPreflight()
+    runSmokeTest(storeArtifactId)
+  }
+
+  protected def runSmokeTest(storeId: String): Unit = {
     val notebookName = fabric.getBlobNameFromFilepath(notebookFile.getPath)
     withTrackedArtifact(fabric.createSJDArtifact(notebookFile.getPath)) { artifactId =>
       val notebookBlobPath = fabric.uploadNotebookToAzure(notebookFile)
-      fabric.updateSJDArtifact(notebookBlobPath, artifactId, storeArtifactId, includePackages = false)
+      fabric.updateSJDArtifact(notebookBlobPath, artifactId, storeId, includePackages = false)
       blocking {
         Thread.sleep(3000) //scalastyle:ignore
       }
@@ -99,55 +139,77 @@ class FabricSmokeTests extends TestBase with HasFabricNotebookTestConnection {
 }
 
 class FabricNotebookTests extends TestBase with HasFabricNotebookTestConnection {
-  SharedNotebookE2ETestUtilities.generateNotebooks()
+  protected def discoverNotebooks(): Array[File] = {
+    SharedNotebookE2ETestUtilities.generateNotebooks()
+    FileUtilities.recursiveListFiles(SharedNotebookE2ETestUtilities.NotebooksDir)
+      .filter(_.getAbsolutePath.endsWith(".py"))
+      .filter(f => FabricNotebookTests.IncludedNotebooks.exists(f.getName.startsWith))
+      .sortBy(_.getAbsolutePath)
+  }
 
-  val selectedPythonFiles: Array[File] = FileUtilities
-    .recursiveListFiles(SharedNotebookE2ETestUtilities.NotebooksDir)
-    .filter(_.getAbsolutePath.endsWith(".py"))
-    .filter(f => FabricNotebookTests.IncludedNotebooks.exists(f.getName.startsWith))
-    .sortBy(_.getAbsolutePath)
+  val selectedPythonFiles: Array[File] = discoverNotebooks()
 
   selectedPythonFiles.foreach(x => println(s"Fabric notebook to be tested: $x"))
   assert(selectedPythonFiles.nonEmpty, "No notebooks found to test")
 
-  val storeArtifactId: String = trackArtifact(fabric.createStoreArtifact())
+  lazy val storeArtifactId: String = preparedStore
 
-  val executorService = Executors.newFixedThreadPool(FabricNotebookTests.MaxConcurrency)
-  implicit val executionContext: ExecutionContext = ExecutionContext.fromExecutor(executorService)
+  @volatile private var executorStarted = false
+  protected def createNotebookExecutor(): ExecutorService =
+    Executors.newFixedThreadPool(FabricNotebookTests.MaxConcurrency)
 
-  // Submit all SJDs in parallel, each Future handles create -> upload -> submit -> monitor
-  val futures: Array[(Future[String], String)] = selectedPythonFiles.map { notebookFile =>
-    val notebookName = fabric.getBlobNameFromFilepath(notebookFile.getPath)
-    val future = Future {
-      withTrackedArtifact(fabric.createSJDArtifact(notebookFile.getPath)) { artifactId =>
-        val notebookBlobPath = fabric.uploadNotebookToAzure(notebookFile)
-        fabric.updateSJDArtifact(notebookBlobPath, artifactId, storeArtifactId)
-        blocking { Thread.sleep(3000) } //scalastyle:ignore
-        val jobInstanceId = fabric.submitJob(artifactId)
-        blocking { Thread.sleep(10000) } //scalastyle:ignore
-        Await.result(
-          fabric.monitorJob(artifactId, jobInstanceId),
-          Duration(fabric.timeoutInMillis.toLong, TimeUnit.MILLISECONDS))
-      }
+  lazy val executorService: ExecutorService = {
+    val executor = createNotebookExecutor()
+    executorStarted = true
+    executor
+  }
+  implicit lazy val executionContext: ExecutionContext = ExecutionContext.fromExecutor(executorService)
+
+  protected def notebookTimeout: Duration =
+    Duration(fabric.timeoutInMillis.toLong, TimeUnit.MILLISECONDS)
+
+  protected def runNotebook(notebookFile: File, storeId: String): String =
+    withTrackedArtifact(fabric.createSJDArtifact(notebookFile.getPath)) { artifactId =>
+      val notebookBlobPath = fabric.uploadNotebookToAzure(notebookFile)
+      fabric.updateSJDArtifact(notebookBlobPath, artifactId, storeId)
+      blocking { Thread.sleep(3000) } //scalastyle:ignore
+      val jobInstanceId = fabric.submitJob(artifactId)
+      blocking { Thread.sleep(10000) } //scalastyle:ignore
+      Await.result(fabric.monitorJob(artifactId, jobInstanceId), notebookTimeout)
+    }
+
+  // Start the existing parallel workload only after the first selected test passes preflight.
+  private lazy val submissions = captureFabricSetup {
+    ensureFabricPreflight()
+    val storeId = storeArtifactId
+    selectedPythonFiles.map { notebookFile =>
+      (Future(runNotebook(notebookFile, storeId)), notebookFile.getName)
     }
-    (future, notebookName)
   }
 
-  futures.foreach { case (future, notebookName) =>
+  lazy val futures: Array[(Future[String], String)] = getFabricSetup(submissions)
+
+  selectedPythonFiles.zipWithIndex.foreach { case (notebookFile, index) =>
+    val notebookName = notebookFile.getName
     test(notebookName) {
+      ensureFabricPreflight()
+      val (future, submittedNotebookName) = futures(index)
       try {
-        Await.result(future, Duration(fabric.timeoutInMillis.toLong, TimeUnit.MILLISECONDS))
+        Await.result(future, notebookTimeout)
       } catch {
-        case t: Throwable =>
-          throw new RuntimeException(s"Job failed for $notebookName", t)
+        case error: InterruptedException =>
+          Thread.currentThread().interrupt()
+          throw error
+        case NonFatal(t) =>
+          throw new RuntimeException(s"Job failed for $submittedNotebookName", t)
       }
     }
   }
 
   override def afterAll(): Unit = {
     try {
       FabricNotebookTests.shutdownAndCleanup(
-        FabricNotebookTests.shutdownExecutor(executorService),
+        if (executorStarted) FabricNotebookTests.shutdownExecutor(executorService),
         cleanupTrackedArtifacts())
     } finally {
       super.afterAll()