quiesce needs to keep track of blueprint ids (#8919)

Author: davepacheco

dev-tools/omdb/src/bin/omdb/nexus/quiesce.rs

@@ -11,7 +11,10 @@ use chrono::TimeDelta;
 use chrono::Utc;
 use clap::Args;
 use clap::Subcommand;
+use nexus_client::types::PendingRecovery;
 use nexus_client::types::QuiesceState;
+use nexus_client::types::QuiesceStatus;
+use nexus_client::types::SagaQuiesceStatus;
 use std::time::Duration;
 
 #[derive(Debug, Args)]
@@ -31,9 +34,9 @@ pub enum QuiesceCommands {
 
 #[derive(Debug, Args)]
 pub struct QuiesceShowArgs {
-    /// Show details about held database connections
+    /// Show stack traces for held database connections
     #[clap(short, long, default_value_t = false)]
-    verbose: bool,
+    stacks: bool,
 }
 
 pub async fn cmd_nexus_quiesce(
@@ -60,7 +63,10 @@ async fn quiesce_show(
         .await
         .context("fetching quiesce state")?
         .into_inner();
-    match quiesce.state {
+
+    let QuiesceStatus { db_claims, sagas, state } = quiesce;
+
+    match state {
         QuiesceState::Undetermined => {
             println!("has not yet determined if it is quiescing");
         }
@@ -145,25 +151,83 @@ async fn quiesce_show(
         }
     }
 
-    println!("sagas running: {}", quiesce.sagas_pending.len());
-    for saga in &quiesce.sagas_pending {
+    let SagaQuiesceStatus {
+        sagas_pending,
+        drained_blueprint_id,
+        first_recovery_complete,
+        new_sagas_allowed,
+        reassignment_blueprint_id,
+        reassignment_generation,
+        reassignment_pending,
+        recovered_blueprint_id,
+        recovered_reassignment_generation,
+        recovery_pending,
+    } = sagas;
+
+    println!("saga quiesce:");
+    println!("    new sagas: {:?}", new_sagas_allowed);
+    println!(
+        "    drained as of blueprint: {}",
+        drained_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    blueprint for last completed recovery pass: {}",
+        recovered_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    blueprint for last reassignment pass: {}",
+        reassignment_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    reassignment generation: {} (pass running: {})",
+        reassignment_generation,
+        if reassignment_pending { "yes" } else { "no" }
+    );
+    println!("    recovered generation: {}", recovered_reassignment_generation);
+    println!(
+        "    recovered at least once successfully: {}",
+        if first_recovery_complete { "yes" } else { "no" },
+    );
+    print!("    recovery pending: ");
+    if let Some(PendingRecovery { generation, blueprint_id }) = recovery_pending
+    {
+        println!(
+            "yes (generation {}, blueprint id {})",
+            generation,
+            blueprint_id.map(|s| s.to_string()).as_deref().unwrap_or("none")
+        );
+    } else {
+        println!("no");
+    }
+
+    println!("    sagas running: {}", sagas_pending.len());
+    for saga in &sagas_pending {
         println!(
-            "    saga {} pending since {} ({})",
+            "        saga {} pending since {} ({})",
             saga.saga_id,
             humantime::format_rfc3339_millis(saga.time_pending.into()),
             saga.saga_name
         );
     }
 
-    println!("database connections held: {}", quiesce.db_claims.len());
-    for claim in &quiesce.db_claims {
+    println!("database connections held: {}", db_claims.len());
+    for claim in &db_claims {
         println!(
             "    claim {} held since {} ({} ago)",
             claim.id,
             claim.held_since,
             format_time_delta(Utc::now() - claim.held_since),
         );
-        if args.verbose {
+        if args.stacks {
             println!("    acquired by:");
             println!("{}", textwrap::indent(&claim.debug, "        "));
         }
@@ -177,7 +241,7 @@ async fn quiesce_start(
     _token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     client.quiesce_start().await.context("quiescing Nexus")?;
-    quiesce_show(client, &QuiesceShowArgs { verbose: false }).await
+    quiesce_show(client, &QuiesceShowArgs { stacks: false }).await
 }
 
 fn format_duration_ms(duration: Duration) -> String {

nexus/reconfigurator/execution/src/lib.rs

@@ -22,6 +22,7 @@ use nexus_types::deployment::execution::{
     StepHandle, StepResult, UpdateEngine,
 };
 use nexus_types::quiesce::SagaQuiesceHandle;
+use nexus_types::quiesce::SagaReassignmentDone;
 use omicron_uuid_kinds::OmicronZoneUuid;
 use slog::info;
 use slog_error_chain::InlineErrorChain;
@@ -662,18 +663,16 @@ fn register_reassign_sagas_step<'a>(
                         match reassigned {
                             Ok(needs_saga_recovery) => (
                                 StepSuccess::new(needs_saga_recovery).build(),
-                                needs_saga_recovery,
+                                SagaReassignmentDone::ReassignedAllAsOf(
+                                    blueprint.id,
+                                    needs_saga_recovery,
+                                ),
+                            ),
+                            Err(error) => (
+                                StepWarning::new(false, error.to_string())
+                                    .build(),
+                                SagaReassignmentDone::Indeterminate,
                             ),
-                            Err(error) => {
-                                // It's possible that we failed after having
-                                // re-assigned sagas in the database.
-                                let maybe_reassigned = true;
-                                (
-                                    StepWarning::new(false, error.to_string())
-                                        .build(),
-                                    maybe_reassigned,
-                                )
-                            }
                         }
                     })
                     .await)

nexus/src/app/quiesce.rs

@@ -32,9 +32,9 @@ impl super::Nexus {
     ) -> LookupResult<QuiesceStatus> {
         opctx.authorize(authz::Action::Read, &authz::QUIESCE_STATE).await?;
         let state = self.quiesce.state();
-        let sagas_pending = self.quiesce.sagas().sagas_pending();
+        let sagas = self.quiesce.sagas().status();
         let db_claims = self.datastore().claims_held();
-        Ok(QuiesceStatus { state, sagas_pending, db_claims })
+        Ok(QuiesceStatus { state, sagas, db_claims })
     }
 }
 
@@ -283,7 +283,7 @@ mod test {
         assert!(duration_total >= duration_draining_db);
         assert!(duration_total >= duration_recording_quiesce);
         assert!(duration_total <= (after - before).to_std().unwrap());
-        assert!(status.sagas_pending.is_empty());
+        assert!(status.sagas.sagas_pending.is_empty());
         assert!(status.db_claims.is_empty());
     }
 
@@ -357,7 +357,9 @@ mod test {
             quiesce_status.state,
             QuiesceState::DrainingSagas { .. }
         );
-        assert!(quiesce_status.sagas_pending.contains_key(&demo_saga.saga_id));
+        assert!(
+            quiesce_status.sagas.sagas_pending.contains_key(&demo_saga.saga_id)
+        );
         // We should see at least one held database claim from the one we took
         // above.
         assert!(!quiesce_status.db_claims.is_empty());
@@ -421,7 +423,7 @@ mod test {
                 if !matches!(rv.state, QuiesceState::DrainingDb { .. }) {
                     return Err(CondCheckError::<NexusClientError>::NotYet);
                 }
-                assert!(rv.sagas_pending.is_empty());
+                assert!(rv.sagas.sagas_pending.is_empty());
                 // The database claim we took is still held.
                 assert!(!rv.db_claims.is_empty());
                 Ok(())

nexus/types/src/internal_api/views.rs

@@ -7,6 +7,7 @@ use crate::deployment::TargetReleaseDescription;
 use crate::inventory::BaseboardId;
 use crate::inventory::CabooseWhich;
 use crate::inventory::Collection;
+use crate::quiesce::SagaQuiesceStatus;
 use chrono::DateTime;
 use chrono::SecondsFormat;
 use chrono::Utc;
@@ -978,12 +979,8 @@ pub struct QuiesceStatus {
     /// what stage of quiescing is Nexus at
     pub state: QuiesceState,
 
-    /// what sagas are currently running or known needing to be recovered
-    ///
-    /// This should only be non-empty when state is `Running` or
-    /// `WaitingForSagas`.  Entries here prevent transitioning from
-    /// `WaitingForSagas` to `WaitingForDb`.
-    pub sagas_pending: IdOrdMap<PendingSagaInfo>,
+    /// information about saga quiescing
+    pub sagas: SagaQuiesceStatus,
 
     /// what database claims are currently held (by any part of Nexus)
     ///