From b626d4a613d00fad5b1cf538dc722ad112f1a245 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 12 Mar 2026 02:16:51 -0700 Subject: [PATCH 1/7] refactor: move PerObjectBlobInfoV1 to blob_info_v1.rs Co-Authored-By: Claude Opus 4.6 --- .../src/node/storage/blob_info.rs | 146 +---------------- .../node/storage/blob_info/blob_info_v1.rs | 149 +++++++++++++++++- 2 files changed, 149 insertions(+), 146 deletions(-) diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index 3d16445536..f31a6106f4 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -1243,151 +1243,7 @@ mod per_object_blob_info { } } - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] - pub(crate) struct PerObjectBlobInfoV1 { - /// The blob ID. - pub blob_id: BlobId, - /// The epoch in which the blob has been registered. - pub registered_epoch: Epoch, - /// The epoch in which the blob was first certified, `None` if the blob is uncertified. - pub certified_epoch: Option, - /// The epoch in which the blob expires. - pub end_epoch: Epoch, - /// Whether the blob is deletable. - pub deletable: bool, - /// The ID of the last blob event related to this object. - pub event: EventID, - /// Whether the blob has been deleted. - pub deleted: bool, - } - - impl CertifiedBlobInfoApi for PerObjectBlobInfoV1 { - fn is_certified(&self, current_epoch: Epoch) -> bool { - self.is_registered(current_epoch) - && self - .certified_epoch - .is_some_and(|epoch| epoch <= current_epoch) - } - - fn initial_certified_epoch(&self) -> Option { - self.certified_epoch - } - } - - impl PerObjectBlobInfoApi for PerObjectBlobInfoV1 { - fn blob_id(&self) -> BlobId { - self.blob_id - } - - fn is_deletable(&self) -> bool { - self.deletable - } - - fn is_registered(&self, current_epoch: Epoch) -> bool { - self.end_epoch > current_epoch && !self.deleted - } - - fn is_deleted(&self) -> bool { - self.deleted - } - } - - impl ToBytes for PerObjectBlobInfoV1 {} - - impl Mergeable for PerObjectBlobInfoV1 { - type MergeOperand = PerObjectBlobInfoMergeOperand; - type Key = ObjectID; - - fn merge_with( - mut self, - PerObjectBlobInfoMergeOperand { - change_type, - change_info, - }: PerObjectBlobInfoMergeOperand, - ) -> Self { - assert_eq!( - self.blob_id, change_info.blob_id, - "blob ID mismatch in merge operand" - ); - assert_eq!( - self.deletable, change_info.deletable, - "deletable mismatch in merge operand" - ); - assert!( - !self.deleted, - "attempt to update an already deleted blob {}", - self.blob_id - ); - self.event = change_info.status_event; - match change_type { - // We ensure that the blob info is only updated a single time for each event. So if - // we see a duplicated registered or certified event for the some object, this is a - // serious bug somewhere. - BlobStatusChangeType::Register => { - panic!( - "cannot register an already registered blob {}", - self.blob_id - ); - } - BlobStatusChangeType::Certify => { - assert!( - self.certified_epoch.is_none(), - "cannot certify an already certified blob {}", - self.blob_id - ); - self.certified_epoch = Some(change_info.epoch); - } - BlobStatusChangeType::Extend => { - assert!( - self.certified_epoch.is_some(), - "cannot extend an uncertified blob {}", - self.blob_id - ); - self.end_epoch = change_info.end_epoch; - } - BlobStatusChangeType::Delete { was_certified } => { - assert_eq!(self.certified_epoch.is_some(), was_certified); - self.deleted = true; - } - } - self - } - - fn merge_new(operand: Self::MergeOperand) -> Option { - let PerObjectBlobInfoMergeOperand { - change_type: BlobStatusChangeType::Register, - change_info: - BlobStatusChangeInfo { - blob_id, - deletable, - epoch, - end_epoch, - status_event, - }, - } = operand - else { - tracing::error!( - ?operand, - "encountered an update other than 'register' for an untracked blob object" - ); - debug_assert!( - false, - "encountered an update other than 'register' for an untracked blob object: \ - {operand:?}" - ); - return None; - }; - Some(Self { - blob_id, - registered_epoch: epoch, - certified_epoch: None, - end_epoch, - deletable, - event: status_event, - deleted: false, - }) - } - } + pub(crate) use super::blob_info_v1::PerObjectBlobInfoV1; } fn deserialize_from_db<'de, T>(data: &'de [u8]) -> Option diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs index 1f8609fb8b..32c12e8208 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs @@ -6,7 +6,7 @@ use std::num::NonZeroU32; use serde::{Deserialize, Serialize}; -use sui_types::event::EventID; +use sui_types::{base_types::ObjectID, event::EventID}; use walrus_core::{BlobId, Epoch}; use walrus_storage_node_client::api::{BlobStatus, DeletableCounts}; @@ -18,6 +18,7 @@ use super::{ CertifiedBlobInfoApi, Mergeable, ToBytes, + per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, }; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -735,6 +736,152 @@ impl Mergeable for BlobInfoV1 { } } +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] +pub(crate) struct PerObjectBlobInfoV1 { + /// The blob ID. + pub blob_id: BlobId, + /// The epoch in which the blob has been registered. + pub registered_epoch: Epoch, + /// The epoch in which the blob was first certified, `None` if the blob is uncertified. + pub certified_epoch: Option, + /// The epoch in which the blob expires. + pub end_epoch: Epoch, + /// Whether the blob is deletable. + pub deletable: bool, + /// The ID of the last blob event related to this object. + pub event: EventID, + /// Whether the blob has been deleted. + pub deleted: bool, +} + +impl CertifiedBlobInfoApi for PerObjectBlobInfoV1 { + fn is_certified(&self, current_epoch: Epoch) -> bool { + self.is_registered(current_epoch) + && self + .certified_epoch + .is_some_and(|epoch| epoch <= current_epoch) + } + + fn initial_certified_epoch(&self) -> Option { + self.certified_epoch + } +} + +impl PerObjectBlobInfoApi for PerObjectBlobInfoV1 { + fn blob_id(&self) -> BlobId { + self.blob_id + } + + fn is_deletable(&self) -> bool { + self.deletable + } + + fn is_registered(&self, current_epoch: Epoch) -> bool { + self.end_epoch > current_epoch && !self.deleted + } + + fn is_deleted(&self) -> bool { + self.deleted + } +} + +impl ToBytes for PerObjectBlobInfoV1 {} + +impl Mergeable for PerObjectBlobInfoV1 { + type MergeOperand = PerObjectBlobInfoMergeOperand; + type Key = ObjectID; + + fn merge_with( + mut self, + PerObjectBlobInfoMergeOperand { + change_type, + change_info, + }: PerObjectBlobInfoMergeOperand, + ) -> Self { + assert_eq!( + self.blob_id, change_info.blob_id, + "blob ID mismatch in merge operand" + ); + assert_eq!( + self.deletable, change_info.deletable, + "deletable mismatch in merge operand" + ); + assert!( + !self.deleted, + "attempt to update an already deleted blob {}", + self.blob_id + ); + self.event = change_info.status_event; + match change_type { + // We ensure that the blob info is only updated a single time for each event. So if + // we see a duplicated registered or certified event for the some object, this is a + // serious bug somewhere. + BlobStatusChangeType::Register => { + panic!( + "cannot register an already registered blob {}", + self.blob_id + ); + } + BlobStatusChangeType::Certify => { + assert!( + self.certified_epoch.is_none(), + "cannot certify an already certified blob {}", + self.blob_id + ); + self.certified_epoch = Some(change_info.epoch); + } + BlobStatusChangeType::Extend => { + assert!( + self.certified_epoch.is_some(), + "cannot extend an uncertified blob {}", + self.blob_id + ); + self.end_epoch = change_info.end_epoch; + } + BlobStatusChangeType::Delete { was_certified } => { + assert_eq!(self.certified_epoch.is_some(), was_certified); + self.deleted = true; + } + } + self + } + + fn merge_new(operand: Self::MergeOperand) -> Option { + let PerObjectBlobInfoMergeOperand { + change_type: BlobStatusChangeType::Register, + change_info: + BlobStatusChangeInfo { + blob_id, + deletable, + epoch, + end_epoch, + status_event, + }, + } = operand + else { + tracing::error!( + ?operand, + "encountered an update other than 'register' for an untracked blob object" + ); + debug_assert!( + false, + "encountered an update other than 'register' for an untracked blob object: \ + {operand:?}" + ); + return None; + }; + Some(Self { + blob_id, + registered_epoch: epoch, + certified_epoch: None, + end_epoch, + deletable, + event: status_event, + deleted: false, + }) + } +} + #[cfg(test)] mod tests { use walrus_sui::test_utils::{event_id_for_testing, fixed_event_id_for_testing}; From 9d9cf54803bc6b9a44261dee25a8e31650479848 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 12 Mar 2026 02:25:16 -0700 Subject: [PATCH 2/7] feat: introduce BlobInfoV2 and PerObjectBlobInfoV2 as copies of V1 Co-Authored-By: Claude Opus 4.6 --- .../src/node/storage/blob_info.rs | 8 +- .../node/storage/blob_info/blob_info_v2.rs | 1586 +++++++++++++++++ 2 files changed, 1593 insertions(+), 1 deletion(-) create mode 100644 crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index f31a6106f4..5493883266 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -4,6 +4,7 @@ //! Keeping track of the status of blob IDs and on-chain `Blob` objects. mod blob_info_v1; +mod blob_info_v2; use std::{ collections::HashSet, @@ -33,6 +34,7 @@ pub(crate) use self::blob_info_v1::PermanentBlobInfoV1; use self::per_object_blob_info::PerObjectBlobInfoMergeOperand; pub(crate) use self::{ blob_info_v1::{BlobInfoV1, ValidBlobInfoV1}, + blob_info_v2::BlobInfoV2, per_object_blob_info::{PerObjectBlobInfo, PerObjectBlobInfoApi}, }; use super::{DatabaseTableOptionsFactory, constants}; @@ -1085,6 +1087,7 @@ impl Ord for BlobCertificationStatus { #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) enum BlobInfo { V1(BlobInfoV1), + V2(BlobInfoV2), } impl BlobInfo { @@ -1134,6 +1137,7 @@ impl Mergeable for BlobInfo { fn merge_with(self, operand: Self::MergeOperand) -> Self { match self { Self::V1(value) => Self::V1(value.merge_with(operand)), + Self::V2(value) => Self::V2(value.merge_with(operand)), } } @@ -1201,6 +1205,7 @@ mod per_object_blob_info { #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) enum PerObjectBlobInfo { V1(PerObjectBlobInfoV1), + V2(PerObjectBlobInfoV2), } impl PerObjectBlobInfo { @@ -1235,6 +1240,7 @@ mod per_object_blob_info { fn merge_with(self, operand: Self::MergeOperand) -> Self { match self { Self::V1(value) => Self::V1(value.merge_with(operand)), + Self::V2(value) => Self::V2(value.merge_with(operand)), } } @@ -1243,7 +1249,7 @@ mod per_object_blob_info { } } - pub(crate) use super::blob_info_v1::PerObjectBlobInfoV1; + pub(crate) use super::{blob_info_v1::PerObjectBlobInfoV1, blob_info_v2::PerObjectBlobInfoV2}; } fn deserialize_from_db<'de, T>(data: &'de [u8]) -> Option diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs new file mode 100644 index 0000000000..ce3f7c6c1b --- /dev/null +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs @@ -0,0 +1,1586 @@ +// Copyright (c) Walrus Foundation +// SPDX-License-Identifier: Apache-2.0 + +//! V2 blob info types and merge logic. + +#![allow(dead_code)] + +use std::num::NonZeroU32; + +use serde::{Deserialize, Serialize}; +use sui_types::{base_types::ObjectID, event::EventID}; +use walrus_core::{BlobId, Epoch}; +use walrus_storage_node_client::api::{BlobStatus, DeletableCounts}; + +use super::{ + BlobInfoApi, + BlobInfoMergeOperand, + BlobStatusChangeInfo, + BlobStatusChangeType, + CertifiedBlobInfoApi, + Mergeable, + ToBytes, + per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) enum BlobInfoV2 { + Invalid { epoch: Epoch, event: EventID }, + Valid(ValidBlobInfoV2), +} + +impl ToBytes for BlobInfoV2 {} + +// INV: count_deletable_total >= count_deletable_certified +// INV: permanent_total.is_none() => permanent_certified.is_none() +// INV: permanent_total.count >= permanent_certified.count +// INV: permanent_total.end_epoch >= permanent_certified.end_epoch +// INV: initial_certified_epoch.is_some() +// <=> count_deletable_certified > 0 || permanent_certified.is_some() +// INV: latest_seen_deletable_registered_end_epoch >= latest_seen_deletable_certified_end_epoch +// See the `check_invariants` method for more details. +// Important: This struct MUST NOT be changed. Instead, if needed, a new `ValidBlobInfoV3` struct +// should be created. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub(crate) struct ValidBlobInfoV2 { + pub is_metadata_stored: bool, + pub count_deletable_total: u32, + pub count_deletable_certified: u32, + pub permanent_total: Option, + pub permanent_certified: Option, + pub initial_certified_epoch: Option, + + // Note: The following helper fields were used in the past to approximate the blob status for + // deletable blobs. They are still used for the following reason: When deletable blobs expire, + // we update the aggregate blob info in the background. So there is a delay between an epoch + // change and the blob info being updated. During this delay, these fields are useful to + // determine that a blob is already expired. + /// The latest end epoch recorded for a registered deletable blob. + pub latest_seen_deletable_registered_end_epoch: Option, + /// The latest end epoch recorded for a certified deletable blob. + pub latest_seen_deletable_certified_end_epoch: Option, +} + +impl From for BlobInfoV2 { + fn from(value: ValidBlobInfoV2) -> Self { + Self::Valid(value) + } +} + +impl ValidBlobInfoV2 { + fn to_blob_status(&self, current_epoch: Epoch) -> BlobStatus { + // The check of the `latest_seen_*_epoch` fields is there to reduce the cases where we + // report the existence of deletable blobs that are already expired. + let count_deletable_total = if self + .latest_seen_deletable_registered_end_epoch + .is_some_and(|e| e > current_epoch) + { + self.count_deletable_total + } else { + Default::default() + }; + let count_deletable_certified = if self + .latest_seen_deletable_certified_end_epoch + .is_some_and(|e| e > current_epoch) + { + self.count_deletable_certified + } else { + Default::default() + }; + let deletable_counts = DeletableCounts { + count_deletable_total, + count_deletable_certified, + }; + + let initial_certified_epoch = self.initial_certified_epoch; + if let Some(PermanentBlobInfoV2 { + end_epoch, event, .. + }) = self.permanent_certified.as_ref() + && *end_epoch > current_epoch + { + return BlobStatus::Permanent { + end_epoch: *end_epoch, + is_certified: true, + status_event: *event, + deletable_counts, + initial_certified_epoch, + }; + } + if let Some(PermanentBlobInfoV2 { + end_epoch, event, .. + }) = self.permanent_total.as_ref() + && *end_epoch > current_epoch + { + return BlobStatus::Permanent { + end_epoch: *end_epoch, + is_certified: false, + status_event: *event, + deletable_counts, + initial_certified_epoch, + }; + } + + if deletable_counts != Default::default() { + BlobStatus::Deletable { + initial_certified_epoch, + deletable_counts, + } + } else { + BlobStatus::Nonexistent + } + } + + // The counts of deletable blobs are decreased when they expire. However, because this only + // happens in the background, it is possible during a short time period at the beginning of an + // epoch that all deletable blobs expired but the counts are not updated yet. For this case, we + // use the `latest_seen_deletable_certified_end_epoch` field as an additional check. + // + // There is still a corner case where the blob with the + // `latest_seen_deletable_certified_end_epoch` was deleted and all other blobs expired. In this + // case, we would nevertheless return `true` during a short time period, before the counts are + // updated in the background. + fn is_certified(&self, current_epoch: Epoch) -> bool { + let exists_certified_permanent_blob = self + .permanent_certified + .as_ref() + .is_some_and(|p| p.end_epoch > current_epoch); + let probably_exists_certified_deletable_blob = self.count_deletable_certified > 0 + && self + .latest_seen_deletable_certified_end_epoch + .is_some_and(|e| e > current_epoch); + self.initial_certified_epoch + .is_some_and(|epoch| epoch <= current_epoch) + && (exists_certified_permanent_blob || probably_exists_certified_deletable_blob) + } + + pub(crate) fn has_no_objects(&self) -> bool { + matches!( + self, + Self { + is_metadata_stored: _, + count_deletable_total: 0, + count_deletable_certified: 0, + permanent_total: None, + permanent_certified: None, + initial_certified_epoch: None, + latest_seen_deletable_registered_end_epoch: None, + latest_seen_deletable_certified_end_epoch: None, + } + ) + } + + #[tracing::instrument] + fn update_status( + &mut self, + change_type: BlobStatusChangeType, + change_info: BlobStatusChangeInfo, + ) { + let was_certified = self.is_certified(change_info.epoch); + if change_info.deletable { + match change_type { + BlobStatusChangeType::Register => { + self.count_deletable_total += 1; + self.maybe_increase_latest_deletable_registered_epoch(change_info.end_epoch); + } + BlobStatusChangeType::Certify => { + if self.count_deletable_total <= self.count_deletable_certified { + tracing::error!( + "attempt to certify a deletable blob before corresponding register" + ); + return; + } + self.count_deletable_certified += 1; + self.maybe_increase_latest_deletable_certified_epoch(change_info.end_epoch); + } + BlobStatusChangeType::Extend => { + self.maybe_increase_latest_deletable_registered_epoch(change_info.end_epoch); + self.maybe_increase_latest_deletable_certified_epoch(change_info.end_epoch); + } + BlobStatusChangeType::Delete { was_certified } => { + self.update_deletable_counters_and_end_epochs(was_certified); + } + } + } else { + match change_type { + BlobStatusChangeType::Register => { + Self::register_permanent(&mut self.permanent_total, &change_info); + } + BlobStatusChangeType::Certify => { + if !Self::certify_permanent( + &self.permanent_total, + &mut self.permanent_certified, + &change_info, + ) { + // Return early to prevent updating the `initial_certified_epoch` below. + return; + } + } + BlobStatusChangeType::Extend => { + Self::extend_permanent(&mut self.permanent_total, &change_info); + Self::extend_permanent(&mut self.permanent_certified, &change_info); + } + BlobStatusChangeType::Delete { .. } => { + tracing::error!("attempt to delete a permanent blob"); + return; + } + } + } + + // Update initial certified epoch. + match change_type { + BlobStatusChangeType::Certify => { + self.update_initial_certified_epoch(change_info.epoch, !was_certified); + } + BlobStatusChangeType::Delete { .. } => { + self.maybe_unset_initial_certified_epoch(); + } + // Explicit matches to make sure we cover all cases. + BlobStatusChangeType::Register | BlobStatusChangeType::Extend => (), + } + } + + fn deletable_expired(&mut self, was_certified: bool) { + self.update_deletable_counters_and_end_epochs(was_certified); + self.maybe_unset_initial_certified_epoch(); + } + + fn update_deletable_counters_and_end_epochs(&mut self, was_certified: bool) { + Self::decrement_deletable_counter_and_maybe_unset_latest_end_epoch( + &mut self.count_deletable_total, + &mut self.latest_seen_deletable_registered_end_epoch, + ); + if was_certified { + Self::decrement_deletable_counter_and_maybe_unset_latest_end_epoch( + &mut self.count_deletable_certified, + &mut self.latest_seen_deletable_certified_end_epoch, + ); + } + } + + fn update_initial_certified_epoch(&mut self, new_certified_epoch: Epoch, force: bool) { + if force + || self + .initial_certified_epoch + .is_none_or(|existing_epoch| existing_epoch > new_certified_epoch) + { + self.initial_certified_epoch = Some(new_certified_epoch); + } + } + + fn maybe_unset_initial_certified_epoch(&mut self) { + if self.count_deletable_certified == 0 && self.permanent_certified.is_none() { + self.initial_certified_epoch = None; + } + } + + fn maybe_increase_latest_deletable_registered_epoch(&mut self, epoch: Epoch) { + self.latest_seen_deletable_registered_end_epoch = Some( + epoch.max( + self.latest_seen_deletable_registered_end_epoch + .unwrap_or_default(), + ), + ) + } + + fn maybe_increase_latest_deletable_certified_epoch(&mut self, epoch: Epoch) { + self.latest_seen_deletable_certified_end_epoch = Some( + epoch.max( + self.latest_seen_deletable_certified_end_epoch + .unwrap_or_default(), + ), + ) + } + + /// Decrements a counter on blob deletion and unsets the corresponding latest seen end epoch in + /// case the counter reaches 0. + /// + /// If the counter is already 0, an error is logged in release builds and the function panics in + /// dev builds. + fn decrement_deletable_counter_and_maybe_unset_latest_end_epoch( + counter: &mut u32, + latest_seen_end_epoch: &mut Option, + ) { + debug_assert!(*counter > 0); + *counter = counter.checked_sub(1).unwrap_or_else(|| { + tracing::error!("attempt to delete blob when count was already 0"); + 0 + }); + if *counter == 0 { + *latest_seen_end_epoch = None; + } + } + + /// Processes a register status change on the [`Option`] object + /// representing all permanent blobs. + fn register_permanent( + permanent_total: &mut Option, + change_info: &BlobStatusChangeInfo, + ) { + PermanentBlobInfoV2::update_optional(permanent_total, change_info) + } + + /// Processes a certify status change on the [`PermanentBlobInfoV2`] objects representing all + /// and the certified permanent blobs. + /// + /// Returns whether the update was successful. + fn certify_permanent( + permanent_total: &Option, + permanent_certified: &mut Option, + change_info: &BlobStatusChangeInfo, + ) -> bool { + let Some(permanent_total) = permanent_total else { + tracing::error!("attempt to certify a permanent blob when none is tracked"); + return false; + }; + + let registered_end_epoch = permanent_total.end_epoch; + let certified_end_epoch = change_info.end_epoch; + if certified_end_epoch > registered_end_epoch { + tracing::error!( + registered_end_epoch, + certified_end_epoch, + "attempt to certify a permanent blob with later end epoch than any registered blob", + ); + return false; + } + if permanent_total.count.get() + <= permanent_certified + .as_ref() + .map(|p| p.count.get()) + .unwrap_or_default() + { + tracing::error!("attempt to certify a permanent blob before corresponding register"); + return false; + } + PermanentBlobInfoV2::update_optional(permanent_certified, change_info); + true + } + + /// Processes an extend status change on the [`PermanentBlobInfoV2`] object representing the + /// certified permanent blobs. + fn extend_permanent( + permanent_info: &mut Option, + change_info: &BlobStatusChangeInfo, + ) { + let Some(permanent_info) = permanent_info else { + tracing::error!("attempt to extend a permanent blob when none is tracked"); + return; + }; + + permanent_info.update(change_info, false); + } + + /// Processes a delete status change on the [`PermanentBlobInfoV2`] objects representing all and + /// the certified permanent blobs. + /// + /// This is called when blobs expire at the end of an epoch. + fn permanent_expired(&mut self, was_certified: bool) { + Self::decrement_blob_info_inner(&mut self.permanent_total); + if was_certified { + Self::decrement_blob_info_inner(&mut self.permanent_certified); + } + self.maybe_unset_initial_certified_epoch(); + } + + fn decrement_blob_info_inner(blob_info_inner: &mut Option) { + match blob_info_inner { + None => tracing::error!("attempt to delete a permanent blob when none is tracked"), + Some(PermanentBlobInfoV2 { count, .. }) => { + if count.get() == 1 { + *blob_info_inner = None; + } else { + *count = NonZeroU32::new(count.get() - 1) + .expect("we just checked that `count` is at least 2") + } + } + } + } + + /// Checks the invariants of the aggregate blob info, returning an error if any invariant is + /// violated. + pub(crate) fn check_invariants(&self) -> anyhow::Result<()> { + let Self { + count_deletable_total, + count_deletable_certified, + permanent_total, + permanent_certified, + initial_certified_epoch, + latest_seen_deletable_registered_end_epoch, + latest_seen_deletable_certified_end_epoch, + .. + } = self; + + anyhow::ensure!(count_deletable_total >= count_deletable_certified); + match initial_certified_epoch { + None => { + anyhow::ensure!(*count_deletable_certified == 0 && permanent_certified.is_none()) + } + Some(_) => { + anyhow::ensure!(*count_deletable_certified > 0 || permanent_certified.is_some()) + } + } + + match (permanent_total, permanent_certified) { + (None, Some(_)) => { + anyhow::bail!("permanent_total.is_none() => permanent_certified.is_none()") + } + (Some(total_inner), Some(certified_inner)) => { + anyhow::ensure!(total_inner.end_epoch >= certified_inner.end_epoch); + anyhow::ensure!(total_inner.count >= certified_inner.count); + } + _ => (), + } + + match ( + latest_seen_deletable_registered_end_epoch, + latest_seen_deletable_certified_end_epoch, + ) { + (None, Some(_)) => anyhow::bail!( + "latest_seen_deletable_registered_end_epoch.is_none() => \ + latest_seen_deletable_certified_end_epoch.is_none()" + ), + (Some(registered), Some(certified)) => { + anyhow::ensure!(registered >= certified); + } + _ => (), + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct PermanentBlobInfoV2 { + /// The total number of `Blob` objects for that blob ID with the given status. + pub count: NonZeroU32, + /// The latest expiration epoch among these objects. + pub end_epoch: Epoch, + /// The ID of the first blob event that led to the status with the given `end_epoch`. + pub event: EventID, +} + +impl PermanentBlobInfoV2 { + /// Creates a new `PermanentBlobInfoV2` object for the first blob with the given `end_epoch` and + /// `event`. + pub(crate) fn new_first(end_epoch: Epoch, event: EventID) -> Self { + Self { + count: NonZeroU32::new(1).expect("1 is non-zero"), + end_epoch, + event, + } + } + + /// Updates `self` with the `change_info`, increasing the count if `increase_count == true`. + /// + /// # Panics + /// + /// Panics if the change info has `deletable == true`. + fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { + assert!(!change_info.deletable); + + if increase_count { + self.count = self.count.saturating_add(1) + }; + if change_info.end_epoch > self.end_epoch { + *self = PermanentBlobInfoV2 { + count: self.count, + end_epoch: change_info.end_epoch, + event: change_info.status_event, + }; + } + } + + /// Updates `existing_info` with the change info or creates a new `Self` if the input is `None`. + /// + /// # Panics + /// + /// Panics if the change info has `deletable == true`. + fn update_optional(existing_info: &mut Option, change_info: &BlobStatusChangeInfo) { + let BlobStatusChangeInfo { + epoch: _, + end_epoch: new_end_epoch, + status_event: new_status_event, + deletable, + blob_id: _, + } = change_info; + assert!(!deletable); + + match existing_info { + None => { + *existing_info = Some(PermanentBlobInfoV2::new_first( + *new_end_epoch, + *new_status_event, + )) + } + Some(permanent_blob_info) => permanent_blob_info.update(change_info, true), + } + } + + #[cfg(test)] + fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { + Self { + count: NonZeroU32::new(count).expect("count must be non-zero"), + end_epoch, + event: walrus_sui::test_utils::fixed_event_id_for_testing(event_seq), + } + } + + #[cfg(test)] + fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { + Self { + count: NonZeroU32::new(count).expect("count must be non-zero"), + end_epoch, + event: walrus_sui::test_utils::event_id_for_testing(), + } + } +} + +impl CertifiedBlobInfoApi for BlobInfoV2 { + fn is_certified(&self, current_epoch: Epoch) -> bool { + if let Self::Valid(valid_blob_info) = self { + valid_blob_info.is_certified(current_epoch) + } else { + false + } + } + + fn initial_certified_epoch(&self) -> Option { + if let Self::Valid(ValidBlobInfoV2 { + initial_certified_epoch, + .. + }) = self + { + *initial_certified_epoch + } else { + None + } + } +} + +impl BlobInfoApi for BlobInfoV2 { + fn is_metadata_stored(&self) -> bool { + matches!( + self, + Self::Valid(ValidBlobInfoV2 { + is_metadata_stored: true, + .. + }) + ) + } + + // Note: See the `is_certified` method for an explanation of the use of the + // `latest_seen_deletable_registered_end_epoch` field. + fn is_registered(&self, current_epoch: Epoch) -> bool { + let Self::Valid(ValidBlobInfoV2 { + count_deletable_total, + permanent_total, + latest_seen_deletable_registered_end_epoch, + .. + }) = self + else { + return false; + }; + + let exists_registered_permanent_blob = permanent_total + .as_ref() + .is_some_and(|p| p.end_epoch > current_epoch); + let probably_exists_registered_deletable_blob = *count_deletable_total > 0 + && latest_seen_deletable_registered_end_epoch.is_some_and(|e| e > current_epoch); + + exists_registered_permanent_blob || probably_exists_registered_deletable_blob + } + + fn can_blob_info_be_deleted(&self, current_epoch: Epoch) -> bool { + match self { + Self::Invalid { .. } => { + // We don't know whether there are any deletable blob objects for this blob ID. + false + } + Self::Valid(ValidBlobInfoV2 { + count_deletable_total, + permanent_total, + .. + }) => { + *count_deletable_total == 0 + && permanent_total.is_none() + && self.can_data_be_deleted(current_epoch) + } + } + } + + fn invalidation_event(&self) -> Option { + if let Self::Invalid { event, .. } = self { + Some(*event) + } else { + None + } + } + + fn to_blob_status(&self, current_epoch: Epoch) -> BlobStatus { + match self { + BlobInfoV2::Invalid { event, .. } => BlobStatus::Invalid { event: *event }, + BlobInfoV2::Valid(valid_blob_info) => valid_blob_info.to_blob_status(current_epoch), + } + } +} + +impl Mergeable for BlobInfoV2 { + type MergeOperand = BlobInfoMergeOperand; + type Key = BlobId; + + fn merge_with(mut self, operand: Self::MergeOperand) -> Self { + match (&mut self, operand) { + // If the blob is already marked as invalid, do not update the status. + (Self::Invalid { .. }, _) => (), + ( + _, + BlobInfoMergeOperand::MarkInvalid { + epoch, + status_event, + }, + ) => { + return Self::Invalid { + epoch, + event: status_event, + }; + } + ( + Self::Valid(ValidBlobInfoV2 { + is_metadata_stored, .. + }), + BlobInfoMergeOperand::MarkMetadataStored(new_is_metadata_stored), + ) => { + *is_metadata_stored = new_is_metadata_stored; + } + ( + Self::Valid(valid_blob_info), + BlobInfoMergeOperand::ChangeStatus { + change_type, + change_info, + }, + ) => valid_blob_info.update_status(change_type, change_info), + ( + Self::Valid(valid_blob_info), + BlobInfoMergeOperand::DeletableExpired { was_certified }, + ) => valid_blob_info.deletable_expired(was_certified), + ( + Self::Valid(valid_blob_info), + BlobInfoMergeOperand::PermanentExpired { was_certified }, + ) => valid_blob_info.permanent_expired(was_certified), + } + self + } + + fn merge_new(operand: Self::MergeOperand) -> Option { + match operand { + BlobInfoMergeOperand::ChangeStatus { + change_type: BlobStatusChangeType::Register, + change_info: + BlobStatusChangeInfo { + deletable, + epoch: _, + end_epoch, + status_event, + blob_id: _, + }, + } => Some( + if deletable { + ValidBlobInfoV2 { + count_deletable_total: 1, + latest_seen_deletable_registered_end_epoch: Some(end_epoch), + ..Default::default() + } + } else { + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV2::new_first( + end_epoch, + status_event, + )), + ..Default::default() + } + } + .into(), + ), + BlobInfoMergeOperand::MarkInvalid { + epoch, + status_event, + } => Some(BlobInfoV2::Invalid { + epoch, + event: status_event, + }), + BlobInfoMergeOperand::MarkMetadataStored(is_metadata_stored) => { + tracing::info!( + is_metadata_stored, + "marking metadata stored for an untracked blob ID; blob info will be removed \ + during the next garbage collection" + ); + Some( + ValidBlobInfoV2 { + is_metadata_stored, + ..Default::default() + } + .into(), + ) + } + BlobInfoMergeOperand::ChangeStatus { .. } + | BlobInfoMergeOperand::DeletableExpired { .. } + | BlobInfoMergeOperand::PermanentExpired { .. } => { + tracing::error!( + ?operand, + "encountered an unexpected update for an untracked blob ID" + ); + debug_assert!( + false, + "encountered an unexpected update for an untracked blob ID: {operand:?}", + ); + None + } + } + } +} + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] +pub(crate) struct PerObjectBlobInfoV2 { + /// The blob ID. + pub blob_id: BlobId, + /// The epoch in which the blob has been registered. + pub registered_epoch: Epoch, + /// The epoch in which the blob was first certified, `None` if the blob is uncertified. + pub certified_epoch: Option, + /// The epoch in which the blob expires. + pub end_epoch: Epoch, + /// Whether the blob is deletable. + pub deletable: bool, + /// The ID of the last blob event related to this object. + pub event: EventID, + /// Whether the blob has been deleted. + pub deleted: bool, +} + +impl CertifiedBlobInfoApi for PerObjectBlobInfoV2 { + fn is_certified(&self, current_epoch: Epoch) -> bool { + self.is_registered(current_epoch) + && self + .certified_epoch + .is_some_and(|epoch| epoch <= current_epoch) + } + + fn initial_certified_epoch(&self) -> Option { + self.certified_epoch + } +} + +impl PerObjectBlobInfoApi for PerObjectBlobInfoV2 { + fn blob_id(&self) -> BlobId { + self.blob_id + } + + fn is_deletable(&self) -> bool { + self.deletable + } + + fn is_registered(&self, current_epoch: Epoch) -> bool { + self.end_epoch > current_epoch && !self.deleted + } + + fn is_deleted(&self) -> bool { + self.deleted + } +} + +impl ToBytes for PerObjectBlobInfoV2 {} + +impl Mergeable for PerObjectBlobInfoV2 { + type MergeOperand = PerObjectBlobInfoMergeOperand; + type Key = ObjectID; + + fn merge_with( + mut self, + PerObjectBlobInfoMergeOperand { + change_type, + change_info, + }: PerObjectBlobInfoMergeOperand, + ) -> Self { + assert_eq!( + self.blob_id, change_info.blob_id, + "blob ID mismatch in merge operand" + ); + assert_eq!( + self.deletable, change_info.deletable, + "deletable mismatch in merge operand" + ); + assert!( + !self.deleted, + "attempt to update an already deleted blob {}", + self.blob_id + ); + self.event = change_info.status_event; + match change_type { + // We ensure that the blob info is only updated a single time for each event. So if + // we see a duplicated registered or certified event for the some object, this is a + // serious bug somewhere. + BlobStatusChangeType::Register => { + panic!( + "cannot register an already registered blob {}", + self.blob_id + ); + } + BlobStatusChangeType::Certify => { + assert!( + self.certified_epoch.is_none(), + "cannot certify an already certified blob {}", + self.blob_id + ); + self.certified_epoch = Some(change_info.epoch); + } + BlobStatusChangeType::Extend => { + assert!( + self.certified_epoch.is_some(), + "cannot extend an uncertified blob {}", + self.blob_id + ); + self.end_epoch = change_info.end_epoch; + } + BlobStatusChangeType::Delete { was_certified } => { + assert_eq!(self.certified_epoch.is_some(), was_certified); + self.deleted = true; + } + } + self + } + + fn merge_new(operand: Self::MergeOperand) -> Option { + let PerObjectBlobInfoMergeOperand { + change_type: BlobStatusChangeType::Register, + change_info: + BlobStatusChangeInfo { + blob_id, + deletable, + epoch, + end_epoch, + status_event, + }, + } = operand + else { + tracing::error!( + ?operand, + "encountered an update other than 'register' for an untracked blob object" + ); + debug_assert!( + false, + "encountered an update other than 'register' for an untracked blob object: \ + {operand:?}" + ); + return None; + }; + Some(Self { + blob_id, + registered_epoch: epoch, + certified_epoch: None, + end_epoch, + deletable, + event: status_event, + deleted: false, + }) + } +} + +#[cfg(test)] +mod tests { + use walrus_sui::test_utils::{event_id_for_testing, fixed_event_id_for_testing}; + use walrus_test_utils::param_test; + + use super::*; + + fn check_invariants(blob_info: &BlobInfoV2) { + if let BlobInfoV2::Valid(valid_blob_info) = blob_info { + valid_blob_info + .check_invariants() + .expect("aggregate blob info invariants violated") + } + } + + param_test! { + test_merge_new_expected_failure_cases: [ + #[should_panic] certify_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify,false, 42, 314, event_id_for_testing() + )), + #[should_panic] certify_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() + )), + #[should_panic] extend: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() + )), + #[should_panic] delete_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: true }, + false, + 42, + 314, + event_id_for_testing(), + )), + #[should_panic] delete_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: false }, + false, + 42, + 314, + event_id_for_testing(), + )), + ] + } + fn test_merge_new_expected_failure_cases(operand: BlobInfoMergeOperand) { + let _ = BlobInfoV2::merge_new(operand); + } + + param_test! { + test_merge_new_expected_success_cases_invariants: [ + register_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, false, 42, 314, event_id_for_testing() + )), + register_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, true, 42, 314, event_id_for_testing() + )), + invalidate: (BlobInfoMergeOperand::MarkInvalid { + epoch: 0, + status_event: event_id_for_testing() + }), + metadata_true: (BlobInfoMergeOperand::MarkMetadataStored(true)), + metadata_false: (BlobInfoMergeOperand::MarkMetadataStored(false)), + ] + } + fn test_merge_new_expected_success_cases_invariants(operand: BlobInfoMergeOperand) { + let blob_info = BlobInfoV2::merge_new(operand).expect("should be some"); + check_invariants(&blob_info); + } + + param_test! { + test_invalid_status_is_not_changed: [ + invalidate: (BlobInfoMergeOperand::MarkInvalid { + epoch: 0, + status_event: event_id_for_testing() + }), + metadata_true: (BlobInfoMergeOperand::MarkMetadataStored(true)), + metadata_false: (BlobInfoMergeOperand::MarkMetadataStored(false)), + register_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, false, 42, 314, event_id_for_testing() + )), + register_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, true, 42, 314, event_id_for_testing() + )), + certify_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, false, 42, 314, event_id_for_testing() + )), + certify_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() + )), + extend: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() + )), + delete_true: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: true }, + false, + 42, + 314, + event_id_for_testing(), + )), + delete_false: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: false }, + false, + 42, + 314, + event_id_for_testing(), + )), + ] + } + fn test_invalid_status_is_not_changed(operand: BlobInfoMergeOperand) { + let blob_info = BlobInfoV2::Invalid { + epoch: 42, + event: event_id_for_testing(), + }; + assert_eq!(blob_info, blob_info.clone().merge_with(operand)); + } + + param_test! { + test_mark_metadata_stored_keeps_everything_else_unchanged: [ + default: (Default::default()), + deletable: (ValidBlobInfoV2{count_deletable_total: 2, ..Default::default()}), + deletable_certified: (ValidBlobInfoV2{ + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(0), + ..Default::default() + }), + permanent: (ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), + ..Default::default() + }), + permanent_certified: (ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + initial_certified_epoch: Some(1), + ..Default::default() + }), + ] + } + fn test_mark_metadata_stored_keeps_everything_else_unchanged( + preexisting_info: ValidBlobInfoV2, + ) { + preexisting_info + .check_invariants() + .expect("preexisting blob info invariants violated"); + let expected_updated_info = ValidBlobInfoV2 { + is_metadata_stored: true, + ..preexisting_info.clone() + }; + expected_updated_info + .check_invariants() + .expect("expected updated blob info invariants violated"); + + let updated_info = BlobInfoV2::Valid(preexisting_info) + .merge_with(BlobInfoMergeOperand::MarkMetadataStored(true)); + + assert_eq!(updated_info, expected_updated_info.into()); + } + + param_test! { + test_mark_invalid_marks_everything_invalid: [ + default: (Default::default()), + deletable: (ValidBlobInfoV2{count_deletable_total: 2, ..Default::default()}), + deletable_certified: (ValidBlobInfoV2{ + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(0), + ..Default::default() + }), + permanent: (ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), + ..Default::default() + }), + permanent_certified: (ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + initial_certified_epoch: Some(1), + ..Default::default() + }), + ] + } + fn test_mark_invalid_marks_everything_invalid(preexisting_info: ValidBlobInfoV2) { + let preexisting_info = preexisting_info.into(); + check_invariants(&preexisting_info); + let event = event_id_for_testing(); + let updated_info = preexisting_info.merge_with(BlobInfoMergeOperand::MarkInvalid { + epoch: 2, + status_event: event, + }); + assert_eq!(BlobInfoV2::Invalid { epoch: 2, event }, updated_info); + } + + param_test! { + test_merge_preexisting_expected_successes: [ + register_first_deletable: ( + Default::default(), + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, true, 1, 2, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 1, + latest_seen_deletable_registered_end_epoch: Some(2), + ..Default::default() + }, + ), + register_additional_deletable1: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + latest_seen_deletable_registered_end_epoch: Some(2), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, true, 1, 5, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 4, + latest_seen_deletable_registered_end_epoch: Some(5), + ..Default::default() + }, + ), + register_additional_deletable2: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + latest_seen_deletable_registered_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, true, 1, 3, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 4, + latest_seen_deletable_registered_end_epoch: Some(4), + ..Default::default() + }, + ), + certify_first_deletable: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + latest_seen_deletable_registered_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 1, 4, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(4), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + ), + certify_additional_deletable1: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 1, + initial_certified_epoch: Some(0), + latest_seen_deletable_registered_end_epoch: Some(4), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 1, 2, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 2, + initial_certified_epoch: Some(0), + latest_seen_deletable_registered_end_epoch: Some(4), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + ), + certify_additional_deletable2: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 0, 5, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 2, + initial_certified_epoch: Some(0), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(5), + ..Default::default() + }, + ), + register_first_permanent: ( + ValidBlobInfoV2{ + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, false, 1, 2, fixed_event_id_for_testing(0) + ), + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + ..Default::default() + }, + ), + extend_deletable: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 1, + initial_certified_epoch: Some(0), + latest_seen_deletable_registered_end_epoch: Some(4), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Extend, true, 3, 42, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 1, + initial_certified_epoch: Some(0), + latest_seen_deletable_registered_end_epoch: Some(42), + latest_seen_deletable_certified_end_epoch: Some(42), + ..Default::default() + }, + ), + extend_permanent: ( + ValidBlobInfoV2{ + initial_certified_epoch: Some(0), + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 4, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 4, 1)), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Extend, false, 3, 42, fixed_event_id_for_testing(2) + ), + ValidBlobInfoV2{ + initial_certified_epoch: Some(0), + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 2)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 42, 2)), + ..Default::default() + }, + ), + certify_outdated_deletable: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(8), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 4, 6, event_id_for_testing() + ), + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 2, + initial_certified_epoch: Some(4), + latest_seen_deletable_registered_end_epoch: Some(8), + latest_seen_deletable_certified_end_epoch: Some(6), + ..Default::default() + }, + ), + certify_outdated_permanent: ( + ValidBlobInfoV2{ + initial_certified_epoch: Some(2), + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_for_testing(1, 5)), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, false, 7, 42, fixed_event_id_for_testing(1) + ), + ValidBlobInfoV2{ + initial_certified_epoch: Some(7), + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 1)), + ..Default::default() + }, + ), + register_additional_permanent: ( + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, false, 2, 3, fixed_event_id_for_testing(1) + ), + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 1)), + ..Default::default() + }, + ), + expire_permanent_blob: ( + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(3, 5, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 1)), + initial_certified_epoch: Some(1), + ..Default::default() + }, + BlobInfoMergeOperand::PermanentExpired { + was_certified: true, + }, + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 1)), + initial_certified_epoch: Some(1), + ..Default::default() + }, + ), + expire_last_permanent_blob: ( + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 1)), + initial_certified_epoch: Some(1), + ..Default::default() + }, + BlobInfoMergeOperand::PermanentExpired { + was_certified: true, + }, + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 0)), + permanent_certified: None, + initial_certified_epoch: None, + ..Default::default() + }, + ), + delete_deletable_blob: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 2, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: true }, + true, + 1, + 6, + event_id_for_testing(), + ), + ValidBlobInfoV2{ + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + ), + expire_deletable_blob: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + count_deletable_certified: 2, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::DeletableExpired { + was_certified: true, + }, + ValidBlobInfoV2{ + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + ), + delete_last_deletable_blob: ( + ValidBlobInfoV2{ + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: true }, + true, + 1, + 4, + event_id_for_testing(), + ), + ValidBlobInfoV2{ + count_deletable_total: 1, + count_deletable_certified: 0, + initial_certified_epoch: None, + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: None, + ..Default::default() + }, + ), + expire_last_deletable_blob: ( + ValidBlobInfoV2{ + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(1), + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: Some(4), + ..Default::default() + }, + BlobInfoMergeOperand::DeletableExpired { + was_certified: true, + }, + ValidBlobInfoV2{ + count_deletable_total: 1, + count_deletable_certified: 0, + initial_certified_epoch: None, + latest_seen_deletable_registered_end_epoch: Some(5), + latest_seen_deletable_certified_end_epoch: None, + ..Default::default() + }, + ), + expire_uncertified_permanent_blob: ( + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(3, 5, 0)), + ..Default::default() + }, + BlobInfoMergeOperand::PermanentExpired { + was_certified: false, + }, + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 0)), + ..Default::default() + }, + ), + expire_last_uncertified_permanent_blob: ( + ValidBlobInfoV2{ + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 0)), + ..Default::default() + }, + BlobInfoMergeOperand::PermanentExpired { + was_certified: false, + }, + ValidBlobInfoV2{ + permanent_total: None, + ..Default::default() + }, + ), + delete_uncertified_deletable_blob: ( + ValidBlobInfoV2{ + count_deletable_total: 3, + latest_seen_deletable_registered_end_epoch: Some(5), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: false }, + true, + 1, + 6, + event_id_for_testing(), + ), + ValidBlobInfoV2{ + count_deletable_total: 2, + latest_seen_deletable_registered_end_epoch: Some(5), + ..Default::default() + }, + ), + delete_last_uncertified_deletable_blob: ( + ValidBlobInfoV2{ + count_deletable_total: 1, + latest_seen_deletable_registered_end_epoch: Some(5), + ..Default::default() + }, + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: false }, + true, + 2, + 6, + event_id_for_testing(), + ), + ValidBlobInfoV2{ + count_deletable_total: 0, + latest_seen_deletable_registered_end_epoch: None, + ..Default::default() + }, + ), + ] + } + fn test_merge_preexisting_expected_successes( + preexisting_info: ValidBlobInfoV2, + operand: BlobInfoMergeOperand, + expected_info: ValidBlobInfoV2, + ) { + preexisting_info + .check_invariants() + .expect("preexisting blob info invariants violated"); + expected_info + .check_invariants() + .expect("expected blob info invariants violated"); + + let updated_info = BlobInfoV2::Valid(preexisting_info).merge_with(operand); + + assert_eq!(updated_info, expected_info.into()); + } + + param_test! { + test_merge_preexisting_expected_failures: [ + certify_permanent_without_register: ( + Default::default(), + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, false, 42, 314, event_id_for_testing() + ), + ), + extend_permanent_without_certify: ( + Default::default(), + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() + ), + ), + certify_deletable_without_register: ( + Default::default(), + BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() + ), + ), + ] + } + fn test_merge_preexisting_expected_failures( + preexisting_info: ValidBlobInfoV2, + operand: BlobInfoMergeOperand, + ) { + preexisting_info + .check_invariants() + .expect("preexisting blob info invariants violated"); + let preexisting_info = BlobInfoV2::Valid(preexisting_info); + let blob_info = preexisting_info.clone().merge_with(operand); + assert_eq!(preexisting_info, blob_info); + } + + param_test! { + test_blob_status_is_inexistent_for_expired_blobs: [ + expired_permanent_registered_0: ( + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + ..Default::default() + }, + 1, + 2, + ), + expired_permanent_registered_1: ( + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), + ..Default::default() + }, + 2, + 4, + ), + expired_permanent_certified: ( + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 2, 0)), + permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + ..Default::default() + }, + 1, + 2, + ), + expired_deletable_registered: ( + ValidBlobInfoV2 { + count_deletable_total: 1, + latest_seen_deletable_registered_end_epoch: Some(2), + ..Default::default() + }, + 1, + 2, + ), + expired_deletable_certified: ( + ValidBlobInfoV2 { + count_deletable_total: 1, + latest_seen_deletable_registered_end_epoch: Some(2), + count_deletable_certified: 1, + latest_seen_deletable_certified_end_epoch: Some(2), + ..Default::default() + }, + 1, + 2, + ), + ] + } + fn test_blob_status_is_inexistent_for_expired_blobs( + blob_info: ValidBlobInfoV2, + epoch_not_expired: Epoch, + epoch_expired: Epoch, + ) { + assert_ne!( + BlobInfoV2::Valid(blob_info.clone()).to_blob_status(epoch_not_expired), + BlobStatus::Nonexistent, + ); + assert_eq!( + BlobInfoV2::Valid(blob_info).to_blob_status(epoch_expired), + BlobStatus::Nonexistent, + ); + } +} From 6a19cf94e8f7af6c22b847d65b99df8b5b87de91 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 12 Mar 2026 02:28:17 -0700 Subject: [PATCH 3/7] feat: implement V2 blob info with storage pool support Co-Authored-By: Claude Opus 4.6 --- .../src/node/storage/blob_info.rs | 195 +- .../node/storage/blob_info/blob_info_v1.rs | 32 +- .../node/storage/blob_info/blob_info_v2.rs | 1749 ++++++++--------- 3 files changed, 954 insertions(+), 1022 deletions(-) diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index 5493883266..b89357b532 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -30,10 +30,10 @@ use walrus_storage_node_client::api::BlobStatus; use walrus_sui::types::{BlobCertified, BlobDeleted, BlobEvent, BlobRegistered, InvalidBlobId}; #[cfg(test)] -pub(crate) use self::blob_info_v1::PermanentBlobInfoV1; +pub(crate) use self::blob_info_v1::{PermanentBlobInfoV1, ValidBlobInfoV1}; use self::per_object_blob_info::PerObjectBlobInfoMergeOperand; pub(crate) use self::{ - blob_info_v1::{BlobInfoV1, ValidBlobInfoV1}, + blob_info_v1::BlobInfoV1, blob_info_v2::BlobInfoV2, per_object_blob_info::{PerObjectBlobInfo, PerObjectBlobInfoApi}, }; @@ -605,10 +605,13 @@ impl BlobInfoTable { .safe_iter_with_snapshot(&snapshot) .context("failed to create per-object blob info snapshot iterator")? { - let Ok((object_id, PerObjectBlobInfo::V1(per_object_blob_info))) = result else { - return Err(anyhow::anyhow!( - "error encountered while iterating over per-object blob info: {result:?}" - )); + let (object_id, per_object_blob_info) = match result { + Ok(v) => v, + Err(e) => { + return Err(anyhow::anyhow!( + "error encountered while iterating over per-object blob info: {e:?}" + )); + } }; let blob_id = per_object_blob_info.blob_id(); per_object_table_blob_ids.insert(blob_id); @@ -621,18 +624,51 @@ impl BlobInfoTable { per-object blob info entry exists (object ID: {object_id})" )); }; - let BlobInfo::V1(BlobInfoV1::Valid(ValidBlobInfoV1 { + + // Extract deletable counts for cross-checking (works for both V1 and V2). + let ( + v1_blob, count_deletable_total, count_deletable_certified, latest_seen_deletable_registered_end_epoch, latest_seen_deletable_certified_end_epoch, - .. - })) = blob_info - else { - continue; + ) = match &blob_info { + BlobInfo::V1(BlobInfoV1::Valid(v)) => ( + true, + v.count_deletable_total, + v.count_deletable_certified, + v.latest_seen_deletable_registered_end_epoch, + v.latest_seen_deletable_certified_end_epoch, + ), + BlobInfo::V2(BlobInfoV2::Valid(v)) => { + anyhow::ensure!( + !matches!(blob_info, BlobInfo::V1(_)), + "V2 per-object blob info requires V2 aggregate blob info, but found V1" + ); + ( + false, + v.count_deletable_total, + v.count_deletable_certified, + // V2 doesn't track end epochs; skip epoch-based assertions. + None, + None, + ) + } + _ => continue, }; - if per_object_blob_info.is_deletable() { - let per_object_end_epoch = per_object_blob_info.end_epoch; + + // Below checks the invariants of last seen epochs on deletable blobs, which is only + // relevant for V1 per-object entries. + if v1_blob && per_object_blob_info.is_deletable() { + let per_object_end_epoch = match &per_object_blob_info { + PerObjectBlobInfo::V1(v) => v.end_epoch, + PerObjectBlobInfo::V2(v) => match v.end_epoch_info { + per_object_blob_info::EndEpochInfo::Individual(e) => e, + per_object_blob_info::EndEpochInfo::StoragePool(_) => { + unreachable!("storage_pool_id() returned None above") + } + }, + }; anyhow::ensure!( count_deletable_total > 0, "count_deletable_total is 0 for blob ID {blob_id}, even though a deletable \ @@ -646,7 +682,13 @@ impl BlobInfoTable { end epoch of a deletable blob object: {per_object_end_epoch} (object ID: \ {object_id})" ); - if per_object_blob_info.certified_epoch.is_some() { + let certified_epoch = match &per_object_blob_info { + PerObjectBlobInfo::V1(v) => v.certified_epoch, + PerObjectBlobInfo::V2(_) => { + unreachable!("v2 blob should not check latest seen epochs") + } + }; + if certified_epoch.is_some() { anyhow::ensure!( count_deletable_certified > 0, "count_deletable_certified is 0 for blob ID {blob_id}, even though a \ @@ -674,18 +716,34 @@ impl BlobInfoTable { "error encountered while iterating over aggregate blob info: {result:?}" )); }; - let BlobInfo::V1(BlobInfoV1::Valid(blob_info)) = blob_info else { - continue; - }; - if !blob_info.has_no_objects() && !per_object_table_blob_ids.contains(&blob_id) { - return Err(anyhow::anyhow!( - "per-object blob info not found for blob ID {blob_id}, even though a \ - valid aggregate blob info entry referencing objects exists: {blob_info:?}" - )); + match &blob_info { + BlobInfo::V1(BlobInfoV1::Valid(v1_info)) => { + if !v1_info.has_no_objects() && !per_object_table_blob_ids.contains(&blob_id) { + return Err(anyhow::anyhow!( + "per-object blob info not found for blob ID {blob_id}, even though a \ + valid V1 aggregate blob info entry referencing objects exists: \ + {v1_info:?}" + )); + } + v1_info.check_invariants().context(format!( + "aggregate blob info invariants violated for blob ID {blob_id}" + ))?; + } + BlobInfo::V2(BlobInfoV2::Valid(v2_info)) => { + if !v2_info.has_no_objects() && !per_object_table_blob_ids.contains(&blob_id) { + return Err(anyhow::anyhow!( + "per-object blob info not found for blob ID {blob_id}, even though a \ + valid V2 aggregate blob info entry referencing objects exists: \ + {v2_info:?}" + )); + } + v2_info.check_invariants().context(format!( + "aggregate blob info V2 invariants violated for blob ID {blob_id}" + ))?; + } + // Invalid blob info. + _ => continue, } - blob_info.check_invariants().context(format!( - "aggregate blob info invariants violated for blob ID {blob_id}" - ))?; } Ok(()) } @@ -801,7 +859,7 @@ where } } -pub(super) trait ToBytes: Serialize + Sized { +pub(crate) trait ToBytes: Serialize + Sized { /// Converts the value to a `Vec`. /// /// Uses BCS encoding (which is assumed to succeed) by default. @@ -809,7 +867,7 @@ pub(super) trait ToBytes: Serialize + Sized { bcs::to_bytes(self).expect("value must be BCS-serializable") } } -trait Mergeable: ToBytes + Debug + DeserializeOwned + Serialize + Sized { +pub(crate) trait Mergeable: ToBytes + Debug + DeserializeOwned + Serialize + Sized { type MergeOperand: Debug + DeserializeOwned + ToBytes; type Key: Debug + DeserializeOwned + std::fmt::Display; @@ -883,16 +941,16 @@ pub(crate) trait BlobInfoApi: CertifiedBlobInfoApi { } #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(super) struct BlobStatusChangeInfo { - pub(super) blob_id: BlobId, - pub(super) deletable: bool, - pub(super) epoch: Epoch, - pub(super) end_epoch: Epoch, - pub(super) status_event: EventID, +pub(crate) struct BlobStatusChangeInfo { + pub(crate) blob_id: BlobId, + pub(crate) deletable: bool, + pub(crate) epoch: Epoch, + pub(crate) end_epoch: Epoch, + pub(crate) status_event: EventID, } #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Copy)] -pub(super) enum BlobStatusChangeType { +pub(crate) enum BlobStatusChangeType { Register, Certify, // INV: Can only be applied to a certified blob. @@ -900,6 +958,17 @@ pub(super) enum BlobStatusChangeType { Delete { was_certified: bool }, } +/// Change info for storage pool blob events. +/// +/// Unlike `BlobStatusChangeInfo`, this does not carry `end_epoch` because the lifetime of pool +/// blobs is determined by the pool's end_epoch (tracked in `storage_pool_end_epochs`), not by +/// the individual blob's end_epoch at registration time. +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] +pub(crate) struct PooledBlobChangeInfo { + pub(crate) epoch: Epoch, + pub(crate) storage_pool_id: ObjectID, +} + trait ChangeTypeAndInfo { fn change_type(&self) -> BlobStatusChangeType; fn change_info(&self) -> BlobStatusChangeInfo; @@ -960,7 +1029,7 @@ impl ChangeTypeAndInfo for BlobDeleted { } #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(super) enum BlobInfoMergeOperand { +pub(crate) enum BlobInfoMergeOperand { MarkMetadataStored(bool), MarkInvalid { epoch: Epoch, @@ -978,6 +1047,16 @@ pub(super) enum BlobInfoMergeOperand { PermanentExpired { was_certified: bool, }, + /// A status change for a blob in a storage pool. + PooledBlobChangeStatus { + change_type: BlobStatusChangeType, + change_info: PooledBlobChangeInfo, + }, + /// A blob in a storage pool has expired (pool lifetime ended). + PoolExpired { + storage_pool_id: ObjectID, + was_certified: bool, + }, } impl ToBytes for BlobInfoMergeOperand {} @@ -1101,8 +1180,6 @@ impl BlobInfo { certified_epoch: Option, invalidated_epoch: Option, ) -> Self { - use blob_info_v1::PermanentBlobInfoV1; - let blob_info = match status { BlobCertificationStatus::Invalid => BlobInfoV1::Invalid { epoch: invalidated_epoch @@ -1136,18 +1213,46 @@ impl Mergeable for BlobInfo { fn merge_with(self, operand: Self::MergeOperand) -> Self { match self { - Self::V1(value) => Self::V1(value.merge_with(operand)), - Self::V2(value) => Self::V2(value.merge_with(operand)), + Self::V1(v1) => match &operand { + // Only upgrade to V2 when a pool operand hits a V1 entry. + // This makes sure that all the storage nodes have consistent behavior despite of + // when they upgrade their nodes. + BlobInfoMergeOperand::PooledBlobChangeStatus { .. } + // Although it's impossible to have pool expired as the first pool event for a + // blob, the internal processing of pool expired will return error. + | BlobInfoMergeOperand::PoolExpired { .. } => { + Self::V2(BlobInfoV2::from(v1).merge_with(operand)) + } + // Regular operands keep V1 as V1. + _ => Self::V1(v1.merge_with(operand)), + }, + // V2 handles all operand types (regular AND pool). + Self::V2(v2) => Self::V2(v2.merge_with(operand)), } } fn merge_new(operand: Self::MergeOperand) -> Option { - BlobInfoV1::merge_new(operand).map(Self::from) + match &operand { + // First event for a blob_id is a pool event → create V2. + BlobInfoMergeOperand::PooledBlobChangeStatus { .. } => { + BlobInfoV2::merge_new(operand).map(Self::V2) + } + // Pool expired is not a valid first event for a blob_id. Adding a branch here to + // account for possible race condition where pool expire event comes after an explicit + // blob delete event. + BlobInfoMergeOperand::PoolExpired { .. } => None, + // First event is a regular event → create V1 (as before). + _ => BlobInfoV1::merge_new(operand).map(Self::V1), + } } } mod per_object_blob_info { use super::*; + pub(crate) use super::{ + blob_info_v1::PerObjectBlobInfoV1, + blob_info_v2::{EndEpochInfo, PerObjectBlobInfoV2}, + }; #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) struct PerObjectBlobInfoMergeOperand { @@ -1198,6 +1303,8 @@ mod per_object_blob_info { fn is_registered(&self, current_epoch: Epoch) -> bool; /// Returns true iff the object is already deleted. fn is_deleted(&self) -> bool; + /// Returns the storage pool ID if this is a storage pool blob. + fn storage_pool_id(&self) -> Option; } #[enum_dispatch(CertifiedBlobInfoApi)] @@ -1245,11 +1352,15 @@ mod per_object_blob_info { } fn merge_new(operand: Self::MergeOperand) -> Option { + // We never create PerObjectBlobInfoV2 via merge operator. This is because the + // PerObjectBlobInfoMergeOperand struct can only used for V1 for registration. + // So any newly created PerObjectBlobInfoV2 is directly inserted into the table. + // + // The certify and delete operation can be used in both V1 and V2, so merge_with above + // works for both. PerObjectBlobInfoV1::merge_new(operand).map(Self::from) } } - - pub(crate) use super::{blob_info_v1::PerObjectBlobInfoV1, blob_info_v2::PerObjectBlobInfoV2}; } fn deserialize_from_db<'de, T>(data: &'de [u8]) -> Option diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs index 32c12e8208..e15167650b 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs @@ -310,7 +310,7 @@ impl ValidBlobInfoV1 { /// Processes a register status change on the [`Option`] object /// representing all permanent blobs. - fn register_permanent( + pub(crate) fn register_permanent( permanent_total: &mut Option, change_info: &BlobStatusChangeInfo, ) { @@ -321,7 +321,7 @@ impl ValidBlobInfoV1 { /// and the certified permanent blobs. /// /// Returns whether the update was successful. - fn certify_permanent( + pub(crate) fn certify_permanent( permanent_total: &Option, permanent_certified: &mut Option, change_info: &BlobStatusChangeInfo, @@ -356,7 +356,7 @@ impl ValidBlobInfoV1 { /// Processes an extend status change on the [`PermanentBlobInfoV1`] object representing the /// certified permanent blobs. - fn extend_permanent( + pub(crate) fn extend_permanent( permanent_info: &mut Option, change_info: &BlobStatusChangeInfo, ) { @@ -380,7 +380,7 @@ impl ValidBlobInfoV1 { self.maybe_unset_initial_certified_epoch(); } - fn decrement_blob_info_inner(blob_info_inner: &mut Option) { + pub(crate) fn decrement_blob_info_inner(blob_info_inner: &mut Option) { match blob_info_inner { None => tracing::error!("attempt to delete a permanent blob when none is tracked"), Some(PermanentBlobInfoV1 { count, .. }) => { @@ -472,7 +472,7 @@ impl PermanentBlobInfoV1 { /// # Panics /// /// Panics if the change info has `deletable == true`. - fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { + pub(crate) fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { assert!(!change_info.deletable); if increase_count { @@ -514,7 +514,7 @@ impl PermanentBlobInfoV1 { } #[cfg(test)] - fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { + pub(crate) fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { Self { count: NonZeroU32::new(count).expect("count must be non-zero"), end_epoch, @@ -523,7 +523,7 @@ impl PermanentBlobInfoV1 { } #[cfg(test)] - fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { + pub(crate) fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { Self { count: NonZeroU32::new(count).expect("count must be non-zero"), end_epoch, @@ -664,6 +664,11 @@ impl Mergeable for BlobInfoV1 { Self::Valid(valid_blob_info), BlobInfoMergeOperand::PermanentExpired { was_certified }, ) => valid_blob_info.permanent_expired(was_certified), + // Pool operands should never reach V1 — they are intercepted at the BlobInfo level. + (_, BlobInfoMergeOperand::PooledBlobChangeStatus { .. }) + | (_, BlobInfoMergeOperand::PoolExpired { .. }) => { + unreachable!("pool operands should be handled in BlobInfoV2") + } } self } @@ -732,10 +737,19 @@ impl Mergeable for BlobInfoV1 { ); None } + // Pool operands should never reach V1 — they are intercepted at the BlobInfo level. + BlobInfoMergeOperand::PooledBlobChangeStatus { .. } + | BlobInfoMergeOperand::PoolExpired { .. } => { + unreachable!("pool operands should be handled in BlobInfoV2") + } } } } +// ============================================================================= +// Per-object blob info V1 +// ============================================================================= + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) struct PerObjectBlobInfoV1 { /// The blob ID. @@ -783,6 +797,10 @@ impl PerObjectBlobInfoApi for PerObjectBlobInfoV1 { fn is_deleted(&self) -> bool { self.deleted } + + fn storage_pool_id(&self) -> Option { + None + } } impl ToBytes for PerObjectBlobInfoV1 {} diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs index ce3f7c6c1b..642645dd61 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs @@ -1,11 +1,7 @@ // Copyright (c) Walrus Foundation // SPDX-License-Identifier: Apache-2.0 -//! V2 blob info types and merge logic. - -#![allow(dead_code)] - -use std::num::NonZeroU32; +//! V2 blob info types and merge logic, supporting both regular blobs and storage pool blobs. use serde::{Deserialize, Serialize}; use sui_types::{base_types::ObjectID, event::EventID}; @@ -20,80 +16,74 @@ use super::{ CertifiedBlobInfoApi, Mergeable, ToBytes, + blob_info_v1::{BlobInfoV1, PermanentBlobInfoV1, ValidBlobInfoV1}, per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, }; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) enum BlobInfoV2 { - Invalid { epoch: Epoch, event: EventID }, - Valid(ValidBlobInfoV2), -} - -impl ToBytes for BlobInfoV2 {} - -// INV: count_deletable_total >= count_deletable_certified -// INV: permanent_total.is_none() => permanent_certified.is_none() -// INV: permanent_total.count >= permanent_certified.count -// INV: permanent_total.end_epoch >= permanent_certified.end_epoch -// INV: initial_certified_epoch.is_some() -// <=> count_deletable_certified > 0 || permanent_certified.is_some() -// INV: latest_seen_deletable_registered_end_epoch >= latest_seen_deletable_certified_end_epoch -// See the `check_invariants` method for more details. -// Important: This struct MUST NOT be changed. Instead, if needed, a new `ValidBlobInfoV3` struct -// should be created. +/// V2 aggregate blob info that supports both regular blobs and storage pool blobs. +/// +/// Regular blob fields are almost identical to V1, except that we no longer track +/// highest seen end epochs for deletable blobs. Highest seen end epoch is an optimization to give +/// better hint about whether a deletable blob may be alive or not. However, it may not give +/// accurate answer given ongoing GC. So we removed it from V2 and rely on GC to keep blob +/// epoch status up to date. +/// +/// Storage pool blobs are tracked via flat counters, similar to deletable blobs. +// +// INV: same invariants as V1 for regular fields, plus: +// INV: count_pooled_refs_total >= count_pooled_refs_certified +// INV: initial_certified_epoch considers both regular AND pool certified counts #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub(crate) struct ValidBlobInfoV2 { + // Common fields for both regular and storage pool blobs. pub is_metadata_stored: bool, + pub initial_certified_epoch: Option, + + // Regular blob fields (same as V1). pub count_deletable_total: u32, pub count_deletable_certified: u32, - pub permanent_total: Option, - pub permanent_certified: Option, - pub initial_certified_epoch: Option, + pub permanent_total: Option, + pub permanent_certified: Option, - // Note: The following helper fields were used in the past to approximate the blob status for - // deletable blobs. They are still used for the following reason: When deletable blobs expire, - // we update the aggregate blob info in the background. So there is a delay between an epoch - // change and the blob info being updated. During this delay, these fields are useful to - // determine that a blob is already expired. - /// The latest end epoch recorded for a registered deletable blob. - pub latest_seen_deletable_registered_end_epoch: Option, - /// The latest end epoch recorded for a certified deletable blob. - pub latest_seen_deletable_certified_end_epoch: Option, + // Storage pool references counters. + pub count_pooled_refs_total: u32, + pub count_pooled_refs_certified: u32, } -impl From for BlobInfoV2 { - fn from(value: ValidBlobInfoV2) -> Self { - Self::Valid(value) +// Conversion from V1 to V2. This is needed because even when storage pool is supported, we still +// create V1 blob info if the blob is a regular blob. Only after seen a pooled blob event, we then +// upgrade to V2. This makes all the storage nodes to have consistent behavior despite of when +// they upgrade their nodes. +impl From for ValidBlobInfoV2 { + fn from(v1: ValidBlobInfoV1) -> Self { + Self { + is_metadata_stored: v1.is_metadata_stored, + count_deletable_total: v1.count_deletable_total, + count_deletable_certified: v1.count_deletable_certified, + permanent_total: v1.permanent_total, + permanent_certified: v1.permanent_certified, + initial_certified_epoch: v1.initial_certified_epoch, + count_pooled_refs_total: 0, + count_pooled_refs_certified: 0, + } } } impl ValidBlobInfoV2 { fn to_blob_status(&self, current_epoch: Epoch) -> BlobStatus { - // The check of the `latest_seen_*_epoch` fields is there to reduce the cases where we - // report the existence of deletable blobs that are already expired. - let count_deletable_total = if self - .latest_seen_deletable_registered_end_epoch - .is_some_and(|e| e > current_epoch) - { - self.count_deletable_total - } else { - Default::default() - }; - let count_deletable_certified = if self - .latest_seen_deletable_certified_end_epoch - .is_some_and(|e| e > current_epoch) - { - self.count_deletable_certified - } else { - Default::default() - }; + let initial_certified_epoch = self.initial_certified_epoch; + + // Deletable counts include both regular and pool refs. let deletable_counts = DeletableCounts { - count_deletable_total, - count_deletable_certified, + count_deletable_total: self + .count_deletable_total + .saturating_add(self.count_pooled_refs_total), + count_deletable_certified: self + .count_deletable_certified + .saturating_add(self.count_pooled_refs_certified), }; - let initial_certified_epoch = self.initial_certified_epoch; - if let Some(PermanentBlobInfoV2 { + if let Some(PermanentBlobInfoV1 { end_epoch, event, .. }) = self.permanent_certified.as_ref() && *end_epoch > current_epoch @@ -106,7 +96,7 @@ impl ValidBlobInfoV2 { initial_certified_epoch, }; } - if let Some(PermanentBlobInfoV2 { + if let Some(PermanentBlobInfoV1 { end_epoch, event, .. }) = self.permanent_total.as_ref() && *end_epoch > current_epoch @@ -130,45 +120,39 @@ impl ValidBlobInfoV2 { } } - // The counts of deletable blobs are decreased when they expire. However, because this only - // happens in the background, it is possible during a short time period at the beginning of an - // epoch that all deletable blobs expired but the counts are not updated yet. For this case, we - // use the `latest_seen_deletable_certified_end_epoch` field as an additional check. + // The counts of deletable and pooled blobs are decreased when they expire. However, because + // this only happens in the background, it is possible during a short time period at the + // beginning of an epoch that all deletable and pooled blobs expired but the counts are not + // updated yet. // - // There is still a corner case where the blob with the - // `latest_seen_deletable_certified_end_epoch` was deleted and all other blobs expired. In this - // case, we would nevertheless return `true` during a short time period, before the counts are - // updated in the background. + // Given that GC per object blob info table is expected to be very fast (in seconds), this + // window is expected to be very short. fn is_certified(&self, current_epoch: Epoch) -> bool { let exists_certified_permanent_blob = self .permanent_certified .as_ref() .is_some_and(|p| p.end_epoch > current_epoch); - let probably_exists_certified_deletable_blob = self.count_deletable_certified > 0 - && self - .latest_seen_deletable_certified_end_epoch - .is_some_and(|e| e > current_epoch); + + // Note that at the beginning of the epoch before GC runs, newly expired deletable blob or + // pooled blob's certified counter may still be non-zero, but the blob is already expired. + // This will make the blob appears to be certified until GC finishes. self.initial_certified_epoch .is_some_and(|epoch| epoch <= current_epoch) - && (exists_certified_permanent_blob || probably_exists_certified_deletable_blob) + && (exists_certified_permanent_blob + || self.count_deletable_certified > 0 + || self.count_pooled_refs_certified > 0) } pub(crate) fn has_no_objects(&self) -> bool { - matches!( - self, - Self { - is_metadata_stored: _, - count_deletable_total: 0, - count_deletable_certified: 0, - permanent_total: None, - permanent_certified: None, - initial_certified_epoch: None, - latest_seen_deletable_registered_end_epoch: None, - latest_seen_deletable_certified_end_epoch: None, - } - ) + self.count_deletable_total == 0 + && self.count_deletable_certified == 0 + && self.permanent_total.is_none() + && self.permanent_certified.is_none() + && self.initial_certified_epoch.is_none() + && self.count_pooled_refs_total == 0 } + /// Handles regular blob status changes (same logic as V1). #[tracing::instrument] fn update_status( &mut self, @@ -176,11 +160,12 @@ impl ValidBlobInfoV2 { change_info: BlobStatusChangeInfo, ) { let was_certified = self.is_certified(change_info.epoch); + // This should be the same as V1, except that we no longer track end epochs for deletable + // blobs. if change_info.deletable { match change_type { BlobStatusChangeType::Register => { self.count_deletable_total += 1; - self.maybe_increase_latest_deletable_registered_epoch(change_info.end_epoch); } BlobStatusChangeType::Certify => { if self.count_deletable_total <= self.count_deletable_certified { @@ -190,23 +175,22 @@ impl ValidBlobInfoV2 { return; } self.count_deletable_certified += 1; - self.maybe_increase_latest_deletable_certified_epoch(change_info.end_epoch); } BlobStatusChangeType::Extend => { - self.maybe_increase_latest_deletable_registered_epoch(change_info.end_epoch); - self.maybe_increase_latest_deletable_certified_epoch(change_info.end_epoch); + // No-op for V2 deletable blobs (no end epoch tracking). } BlobStatusChangeType::Delete { was_certified } => { - self.update_deletable_counters_and_end_epochs(was_certified); + self.update_deletable_counters(was_certified); } } } else { + // These should be the same as V1. match change_type { BlobStatusChangeType::Register => { - Self::register_permanent(&mut self.permanent_total, &change_info); + ValidBlobInfoV1::register_permanent(&mut self.permanent_total, &change_info); } BlobStatusChangeType::Certify => { - if !Self::certify_permanent( + if !ValidBlobInfoV1::certify_permanent( &self.permanent_total, &mut self.permanent_certified, &change_info, @@ -216,8 +200,8 @@ impl ValidBlobInfoV2 { } } BlobStatusChangeType::Extend => { - Self::extend_permanent(&mut self.permanent_total, &change_info); - Self::extend_permanent(&mut self.permanent_certified, &change_info); + ValidBlobInfoV1::extend_permanent(&mut self.permanent_total, &change_info); + ValidBlobInfoV1::extend_permanent(&mut self.permanent_certified, &change_info); } BlobStatusChangeType::Delete { .. } => { tracing::error!("attempt to delete a permanent blob"); @@ -240,20 +224,14 @@ impl ValidBlobInfoV2 { } fn deletable_expired(&mut self, was_certified: bool) { - self.update_deletable_counters_and_end_epochs(was_certified); + self.update_deletable_counters(was_certified); self.maybe_unset_initial_certified_epoch(); } - fn update_deletable_counters_and_end_epochs(&mut self, was_certified: bool) { - Self::decrement_deletable_counter_and_maybe_unset_latest_end_epoch( - &mut self.count_deletable_total, - &mut self.latest_seen_deletable_registered_end_epoch, - ); + fn update_deletable_counters(&mut self, was_certified: bool) { + self.count_deletable_total = self.count_deletable_total.saturating_sub(1); if was_certified { - Self::decrement_deletable_counter_and_maybe_unset_latest_end_epoch( - &mut self.count_deletable_certified, - &mut self.latest_seen_deletable_certified_end_epoch, - ); + self.count_deletable_certified = self.count_deletable_certified.saturating_sub(1); } } @@ -268,159 +246,39 @@ impl ValidBlobInfoV2 { } fn maybe_unset_initial_certified_epoch(&mut self) { - if self.count_deletable_certified == 0 && self.permanent_certified.is_none() { - self.initial_certified_epoch = None; - } - } - - fn maybe_increase_latest_deletable_registered_epoch(&mut self, epoch: Epoch) { - self.latest_seen_deletable_registered_end_epoch = Some( - epoch.max( - self.latest_seen_deletable_registered_end_epoch - .unwrap_or_default(), - ), - ) - } - - fn maybe_increase_latest_deletable_certified_epoch(&mut self, epoch: Epoch) { - self.latest_seen_deletable_certified_end_epoch = Some( - epoch.max( - self.latest_seen_deletable_certified_end_epoch - .unwrap_or_default(), - ), - ) - } - - /// Decrements a counter on blob deletion and unsets the corresponding latest seen end epoch in - /// case the counter reaches 0. - /// - /// If the counter is already 0, an error is logged in release builds and the function panics in - /// dev builds. - fn decrement_deletable_counter_and_maybe_unset_latest_end_epoch( - counter: &mut u32, - latest_seen_end_epoch: &mut Option, - ) { - debug_assert!(*counter > 0); - *counter = counter.checked_sub(1).unwrap_or_else(|| { - tracing::error!("attempt to delete blob when count was already 0"); - 0 - }); - if *counter == 0 { - *latest_seen_end_epoch = None; - } - } - - /// Processes a register status change on the [`Option`] object - /// representing all permanent blobs. - fn register_permanent( - permanent_total: &mut Option, - change_info: &BlobStatusChangeInfo, - ) { - PermanentBlobInfoV2::update_optional(permanent_total, change_info) - } - - /// Processes a certify status change on the [`PermanentBlobInfoV2`] objects representing all - /// and the certified permanent blobs. - /// - /// Returns whether the update was successful. - fn certify_permanent( - permanent_total: &Option, - permanent_certified: &mut Option, - change_info: &BlobStatusChangeInfo, - ) -> bool { - let Some(permanent_total) = permanent_total else { - tracing::error!("attempt to certify a permanent blob when none is tracked"); - return false; - }; - - let registered_end_epoch = permanent_total.end_epoch; - let certified_end_epoch = change_info.end_epoch; - if certified_end_epoch > registered_end_epoch { - tracing::error!( - registered_end_epoch, - certified_end_epoch, - "attempt to certify a permanent blob with later end epoch than any registered blob", - ); - return false; - } - if permanent_total.count.get() - <= permanent_certified - .as_ref() - .map(|p| p.count.get()) - .unwrap_or_default() + if self.count_pooled_refs_certified == 0 + && self.count_deletable_certified == 0 + && self.permanent_certified.is_none() { - tracing::error!("attempt to certify a permanent blob before corresponding register"); - return false; + self.initial_certified_epoch = None; } - PermanentBlobInfoV2::update_optional(permanent_certified, change_info); - true } - /// Processes an extend status change on the [`PermanentBlobInfoV2`] object representing the - /// certified permanent blobs. - fn extend_permanent( - permanent_info: &mut Option, - change_info: &BlobStatusChangeInfo, - ) { - let Some(permanent_info) = permanent_info else { - tracing::error!("attempt to extend a permanent blob when none is tracked"); - return; - }; - - permanent_info.update(change_info, false); - } - - /// Processes a delete status change on the [`PermanentBlobInfoV2`] objects representing all and - /// the certified permanent blobs. - /// - /// This is called when blobs expire at the end of an epoch. fn permanent_expired(&mut self, was_certified: bool) { - Self::decrement_blob_info_inner(&mut self.permanent_total); + ValidBlobInfoV1::decrement_blob_info_inner(&mut self.permanent_total); if was_certified { - Self::decrement_blob_info_inner(&mut self.permanent_certified); + ValidBlobInfoV1::decrement_blob_info_inner(&mut self.permanent_certified); } self.maybe_unset_initial_certified_epoch(); } - fn decrement_blob_info_inner(blob_info_inner: &mut Option) { - match blob_info_inner { - None => tracing::error!("attempt to delete a permanent blob when none is tracked"), - Some(PermanentBlobInfoV2 { count, .. }) => { - if count.get() == 1 { - *blob_info_inner = None; - } else { - *count = NonZeroU32::new(count.get() - 1) - .expect("we just checked that `count` is at least 2") - } - } - } - } - - /// Checks the invariants of the aggregate blob info, returning an error if any invariant is - /// violated. + /// Checks the invariants of this V2 blob info. pub(crate) fn check_invariants(&self) -> anyhow::Result<()> { - let Self { - count_deletable_total, - count_deletable_certified, - permanent_total, - permanent_certified, - initial_certified_epoch, - latest_seen_deletable_registered_end_epoch, - latest_seen_deletable_certified_end_epoch, - .. - } = self; - - anyhow::ensure!(count_deletable_total >= count_deletable_certified); - match initial_certified_epoch { + let has_regular_certified = + self.count_deletable_certified > 0 || self.permanent_certified.is_some(); + let has_pool_certified = self.count_pooled_refs_certified > 0; + + anyhow::ensure!(self.count_deletable_total >= self.count_deletable_certified); + match self.initial_certified_epoch { None => { - anyhow::ensure!(*count_deletable_certified == 0 && permanent_certified.is_none()) + anyhow::ensure!(!has_regular_certified && !has_pool_certified) } Some(_) => { - anyhow::ensure!(*count_deletable_certified > 0 || permanent_certified.is_some()) + anyhow::ensure!(has_regular_certified || has_pool_certified) } } - match (permanent_total, permanent_certified) { + match (&self.permanent_total, &self.permanent_certified) { (None, Some(_)) => { anyhow::bail!("permanent_total.is_none() => permanent_certified.is_none()") } @@ -431,127 +289,76 @@ impl ValidBlobInfoV2 { _ => (), } - match ( - latest_seen_deletable_registered_end_epoch, - latest_seen_deletable_certified_end_epoch, - ) { - (None, Some(_)) => anyhow::bail!( - "latest_seen_deletable_registered_end_epoch.is_none() => \ - latest_seen_deletable_certified_end_epoch.is_none()" - ), - (Some(registered), Some(certified)) => { - anyhow::ensure!(registered >= certified); - } - _ => (), - } + anyhow::ensure!( + self.count_pooled_refs_total >= self.count_pooled_refs_certified, + "pool ref count_pooled_refs_total < count_pooled_refs_certified" + ); + Ok(()) } -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct PermanentBlobInfoV2 { - /// The total number of `Blob` objects for that blob ID with the given status. - pub count: NonZeroU32, - /// The latest expiration epoch among these objects. - pub end_epoch: Epoch, - /// The ID of the first blob event that led to the status with the given `end_epoch`. - pub event: EventID, -} + // --- Storage pool operations --- -impl PermanentBlobInfoV2 { - /// Creates a new `PermanentBlobInfoV2` object for the first blob with the given `end_epoch` and - /// `event`. - pub(crate) fn new_first(end_epoch: Epoch, event: EventID) -> Self { - Self { - count: NonZeroU32::new(1).expect("1 is non-zero"), - end_epoch, - event, - } + /// Registers a blob in a storage pool. + fn pool_register(&mut self) { + self.count_pooled_refs_total += 1; } - /// Updates `self` with the `change_info`, increasing the count if `increase_count == true`. - /// - /// # Panics - /// - /// Panics if the change info has `deletable == true`. - fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { - assert!(!change_info.deletable); + /// Certifies a blob in a storage pool. + fn pool_certify(&mut self, epoch: Epoch) { + let was_certified = self.is_certified(epoch); - if increase_count { - self.count = self.count.saturating_add(1) - }; - if change_info.end_epoch > self.end_epoch { - *self = PermanentBlobInfoV2 { - count: self.count, - end_epoch: change_info.end_epoch, - event: change_info.status_event, - }; + if self.count_pooled_refs_total <= self.count_pooled_refs_certified { + tracing::error!("attempt to certify a pool blob before corresponding register"); + return; } + self.count_pooled_refs_certified += 1; + self.update_initial_certified_epoch(epoch, !was_certified); } - /// Updates `existing_info` with the change info or creates a new `Self` if the input is `None`. - /// - /// # Panics - /// - /// Panics if the change info has `deletable == true`. - fn update_optional(existing_info: &mut Option, change_info: &BlobStatusChangeInfo) { - let BlobStatusChangeInfo { - epoch: _, - end_epoch: new_end_epoch, - status_event: new_status_event, - deletable, - blob_id: _, - } = change_info; - assert!(!deletable); - - match existing_info { - None => { - *existing_info = Some(PermanentBlobInfoV2::new_first( - *new_end_epoch, - *new_status_event, - )) - } - Some(permanent_blob_info) => permanent_blob_info.update(change_info, true), + /// Deletes a blob from a storage pool. + fn pool_delete(&mut self, was_certified: bool) { + if self.count_pooled_refs_total == 0 { + tracing::error!("attempt to delete a pool blob that is not registered"); + return; } - } - - #[cfg(test)] - fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { - Self { - count: NonZeroU32::new(count).expect("count must be non-zero"), - end_epoch, - event: walrus_sui::test_utils::fixed_event_id_for_testing(event_seq), + self.count_pooled_refs_total = self.count_pooled_refs_total.saturating_sub(1); + if was_certified { + self.count_pooled_refs_certified = self.count_pooled_refs_certified.saturating_sub(1); } + self.maybe_unset_initial_certified_epoch(); } +} - #[cfg(test)] - fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { - Self { - count: NonZeroU32::new(count).expect("count must be non-zero"), - end_epoch, - event: walrus_sui::test_utils::event_id_for_testing(), +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) enum BlobInfoV2 { + Invalid { epoch: Epoch, event: EventID }, + Valid(ValidBlobInfoV2), +} + +impl ToBytes for BlobInfoV2 {} + +impl From for BlobInfoV2 { + fn from(v1: BlobInfoV1) -> Self { + match v1 { + BlobInfoV1::Invalid { epoch, event } => BlobInfoV2::Invalid { epoch, event }, + BlobInfoV1::Valid(v) => BlobInfoV2::Valid(v.into()), } } } impl CertifiedBlobInfoApi for BlobInfoV2 { fn is_certified(&self, current_epoch: Epoch) -> bool { - if let Self::Valid(valid_blob_info) = self { - valid_blob_info.is_certified(current_epoch) - } else { - false + match self { + Self::Valid(v) => v.is_certified(current_epoch), + Self::Invalid { .. } => false, } } fn initial_certified_epoch(&self) -> Option { - if let Self::Valid(ValidBlobInfoV2 { - initial_certified_epoch, - .. - }) = self - { - *initial_certified_epoch - } else { - None + match self { + Self::Valid(v) => v.initial_certified_epoch, + Self::Invalid { .. } => None, } } } @@ -567,41 +374,31 @@ impl BlobInfoApi for BlobInfoV2 { ) } - // Note: See the `is_certified` method for an explanation of the use of the - // `latest_seen_deletable_registered_end_epoch` field. - fn is_registered(&self, current_epoch: Epoch) -> bool { - let Self::Valid(ValidBlobInfoV2 { - count_deletable_total, - permanent_total, - latest_seen_deletable_registered_end_epoch, - .. - }) = self - else { + fn is_registered(&self, _current_epoch: Epoch) -> bool { + let Self::Valid(v) = self else { return false; }; - let exists_registered_permanent_blob = permanent_total + let exists_registered_permanent_blob = v + .permanent_total .as_ref() - .is_some_and(|p| p.end_epoch > current_epoch); - let probably_exists_registered_deletable_blob = *count_deletable_total > 0 - && latest_seen_deletable_registered_end_epoch.is_some_and(|e| e > current_epoch); - - exists_registered_permanent_blob || probably_exists_registered_deletable_blob + .is_some_and(|p| p.end_epoch > _current_epoch); + + // Note that at the beginning of the epoch before GC runs, newly expired deletable blob or + // pooled blob's registered counter may still be non-zero, but the blob is already expired. + // This will make the blob appears to be registered until GC finishes. + exists_registered_permanent_blob + || v.count_deletable_total > 0 + || v.count_pooled_refs_total > 0 } fn can_blob_info_be_deleted(&self, current_epoch: Epoch) -> bool { match self { - Self::Invalid { .. } => { - // We don't know whether there are any deletable blob objects for this blob ID. - false - } - Self::Valid(ValidBlobInfoV2 { - count_deletable_total, - permanent_total, - .. - }) => { - *count_deletable_total == 0 - && permanent_total.is_none() + Self::Invalid { .. } => false, + Self::Valid(v) => { + v.count_deletable_total == 0 + && v.permanent_total.is_none() + && v.count_pooled_refs_total == 0 && self.can_data_be_deleted(current_epoch) } } @@ -651,6 +448,7 @@ impl Mergeable for BlobInfoV2 { ) => { *is_metadata_stored = new_is_metadata_stored; } + // Regular blob status changes. ( Self::Valid(valid_blob_info), BlobInfoMergeOperand::ChangeStatus { @@ -666,40 +464,47 @@ impl Mergeable for BlobInfoV2 { Self::Valid(valid_blob_info), BlobInfoMergeOperand::PermanentExpired { was_certified }, ) => valid_blob_info.permanent_expired(was_certified), + // Storage pool operations. + ( + Self::Valid(valid_blob_info), + BlobInfoMergeOperand::PooledBlobChangeStatus { + change_type, + change_info, + .. + }, + ) => match change_type { + BlobStatusChangeType::Register => { + valid_blob_info.pool_register(); + } + BlobStatusChangeType::Certify => { + valid_blob_info.pool_certify(change_info.epoch); + } + BlobStatusChangeType::Extend => { + // Extensions don't change ref counts for storage pool. + // The pool's end_epoch is tracked separately. + } + BlobStatusChangeType::Delete { was_certified } => { + valid_blob_info.pool_delete(was_certified); + } + }, + ( + Self::Valid(valid_blob_info), + BlobInfoMergeOperand::PoolExpired { was_certified, .. }, + ) => valid_blob_info.pool_delete(was_certified), } self } fn merge_new(operand: Self::MergeOperand) -> Option { match operand { - BlobInfoMergeOperand::ChangeStatus { + BlobInfoMergeOperand::PooledBlobChangeStatus { change_type: BlobStatusChangeType::Register, - change_info: - BlobStatusChangeInfo { - deletable, - epoch: _, - end_epoch, - status_event, - blob_id: _, - }, - } => Some( - if deletable { - ValidBlobInfoV2 { - count_deletable_total: 1, - latest_seen_deletable_registered_end_epoch: Some(end_epoch), - ..Default::default() - } - } else { - ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV2::new_first( - end_epoch, - status_event, - )), - ..Default::default() - } - } - .into(), - ), + .. + } => { + let mut v2 = ValidBlobInfoV2::default(); + v2.pool_register(); + Some(BlobInfoV2::Valid(v2)) + } BlobInfoMergeOperand::MarkInvalid { epoch, status_event, @@ -713,24 +518,39 @@ impl Mergeable for BlobInfoV2 { "marking metadata stored for an untracked blob ID; blob info will be removed \ during the next garbage collection" ); - Some( - ValidBlobInfoV2 { - is_metadata_stored, - ..Default::default() - } - .into(), - ) + Some(BlobInfoV2::Valid(ValidBlobInfoV2 { + is_metadata_stored, + ..Default::default() + })) } - BlobInfoMergeOperand::ChangeStatus { .. } - | BlobInfoMergeOperand::DeletableExpired { .. } - | BlobInfoMergeOperand::PermanentExpired { .. } => { + BlobInfoMergeOperand::ChangeStatus { + change_type: BlobStatusChangeType::Register, + change_info: + BlobStatusChangeInfo { + deletable, + end_epoch, + status_event, + .. + }, + } => Some(BlobInfoV2::Valid(if deletable { + ValidBlobInfoV2 { + count_deletable_total: 1, + ..Default::default() + } + } else { + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_first(end_epoch, status_event)), + ..Default::default() + } + })), + _ => { tracing::error!( ?operand, - "encountered an unexpected update for an untracked blob ID" + "encountered an unexpected update for an untracked blob ID (V2)" ); debug_assert!( false, - "encountered an unexpected update for an untracked blob ID: {operand:?}", + "encountered an unexpected update for an untracked blob ID (V2): {operand:?}", ); None } @@ -738,6 +558,24 @@ impl Mergeable for BlobInfoV2 { } } +// ============================================================================= +// Per-object blob info V2 +// ============================================================================= + +/// How the end epoch of a per-object blob is determined. +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] +pub(crate) enum EndEpochInfo { + /// Regular blob with an individual end epoch. + Individual(Epoch), + /// Pooled blob whose lifetime is determined by the storage pool's end epoch. + StoragePool(ObjectID), +} + +/// V2 per-object blob info that supports both regular blobs and storage pool blobs. +/// +/// Unlike V1, the end epoch and storage pool ID are combined into a single `EndEpochInfo` +/// enum: regular blobs have `EndEpochInfo::Individual(end_epoch)`, while pool blobs have +/// `EndEpochInfo::StoragePool(pool_id)`. #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) struct PerObjectBlobInfoV2 { /// The blob ID. @@ -746,8 +584,8 @@ pub(crate) struct PerObjectBlobInfoV2 { pub registered_epoch: Epoch, /// The epoch in which the blob was first certified, `None` if the blob is uncertified. pub certified_epoch: Option, - /// The epoch in which the blob expires. - pub end_epoch: Epoch, + /// How the blob's end epoch is determined. + pub end_epoch_info: EndEpochInfo, /// Whether the blob is deletable. pub deletable: bool, /// The ID of the last blob event related to this object. @@ -779,12 +617,26 @@ impl PerObjectBlobInfoApi for PerObjectBlobInfoV2 { } fn is_registered(&self, current_epoch: Epoch) -> bool { - self.end_epoch > current_epoch && !self.deleted + if self.deleted { + return false; + } + match self.end_epoch_info { + EndEpochInfo::Individual(end_epoch) => end_epoch > current_epoch, + // Pool blob liveness depends on the pool, not the blob. + EndEpochInfo::StoragePool(_) => true, + } } fn is_deleted(&self) -> bool { self.deleted } + + fn storage_pool_id(&self) -> Option { + match &self.end_epoch_info { + EndEpochInfo::StoragePool(id) => Some(*id), + EndEpochInfo::Individual(_) => None, + } + } } impl ToBytes for PerObjectBlobInfoV2 {} @@ -838,7 +690,7 @@ impl Mergeable for PerObjectBlobInfoV2 { "cannot extend an uncertified blob {}", self.blob_id ); - self.end_epoch = change_info.end_epoch; + self.end_epoch_info = EndEpochInfo::Individual(change_info.end_epoch); } BlobStatusChangeType::Delete { was_certified } => { assert_eq!(self.certified_epoch.is_some(), was_certified); @@ -849,89 +701,59 @@ impl Mergeable for PerObjectBlobInfoV2 { } fn merge_new(operand: Self::MergeOperand) -> Option { - let PerObjectBlobInfoMergeOperand { - change_type: BlobStatusChangeType::Register, - change_info: - BlobStatusChangeInfo { - blob_id, - deletable, - epoch, - end_epoch, - status_event, - }, - } = operand - else { - tracing::error!( - ?operand, - "encountered an update other than 'register' for an untracked blob object" - ); - debug_assert!( - false, - "encountered an update other than 'register' for an untracked blob object: \ - {operand:?}" - ); - return None; - }; - Some(Self { - blob_id, - registered_epoch: epoch, - certified_epoch: None, - end_epoch, - deletable, - event: status_event, - deleted: false, - }) + // V2 entries are created via direct insert, not merge_new. + tracing::error!( + ?operand, + "PerObjectBlobInfoV2::merge_new should not be called; V2 entries are created via \ + direct insert" + ); + debug_assert!( + false, + "PerObjectBlobInfoV2::merge_new should not be called: {operand:?}" + ); + None } } #[cfg(test)] mod tests { + use sui_types::base_types::ObjectID; use walrus_sui::test_utils::{event_id_for_testing, fixed_event_id_for_testing}; use walrus_test_utils::param_test; - use super::*; + use super::{EndEpochInfo, PerObjectBlobInfoV2, *}; + use crate::node::storage::blob_info::{ + BlobInfo, + PooledBlobChangeInfo, + per_object_blob_info::PerObjectBlobInfoMergeOperand, + }; - fn check_invariants(blob_info: &BlobInfoV2) { - if let BlobInfoV2::Valid(valid_blob_info) = blob_info { - valid_blob_info - .check_invariants() - .expect("aggregate blob info invariants violated") - } + fn pool_id() -> ObjectID { + walrus_sui::test_utils::object_id_for_testing() } - param_test! { - test_merge_new_expected_failure_cases: [ - #[should_panic] certify_permanent: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify,false, 42, 314, event_id_for_testing() - )), - #[should_panic] certify_deletable: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() - )), - #[should_panic] extend: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() - )), - #[should_panic] delete_deletable: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: true }, - false, - 42, - 314, - event_id_for_testing(), - )), - #[should_panic] delete_permanent: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: false }, - false, - 42, - 314, - event_id_for_testing(), - )), - ] + /// Shorthand: build a pool register/certify/delete operand. + fn pool_op(change_type: BlobStatusChangeType, epoch: Epoch) -> BlobInfoMergeOperand { + BlobInfoMergeOperand::PooledBlobChangeStatus { + change_type, + change_info: PooledBlobChangeInfo { + epoch, + storage_pool_id: pool_id(), + }, + } } - fn test_merge_new_expected_failure_cases(operand: BlobInfoMergeOperand) { - let _ = BlobInfoV2::merge_new(operand); + + fn check_v2_invariants(info: &BlobInfoV2) { + if let BlobInfoV2::Valid(v) = info { + v.check_invariants() + .expect("V2 blob info invariants violated") + } } + // --- V2 merge_new success cases --- + param_test! { - test_merge_new_expected_success_cases_invariants: [ + test_v2_merge_new_expected_success_cases_invariants: [ register_permanent: (BlobInfoMergeOperand::new_change_for_testing( BlobStatusChangeType::Register, false, 42, 314, event_id_for_testing() )), @@ -939,648 +761,629 @@ mod tests { BlobStatusChangeType::Register, true, 42, 314, event_id_for_testing() )), invalidate: (BlobInfoMergeOperand::MarkInvalid { - epoch: 0, - status_event: event_id_for_testing() + epoch: 0, status_event: event_id_for_testing() }), metadata_true: (BlobInfoMergeOperand::MarkMetadataStored(true)), metadata_false: (BlobInfoMergeOperand::MarkMetadataStored(false)), + pool_register: (pool_op(BlobStatusChangeType::Register, 1)), ] } - fn test_merge_new_expected_success_cases_invariants(operand: BlobInfoMergeOperand) { + fn test_v2_merge_new_expected_success_cases_invariants(operand: BlobInfoMergeOperand) { let blob_info = BlobInfoV2::merge_new(operand).expect("should be some"); - check_invariants(&blob_info); + check_v2_invariants(&blob_info); } - param_test! { - test_invalid_status_is_not_changed: [ - invalidate: (BlobInfoMergeOperand::MarkInvalid { - epoch: 0, - status_event: event_id_for_testing() - }), - metadata_true: (BlobInfoMergeOperand::MarkMetadataStored(true)), - metadata_false: (BlobInfoMergeOperand::MarkMetadataStored(false)), - register_permanent: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Register, false, 42, 314, event_id_for_testing() - )), - register_deletable: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Register, true, 42, 314, event_id_for_testing() - )), - certify_permanent: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, false, 42, 314, event_id_for_testing() - )), - certify_deletable: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() - )), - extend: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() - )), - delete_true: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: true }, - false, - 42, - 314, - event_id_for_testing(), - )), - delete_false: (BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: false }, - false, - 42, - 314, - event_id_for_testing(), - )), - ] - } - fn test_invalid_status_is_not_changed(operand: BlobInfoMergeOperand) { - let blob_info = BlobInfoV2::Invalid { - epoch: 42, - event: event_id_for_testing(), - }; - assert_eq!(blob_info, blob_info.clone().merge_with(operand)); - } + // --- V2 mark metadata stored keeps everything else unchanged --- param_test! { - test_mark_metadata_stored_keeps_everything_else_unchanged: [ - default: (Default::default()), - deletable: (ValidBlobInfoV2{count_deletable_total: 2, ..Default::default()}), - deletable_certified: (ValidBlobInfoV2{ - count_deletable_total: 2, - count_deletable_certified: 1, - initial_certified_epoch: Some(0), - ..Default::default() + test_v2_mark_metadata_stored_keeps_everything_else_unchanged: [ + default: (ValidBlobInfoV2::default()), + deletable: (ValidBlobInfoV2 { + count_deletable_total: 2, ..Default::default() }), - permanent: (ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), - ..Default::default() + deletable_certified: (ValidBlobInfoV2 { + count_deletable_total: 2, count_deletable_certified: 1, + initial_certified_epoch: Some(0), ..Default::default() }), - permanent_certified: (ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), - initial_certified_epoch: Some(1), - ..Default::default() + permanent_certified: (ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + initial_certified_epoch: Some(1), ..Default::default() + }), + pool_refs: (ValidBlobInfoV2 { + count_pooled_refs_total: 3, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(2), ..Default::default() + }), + mixed_regular_and_pool: (ValidBlobInfoV2 { + count_deletable_total: 1, count_deletable_certified: 1, + count_pooled_refs_total: 2, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }), ] } - fn test_mark_metadata_stored_keeps_everything_else_unchanged( + fn test_v2_mark_metadata_stored_keeps_everything_else_unchanged( preexisting_info: ValidBlobInfoV2, ) { preexisting_info .check_invariants() - .expect("preexisting blob info invariants violated"); - let expected_updated_info = ValidBlobInfoV2 { + .expect("preexisting invariants violated"); + let expected = ValidBlobInfoV2 { is_metadata_stored: true, ..preexisting_info.clone() }; - expected_updated_info + expected .check_invariants() - .expect("expected updated blob info invariants violated"); + .expect("expected invariants violated"); - let updated_info = BlobInfoV2::Valid(preexisting_info) + let updated = BlobInfoV2::Valid(preexisting_info) .merge_with(BlobInfoMergeOperand::MarkMetadataStored(true)); - - assert_eq!(updated_info, expected_updated_info.into()); + assert_eq!(updated, BlobInfoV2::Valid(expected)); } + // --- V2 mark invalid marks everything invalid --- + param_test! { - test_mark_invalid_marks_everything_invalid: [ - default: (Default::default()), - deletable: (ValidBlobInfoV2{count_deletable_total: 2, ..Default::default()}), - deletable_certified: (ValidBlobInfoV2{ - count_deletable_total: 2, - count_deletable_certified: 1, - initial_certified_epoch: Some(0), - ..Default::default() + test_v2_mark_invalid_marks_everything_invalid: [ + default: (ValidBlobInfoV2::default()), + deletable: (ValidBlobInfoV2 { + count_deletable_total: 2, ..Default::default() }), - permanent: (ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), - ..Default::default() + deletable_certified: (ValidBlobInfoV2 { + count_deletable_total: 2, count_deletable_certified: 1, + initial_certified_epoch: Some(0), ..Default::default() }), - permanent_certified: (ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), - initial_certified_epoch: Some(1), - ..Default::default() + permanent_certified: (ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + initial_certified_epoch: Some(1), ..Default::default() + }), + pool_refs: (ValidBlobInfoV2 { + count_pooled_refs_total: 3, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(2), ..Default::default() + }), + mixed_regular_and_pool: (ValidBlobInfoV2 { + count_deletable_total: 1, count_deletable_certified: 1, + count_pooled_refs_total: 2, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }), ] } - fn test_mark_invalid_marks_everything_invalid(preexisting_info: ValidBlobInfoV2) { - let preexisting_info = preexisting_info.into(); - check_invariants(&preexisting_info); + fn test_v2_mark_invalid_marks_everything_invalid(preexisting_info: ValidBlobInfoV2) { + preexisting_info + .check_invariants() + .expect("preexisting invariants violated"); let event = event_id_for_testing(); - let updated_info = preexisting_info.merge_with(BlobInfoMergeOperand::MarkInvalid { - epoch: 2, - status_event: event, - }); - assert_eq!(BlobInfoV2::Invalid { epoch: 2, event }, updated_info); + let updated = + BlobInfoV2::Valid(preexisting_info).merge_with(BlobInfoMergeOperand::MarkInvalid { + epoch: 2, + status_event: event, + }); + assert_eq!(BlobInfoV2::Invalid { epoch: 2, event }, updated); } + // --- V2 merge preexisting expected successes --- + // Covers regular ops (deletable, permanent, expire) and pool ops on ValidBlobInfoV2. + // V2 does not track latest_seen_deletable_*_end_epoch, so those cases are omitted. + param_test! { - test_merge_preexisting_expected_successes: [ + test_v2_merge_preexisting_expected_successes: [ + // --- regular deletable --- register_first_deletable: ( - Default::default(), + ValidBlobInfoV2::default(), BlobInfoMergeOperand::new_change_for_testing( BlobStatusChangeType::Register, true, 1, 2, event_id_for_testing() ), - ValidBlobInfoV2{ - count_deletable_total: 1, - latest_seen_deletable_registered_end_epoch: Some(2), - ..Default::default() - }, + ValidBlobInfoV2 { count_deletable_total: 1, ..Default::default() }, ), - register_additional_deletable1: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - latest_seen_deletable_registered_end_epoch: Some(2), - ..Default::default() - }, + register_additional_deletable: ( + ValidBlobInfoV2 { count_deletable_total: 3, ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( BlobStatusChangeType::Register, true, 1, 5, event_id_for_testing() ), - ValidBlobInfoV2{ - count_deletable_total: 4, - latest_seen_deletable_registered_end_epoch: Some(5), - ..Default::default() - }, - ), - register_additional_deletable2: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - latest_seen_deletable_registered_end_epoch: Some(4), - ..Default::default() - }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Register, true, 1, 3, event_id_for_testing() - ), - ValidBlobInfoV2{ - count_deletable_total: 4, - latest_seen_deletable_registered_end_epoch: Some(4), - ..Default::default() - }, + ValidBlobInfoV2 { count_deletable_total: 4, ..Default::default() }, ), certify_first_deletable: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - latest_seen_deletable_registered_end_epoch: Some(4), - ..Default::default() - }, + ValidBlobInfoV2 { count_deletable_total: 3, ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( BlobStatusChangeType::Certify, true, 1, 4, event_id_for_testing() ), - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(4), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() + ValidBlobInfoV2 { + count_deletable_total: 3, count_deletable_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, ), - certify_additional_deletable1: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 1, - initial_certified_epoch: Some(0), - latest_seen_deletable_registered_end_epoch: Some(4), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() + certify_additional_deletable_keeps_earlier_epoch: ( + ValidBlobInfoV2 { + count_deletable_total: 3, count_deletable_certified: 1, + initial_certified_epoch: Some(0), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( BlobStatusChangeType::Certify, true, 1, 2, event_id_for_testing() ), - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 2, - initial_certified_epoch: Some(0), - latest_seen_deletable_registered_end_epoch: Some(4), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() + ValidBlobInfoV2 { + count_deletable_total: 3, count_deletable_certified: 2, + initial_certified_epoch: Some(0), ..Default::default() }, ), - certify_additional_deletable2: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() + delete_deletable_blob: ( + ValidBlobInfoV2 { + count_deletable_total: 3, count_deletable_certified: 2, + initial_certified_epoch: Some(1), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, true, 0, 5, event_id_for_testing() + BlobStatusChangeType::Delete { was_certified: true }, true, 1, 6, + event_id_for_testing(), ), - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 2, - initial_certified_epoch: Some(0), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(5), - ..Default::default() + ValidBlobInfoV2 { + count_deletable_total: 2, count_deletable_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, ), - register_first_permanent: ( - ValidBlobInfoV2{ - ..Default::default() + delete_last_certified_deletable_clears_epoch: ( + ValidBlobInfoV2 { + count_deletable_total: 2, count_deletable_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Register, false, 1, 2, fixed_event_id_for_testing(0) + BlobStatusChangeType::Delete { was_certified: true }, true, 1, 4, + event_id_for_testing(), ), - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), - ..Default::default() + ValidBlobInfoV2 { + count_deletable_total: 1, count_deletable_certified: 0, + initial_certified_epoch: None, ..Default::default() }, ), - extend_deletable: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 1, - initial_certified_epoch: Some(0), - latest_seen_deletable_registered_end_epoch: Some(4), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() + expire_deletable_blob: ( + ValidBlobInfoV2 { + count_deletable_total: 3, count_deletable_certified: 2, + initial_certified_epoch: Some(1), ..Default::default() }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Extend, true, 3, 42, event_id_for_testing() - ), - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 1, - initial_certified_epoch: Some(0), - latest_seen_deletable_registered_end_epoch: Some(42), - latest_seen_deletable_certified_end_epoch: Some(42), - ..Default::default() + BlobInfoMergeOperand::DeletableExpired { was_certified: true }, + ValidBlobInfoV2 { + count_deletable_total: 2, count_deletable_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, ), - extend_permanent: ( - ValidBlobInfoV2{ - initial_certified_epoch: Some(0), - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 4, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 4, 1)), - ..Default::default() + expire_last_certified_deletable_clears_epoch: ( + ValidBlobInfoV2 { + count_deletable_total: 2, count_deletable_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Extend, false, 3, 42, fixed_event_id_for_testing(2) - ), - ValidBlobInfoV2{ - initial_certified_epoch: Some(0), - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 2)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 42, 2)), - ..Default::default() + BlobInfoMergeOperand::DeletableExpired { was_certified: true }, + ValidBlobInfoV2 { + count_deletable_total: 1, count_deletable_certified: 0, + initial_certified_epoch: None, ..Default::default() }, ), - certify_outdated_deletable: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(8), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, + // --- regular permanent --- + register_first_permanent: ( + ValidBlobInfoV2::default(), BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, true, 4, 6, event_id_for_testing() + BlobStatusChangeType::Register, false, 1, 2, + fixed_event_id_for_testing(0) ), - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 2, - initial_certified_epoch: Some(4), - latest_seen_deletable_registered_end_epoch: Some(8), - latest_seen_deletable_certified_end_epoch: Some(6), + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, ), - certify_outdated_permanent: ( - ValidBlobInfoV2{ - initial_certified_epoch: Some(2), - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_for_testing(1, 5)), + register_additional_permanent: ( + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, false, 7, 42, fixed_event_id_for_testing(1) + BlobStatusChangeType::Register, false, 2, 3, + fixed_event_id_for_testing(1) ), - ValidBlobInfoV2{ - initial_certified_epoch: Some(7), - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 42, 1)), + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 1)), ..Default::default() }, ), - register_additional_permanent: ( - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), + extend_permanent: ( + ValidBlobInfoV2 { + initial_certified_epoch: Some(0), + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 4, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 4, 1)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Register, false, 2, 3, fixed_event_id_for_testing(1) + BlobStatusChangeType::Extend, false, 3, 42, + fixed_event_id_for_testing(2) ), - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 1)), + ValidBlobInfoV2 { + initial_certified_epoch: Some(0), + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 42, 2)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 42, 2)), ..Default::default() }, ), expire_permanent_blob: ( - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(3, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 1)), - initial_certified_epoch: Some(1), - ..Default::default() - }, - BlobInfoMergeOperand::PermanentExpired { - was_certified: true, + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(3, 5, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 1)), + initial_certified_epoch: Some(1), ..Default::default() }, - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 1)), - initial_certified_epoch: Some(1), - ..Default::default() + BlobInfoMergeOperand::PermanentExpired { was_certified: true }, + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 1)), + initial_certified_epoch: Some(1), ..Default::default() }, ), - expire_last_permanent_blob: ( - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 1)), - initial_certified_epoch: Some(1), - ..Default::default() - }, - BlobInfoMergeOperand::PermanentExpired { - was_certified: true, + expire_last_certified_permanent_clears_epoch: ( + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 1)), + initial_certified_epoch: Some(1), ..Default::default() }, - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 0)), + BlobInfoMergeOperand::PermanentExpired { was_certified: true }, + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 0)), permanent_certified: None, - initial_certified_epoch: None, - ..Default::default() + initial_certified_epoch: None, ..Default::default() }, ), - delete_deletable_blob: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 2, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: true }, - true, - 1, - 6, - event_id_for_testing(), - ), - ValidBlobInfoV2{ - count_deletable_total: 2, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, + // --- pool operations --- + pool_register: ( + ValidBlobInfoV2::default(), + pool_op(BlobStatusChangeType::Register, 1), + ValidBlobInfoV2 { count_pooled_refs_total: 1, ..Default::default() }, ), - expire_deletable_blob: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - count_deletable_certified: 2, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, - BlobInfoMergeOperand::DeletableExpired { - was_certified: true, - }, - ValidBlobInfoV2{ - count_deletable_total: 2, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, - ), - delete_last_deletable_blob: ( - ValidBlobInfoV2{ - count_deletable_total: 2, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: true }, - true, - 1, - 4, - event_id_for_testing(), - ), - ValidBlobInfoV2{ - count_deletable_total: 1, - count_deletable_certified: 0, - initial_certified_epoch: None, - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: None, - ..Default::default() + pool_certify: ( + ValidBlobInfoV2 { count_pooled_refs_total: 1, ..Default::default() }, + pool_op(BlobStatusChangeType::Certify, 3), + ValidBlobInfoV2 { + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(3), ..Default::default() }, ), - expire_last_deletable_blob: ( - ValidBlobInfoV2{ - count_deletable_total: 2, - count_deletable_certified: 1, - initial_certified_epoch: Some(1), - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: Some(4), - ..Default::default() - }, - BlobInfoMergeOperand::DeletableExpired { - was_certified: true, + pool_delete_certified: ( + ValidBlobInfoV2 { + count_pooled_refs_total: 2, count_pooled_refs_certified: 2, + initial_certified_epoch: Some(1), ..Default::default() }, - ValidBlobInfoV2{ - count_deletable_total: 1, - count_deletable_certified: 0, - initial_certified_epoch: None, - latest_seen_deletable_registered_end_epoch: Some(5), - latest_seen_deletable_certified_end_epoch: None, - ..Default::default() + pool_op(BlobStatusChangeType::Delete { was_certified: true }, 5), + ValidBlobInfoV2 { + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, ), - expire_uncertified_permanent_blob: ( - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(3, 5, 0)), - ..Default::default() - }, - BlobInfoMergeOperand::PermanentExpired { - was_certified: false, + pool_delete_last_certified_clears_epoch: ( + ValidBlobInfoV2 { + count_pooled_refs_total: 2, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 5, 0)), - ..Default::default() + pool_op(BlobStatusChangeType::Delete { was_certified: true }, 5), + ValidBlobInfoV2 { + count_pooled_refs_total: 1, count_pooled_refs_certified: 0, + initial_certified_epoch: None, ..Default::default() }, ), - expire_last_uncertified_permanent_blob: ( - ValidBlobInfoV2{ - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 5, 0)), - ..Default::default() - }, - BlobInfoMergeOperand::PermanentExpired { - was_certified: false, - }, - ValidBlobInfoV2{ - permanent_total: None, - ..Default::default() + pool_expired: ( + ValidBlobInfoV2 { + count_pooled_refs_total: 2, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() }, - ), - delete_uncertified_deletable_blob: ( - ValidBlobInfoV2{ - count_deletable_total: 3, - latest_seen_deletable_registered_end_epoch: Some(5), - ..Default::default() + BlobInfoMergeOperand::PoolExpired { + storage_pool_id: pool_id(), was_certified: true, }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: false }, - true, - 1, - 6, - event_id_for_testing(), - ), - ValidBlobInfoV2{ - count_deletable_total: 2, - latest_seen_deletable_registered_end_epoch: Some(5), - ..Default::default() + ValidBlobInfoV2 { + count_pooled_refs_total: 1, count_pooled_refs_certified: 0, + initial_certified_epoch: None, ..Default::default() }, ), - delete_last_uncertified_deletable_blob: ( - ValidBlobInfoV2{ - count_deletable_total: 1, - latest_seen_deletable_registered_end_epoch: Some(5), - ..Default::default() + // --- mixed: pool cert keeps epoch alive after regular deletion --- + mixed_regular_delete_pool_cert_keeps_epoch: ( + ValidBlobInfoV2 { + count_deletable_total: 1, count_deletable_certified: 1, + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(3), ..Default::default() }, - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Delete { was_certified: false }, - true, - 2, - 6, - event_id_for_testing(), - ), - ValidBlobInfoV2{ - count_deletable_total: 0, - latest_seen_deletable_registered_end_epoch: None, - ..Default::default() + BlobInfoMergeOperand::DeletableExpired { was_certified: true }, + ValidBlobInfoV2 { + count_deletable_total: 0, count_deletable_certified: 0, + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(3), ..Default::default() }, ), ] } - fn test_merge_preexisting_expected_successes( - preexisting_info: ValidBlobInfoV2, + fn test_v2_merge_preexisting_expected_successes( + preexisting: ValidBlobInfoV2, operand: BlobInfoMergeOperand, - expected_info: ValidBlobInfoV2, + expected: ValidBlobInfoV2, ) { - preexisting_info + preexisting .check_invariants() - .expect("preexisting blob info invariants violated"); - expected_info + .expect("preexisting invariants violated"); + expected .check_invariants() - .expect("expected blob info invariants violated"); + .expect("expected invariants violated"); + let updated = BlobInfoV2::Valid(preexisting).merge_with(operand); + assert_eq!(updated, BlobInfoV2::Valid(expected)); + } - let updated_info = BlobInfoV2::Valid(preexisting_info).merge_with(operand); + // --- V1->V2 conversion preserves fields and zeroes pool counters --- + #[test] + fn v1_to_v2_conversion() { + let v1 = ValidBlobInfoV1 { + is_metadata_stored: true, + count_deletable_total: 3, + count_deletable_certified: 1, + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 10, 0)), + permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 10, 1)), + initial_certified_epoch: Some(5), + latest_seen_deletable_registered_end_epoch: Some(8), + latest_seen_deletable_certified_end_epoch: Some(7), + }; + let v2: ValidBlobInfoV2 = v1.clone().into(); + assert_eq!(v2.count_deletable_total, v1.count_deletable_total); + assert_eq!(v2.permanent_total, v1.permanent_total); + assert_eq!(v2.initial_certified_epoch, v1.initial_certified_epoch); + assert_eq!( + (v2.count_pooled_refs_total, v2.count_pooled_refs_certified), + (0, 0) + ); + v2.check_invariants() + .expect("v2 blob info invariants violated"); - assert_eq!(updated_info, expected_info.into()); + // Invalid converts too. + let inv = BlobInfoV1::Invalid { + epoch: 5, + event: event_id_for_testing(), + }; + assert!(matches!( + BlobInfoV2::from(inv), + BlobInfoV2::Invalid { epoch: 5, .. } + )); } + // --- Invalid state --- param_test! { - test_merge_preexisting_expected_failures: [ - certify_permanent_without_register: ( - Default::default(), - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, false, 42, 314, event_id_for_testing() - ), - ), - extend_permanent_without_certify: ( - Default::default(), - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() - ), - ), - certify_deletable_without_register: ( - Default::default(), - BlobInfoMergeOperand::new_change_for_testing( - BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() - ), - ), + test_v2_invalid_status_is_not_changed: [ + invalidate: (BlobInfoMergeOperand::MarkInvalid { + epoch: 0, status_event: event_id_for_testing() + }), + metadata_true: (BlobInfoMergeOperand::MarkMetadataStored(true)), + metadata_false: (BlobInfoMergeOperand::MarkMetadataStored(false)), + register_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, false, 42, 314, event_id_for_testing() + )), + register_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, true, 42, 314, event_id_for_testing() + )), + certify_permanent: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, false, 42, 314, event_id_for_testing() + )), + certify_deletable: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Certify, true, 42, 314, event_id_for_testing() + )), + extend: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Extend, false, 42, 314, event_id_for_testing() + )), + delete_true: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: true }, + false, 42, 314, event_id_for_testing(), + )), + delete_false: (BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Delete { was_certified: false }, + false, 42, 314, event_id_for_testing(), + )), + pool_register: (pool_op(BlobStatusChangeType::Register, 1)), + pool_certify: (pool_op(BlobStatusChangeType::Certify, 1)), + pool_delete: (pool_op(BlobStatusChangeType::Delete { was_certified: true }, 1)), + pool_expired: (BlobInfoMergeOperand::PoolExpired { + storage_pool_id: pool_id(), was_certified: false, + }), ] } - fn test_merge_preexisting_expected_failures( - preexisting_info: ValidBlobInfoV2, - operand: BlobInfoMergeOperand, - ) { - preexisting_info - .check_invariants() - .expect("preexisting blob info invariants violated"); - let preexisting_info = BlobInfoV2::Valid(preexisting_info); - let blob_info = preexisting_info.clone().merge_with(operand); - assert_eq!(preexisting_info, blob_info); + fn test_v2_invalid_status_is_not_changed(operand: BlobInfoMergeOperand) { + let blob_info = BlobInfoV2::Invalid { + epoch: 42, + event: event_id_for_testing(), + }; + assert_eq!(blob_info, blob_info.clone().merge_with(operand)); + } + + // --- BlobInfo enum: V1->V2 upgrade routing --- + #[test] + fn blob_info_upgrades_v1_to_v2_on_pool_operand() { + let v1 = BlobInfo::V1(BlobInfoV1::Valid(ValidBlobInfoV1 { + count_deletable_total: 2, + count_deletable_certified: 1, + initial_certified_epoch: Some(3), + latest_seen_deletable_registered_end_epoch: Some(10), + latest_seen_deletable_certified_end_epoch: Some(10), + ..Default::default() + })); + let result = v1.merge_with(pool_op(BlobStatusChangeType::Register, 1)); + let BlobInfo::V2(BlobInfoV2::Valid(v)) = &result else { + panic!("expected V2, got {result:?}") + }; + assert_eq!(v.count_deletable_total, 2); + assert_eq!(v.count_pooled_refs_total, 1); + v.check_invariants().unwrap(); + } + + #[test] + fn blob_info_stays_v1_for_regular_operand() { + let v1 = BlobInfo::V1(BlobInfoV1::Valid(Default::default())); + let result = v1.merge_with(BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, + true, + 1, + 10, + event_id_for_testing(), + )); + assert!(matches!(result, BlobInfo::V1(_))); } + #[test] + fn blob_info_merge_new_creates_v2_for_pool_v1_for_regular() { + let pool = BlobInfo::merge_new(pool_op(BlobStatusChangeType::Register, 1)).unwrap(); + assert!(matches!(pool, BlobInfo::V2(_))); + + let regular = BlobInfo::merge_new(BlobInfoMergeOperand::new_change_for_testing( + BlobStatusChangeType::Register, + true, + 1, + 10, + event_id_for_testing(), + )) + .unwrap(); + assert!(matches!(regular, BlobInfo::V1(_))); + } + + // --- CertifiedBlobInfoApi on V2 --- param_test! { - test_blob_status_is_inexistent_for_expired_blobs: [ - expired_permanent_registered_0: ( + test_v2_is_certified: [ + before_certified_epoch: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), - ..Default::default() - }, - 1, - 2, + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(5), ..Default::default() + }, 4, false, ), - expired_permanent_registered_1: ( + at_certified_epoch: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 3, 0)), - ..Default::default() - }, - 2, - 4, + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + initial_certified_epoch: Some(5), ..Default::default() + }, 5, true, ), - expired_permanent_certified: ( + no_certified_refs: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV2::new_fixed_for_testing(2, 2, 0)), - permanent_certified: Some(PermanentBlobInfoV2::new_fixed_for_testing(1, 2, 0)), - ..Default::default() - }, - 1, - 2, + count_pooled_refs_total: 1, ..Default::default() + }, 0, false, ), - expired_deletable_registered: ( + regular_deletable_certified: ( ValidBlobInfoV2 { - count_deletable_total: 1, - latest_seen_deletable_registered_end_epoch: Some(2), + count_deletable_total: 1, count_deletable_certified: 1, + initial_certified_epoch: Some(2), ..Default::default() + }, 3, true, + ), + ] + } + fn test_v2_is_certified(info: ValidBlobInfoV2, epoch: Epoch, expected: bool) { + info.check_invariants().unwrap(); + assert_eq!(BlobInfoV2::Valid(info).is_certified(epoch), expected); + } + + // --- BlobInfoApi on V2 --- + param_test! { + test_v2_is_registered: [ + pool_refs: ( + ValidBlobInfoV2 { count_pooled_refs_total: 1, ..Default::default() }, + 9999, true, + ), + deletable_refs: ( + ValidBlobInfoV2 { count_deletable_total: 1, ..Default::default() }, + 9999, true, + ), + permanent_before_expiry: ( + ValidBlobInfoV2 { + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 10, 0)), ..Default::default() - }, - 1, - 2, + }, 9, true, ), - expired_deletable_certified: ( + permanent_at_expiry: ( ValidBlobInfoV2 { - count_deletable_total: 1, - latest_seen_deletable_registered_end_epoch: Some(2), - count_deletable_certified: 1, - latest_seen_deletable_certified_end_epoch: Some(2), + permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 10, 0)), ..Default::default() - }, - 1, - 2, + }, 10, false, ), + empty: (ValidBlobInfoV2::default(), 0, false), ] } - fn test_blob_status_is_inexistent_for_expired_blobs( - blob_info: ValidBlobInfoV2, - epoch_not_expired: Epoch, - epoch_expired: Epoch, - ) { - assert_ne!( - BlobInfoV2::Valid(blob_info.clone()).to_blob_status(epoch_not_expired), - BlobStatus::Nonexistent, - ); - assert_eq!( - BlobInfoV2::Valid(blob_info).to_blob_status(epoch_expired), - BlobStatus::Nonexistent, - ); + fn test_v2_is_registered(info: ValidBlobInfoV2, epoch: Epoch, expected: bool) { + info.check_invariants().unwrap(); + assert_eq!(BlobInfoV2::Valid(info).is_registered(epoch), expected); + } + + #[test] + fn v2_to_blob_status_combines_regular_and_pool_counts() { + let info = BlobInfoV2::Valid(ValidBlobInfoV2 { + count_deletable_total: 2, + count_deletable_certified: 1, + count_pooled_refs_total: 3, + count_pooled_refs_certified: 2, + initial_certified_epoch: Some(1), + ..Default::default() + }); + let BlobStatus::Deletable { + deletable_counts, .. + } = info.to_blob_status(0) + else { + panic!("expected Deletable") + }; + assert_eq!(deletable_counts.count_deletable_total, 5); + assert_eq!(deletable_counts.count_deletable_certified, 3); + } + + // --- V2 invariant violation detection --- + param_test! { + test_v2_invariant_violations: [ + pool_total_lt_certified: (ValidBlobInfoV2 { + count_pooled_refs_certified: 1, + initial_certified_epoch: Some(1), ..Default::default() + }), + certified_epoch_without_refs: (ValidBlobInfoV2 { + initial_certified_epoch: Some(1), ..Default::default() + }), + certified_refs_without_epoch: (ValidBlobInfoV2 { + count_pooled_refs_total: 1, count_pooled_refs_certified: 1, + ..Default::default() + }), + ] + } + fn test_v2_invariant_violations(info: ValidBlobInfoV2) { + assert!(info.check_invariants().is_err()); + } + + // --- PerObjectBlobInfoV2 --- + fn make_per_object_v2(end_epoch_info: EndEpochInfo) -> PerObjectBlobInfoV2 { + PerObjectBlobInfoV2 { + blob_id: walrus_core::test_utils::blob_id_from_u64(42), + registered_epoch: 1, + certified_epoch: None, + end_epoch_info, + deletable: true, + event: event_id_for_testing(), + deleted: false, + } + } + + #[test] + fn per_object_v2_certify_and_delete_merge() { + let info = make_per_object_v2(EndEpochInfo::StoragePool(pool_id())); + let cert_operand = PerObjectBlobInfoMergeOperand { + change_type: BlobStatusChangeType::Certify, + change_info: BlobStatusChangeInfo { + blob_id: walrus_core::test_utils::blob_id_from_u64(42), + deletable: true, + epoch: 3, + end_epoch: 0, + status_event: event_id_for_testing(), + }, + }; + let certified = info.merge_with(cert_operand); + assert_eq!(certified.certified_epoch, Some(3)); + assert!(!certified.deleted); + + let del_operand = PerObjectBlobInfoMergeOperand { + change_type: BlobStatusChangeType::Delete { + was_certified: true, + }, + change_info: BlobStatusChangeInfo { + blob_id: walrus_core::test_utils::blob_id_from_u64(42), + deletable: true, + epoch: 5, + end_epoch: 0, + status_event: event_id_for_testing(), + }, + }; + let deleted = certified.merge_with(del_operand); + assert!(deleted.deleted); + assert!(!deleted.is_registered(0)); } } From 8eba00b72f223a5cceb452e21b026f38a2981c98 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 12 Mar 2026 14:15:32 -0700 Subject: [PATCH 4/7] rename PermanentBlobInfoV1 to PermanentBlobInfo and move it to top level blob info module. This commit also include some minor code fixes --- crates/walrus-service/src/node/storage.rs | 4 +- .../src/node/storage/blob_info.rs | 92 ++++++++- .../node/storage/blob_info/blob_info_v1.rs | 192 +++++------------- .../node/storage/blob_info/blob_info_v2.rs | 62 +++--- 4 files changed, 178 insertions(+), 172 deletions(-) diff --git a/crates/walrus-service/src/node/storage.rs b/crates/walrus-service/src/node/storage.rs index eaa0e445c7..4ed9e67fcb 100644 --- a/crates/walrus-service/src/node/storage.rs +++ b/crates/walrus-service/src/node/storage.rs @@ -1299,7 +1299,7 @@ pub(crate) mod tests { BlobInfoMergeOperand, BlobInfoV1, BlobStatusChangeType, - PermanentBlobInfoV1, + PermanentBlobInfo, ValidBlobInfoV1, }; use constants::{ @@ -1508,7 +1508,7 @@ pub(crate) mod tests { // Set correct registered event. let BlobInfo::V1(BlobInfoV1::Valid(ValidBlobInfoV1 { - permanent_total: Some(PermanentBlobInfoV1 { event, .. }), + permanent_total: Some(PermanentBlobInfo { event, .. }), .. })) = &mut state1 else { diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index b89357b532..f7a5a1ae21 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -9,6 +9,7 @@ mod blob_info_v2; use std::{ collections::HashSet, fmt::Debug, + num::NonZeroU32, ops::Bound::{self, Unbounded}, sync::{Arc, Mutex}, }; @@ -30,7 +31,7 @@ use walrus_storage_node_client::api::BlobStatus; use walrus_sui::types::{BlobCertified, BlobDeleted, BlobEvent, BlobRegistered, InvalidBlobId}; #[cfg(test)] -pub(crate) use self::blob_info_v1::{PermanentBlobInfoV1, ValidBlobInfoV1}; +pub(crate) use self::blob_info_v1::ValidBlobInfoV1; use self::per_object_blob_info::PerObjectBlobInfoMergeOperand; pub(crate) use self::{ blob_info_v1::BlobInfoV1, @@ -1028,6 +1029,92 @@ impl ChangeTypeAndInfo for BlobDeleted { } } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct PermanentBlobInfo { + /// The total number of `Blob` objects for that blob ID with the given status. + pub count: NonZeroU32, + /// The latest expiration epoch among these objects. + pub end_epoch: Epoch, + /// The ID of the first blob event that led to the status with the given `end_epoch`. + pub event: EventID, +} + +impl PermanentBlobInfo { + /// Creates a new `PermanentBlobInfo` object for the first blob with the given `end_epoch` and + /// `event`. + pub(crate) fn new_first(end_epoch: Epoch, event: EventID) -> Self { + Self { + count: NonZeroU32::new(1).expect("1 is non-zero"), + end_epoch, + event, + } + } + + /// Updates `self` with the `change_info`, increasing the count if `increase_count == true`. + /// + /// # Panics + /// + /// Panics if the change info has `deletable == true`. + pub(crate) fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { + assert!(!change_info.deletable); + + if increase_count { + self.count = self.count.saturating_add(1) + }; + if change_info.end_epoch > self.end_epoch { + *self = PermanentBlobInfo { + count: self.count, + end_epoch: change_info.end_epoch, + event: change_info.status_event, + }; + } + } + + /// Updates `existing_info` with the change info or creates a new `Self` if the input is `None`. + /// + /// # Panics + /// + /// Panics if the change info has `deletable == true`. + fn update_optional(existing_info: &mut Option, change_info: &BlobStatusChangeInfo) { + let BlobStatusChangeInfo { + epoch: _, + end_epoch: new_end_epoch, + status_event: new_status_event, + deletable, + blob_id: _, + } = change_info; + assert!(!deletable); + + match existing_info { + None => { + *existing_info = Some(PermanentBlobInfo::new_first( + *new_end_epoch, + *new_status_event, + )) + } + Some(permanent_blob_info) => permanent_blob_info.update(change_info, true), + } + } + + #[cfg(test)] + pub(crate) fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { + Self { + count: NonZeroU32::new(count).expect("count must be non-zero"), + end_epoch, + event: walrus_sui::test_utils::fixed_event_id_for_testing(event_seq), + } + } + + #[cfg(test)] + pub(crate) fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { + Self { + count: NonZeroU32::new(count).expect("count must be non-zero"), + end_epoch, + event: walrus_sui::test_utils::event_id_for_testing(), + } + } +} + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) enum BlobInfoMergeOperand { MarkMetadataStored(bool), @@ -1188,8 +1275,7 @@ impl BlobInfo { }, BlobCertificationStatus::Registered | BlobCertificationStatus::Certified => { - let permanent_total = - PermanentBlobInfoV1::new_first(end_epoch, current_status_event); + let permanent_total = PermanentBlobInfo::new_first(end_epoch, current_status_event); let permanent_certified = matches!(status, BlobCertificationStatus::Certified) .then(|| permanent_total.clone()); ValidBlobInfoV1 { diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs index e15167650b..8a2538d737 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs @@ -17,6 +17,7 @@ use super::{ BlobStatusChangeType, CertifiedBlobInfoApi, Mergeable, + PermanentBlobInfo, ToBytes, per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, }; @@ -44,8 +45,8 @@ pub(crate) struct ValidBlobInfoV1 { pub is_metadata_stored: bool, pub count_deletable_total: u32, pub count_deletable_certified: u32, - pub permanent_total: Option, - pub permanent_certified: Option, + pub permanent_total: Option, + pub permanent_certified: Option, pub initial_certified_epoch: Option, // Note: The following helper fields were used in the past to approximate the blob status for @@ -91,7 +92,7 @@ impl ValidBlobInfoV1 { }; let initial_certified_epoch = self.initial_certified_epoch; - if let Some(PermanentBlobInfoV1 { + if let Some(PermanentBlobInfo { end_epoch, event, .. }) = self.permanent_certified.as_ref() && *end_epoch > current_epoch @@ -104,7 +105,7 @@ impl ValidBlobInfoV1 { initial_certified_epoch, }; } - if let Some(PermanentBlobInfoV1 { + if let Some(PermanentBlobInfo { end_epoch, event, .. }) = self.permanent_total.as_ref() && *end_epoch > current_epoch @@ -308,22 +309,22 @@ impl ValidBlobInfoV1 { } } - /// Processes a register status change on the [`Option`] object + /// Processes a register status change on the [`Option`] object /// representing all permanent blobs. pub(crate) fn register_permanent( - permanent_total: &mut Option, + permanent_total: &mut Option, change_info: &BlobStatusChangeInfo, ) { - PermanentBlobInfoV1::update_optional(permanent_total, change_info) + PermanentBlobInfo::update_optional(permanent_total, change_info) } - /// Processes a certify status change on the [`PermanentBlobInfoV1`] objects representing all + /// Processes a certify status change on the [`PermanentBlobInfo`] objects representing all /// and the certified permanent blobs. /// /// Returns whether the update was successful. pub(crate) fn certify_permanent( - permanent_total: &Option, - permanent_certified: &mut Option, + permanent_total: &Option, + permanent_certified: &mut Option, change_info: &BlobStatusChangeInfo, ) -> bool { let Some(permanent_total) = permanent_total else { @@ -350,14 +351,14 @@ impl ValidBlobInfoV1 { tracing::error!("attempt to certify a permanent blob before corresponding register"); return false; } - PermanentBlobInfoV1::update_optional(permanent_certified, change_info); + PermanentBlobInfo::update_optional(permanent_certified, change_info); true } - /// Processes an extend status change on the [`PermanentBlobInfoV1`] object representing the + /// Processes an extend status change on the [`PermanentBlobInfo`] object representing the /// certified permanent blobs. pub(crate) fn extend_permanent( - permanent_info: &mut Option, + permanent_info: &mut Option, change_info: &BlobStatusChangeInfo, ) { let Some(permanent_info) = permanent_info else { @@ -368,7 +369,7 @@ impl ValidBlobInfoV1 { permanent_info.update(change_info, false); } - /// Processes a delete status change on the [`PermanentBlobInfoV1`] objects representing all and + /// Processes a delete status change on the [`PermanentBlobInfo`] objects representing all and /// the certified permanent blobs. /// /// This is called when blobs expire at the end of an epoch. @@ -380,10 +381,10 @@ impl ValidBlobInfoV1 { self.maybe_unset_initial_certified_epoch(); } - pub(crate) fn decrement_blob_info_inner(blob_info_inner: &mut Option) { + pub(crate) fn decrement_blob_info_inner(blob_info_inner: &mut Option) { match blob_info_inner { None => tracing::error!("attempt to delete a permanent blob when none is tracked"), - Some(PermanentBlobInfoV1 { count, .. }) => { + Some(PermanentBlobInfo { count, .. }) => { if count.get() == 1 { *blob_info_inner = None; } else { @@ -446,92 +447,6 @@ impl ValidBlobInfoV1 { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct PermanentBlobInfoV1 { - /// The total number of `Blob` objects for that blob ID with the given status. - pub count: NonZeroU32, - /// The latest expiration epoch among these objects. - pub end_epoch: Epoch, - /// The ID of the first blob event that led to the status with the given `end_epoch`. - pub event: EventID, -} - -impl PermanentBlobInfoV1 { - /// Creates a new `PermanentBlobInfoV1` object for the first blob with the given `end_epoch` and - /// `event`. - pub(crate) fn new_first(end_epoch: Epoch, event: EventID) -> Self { - Self { - count: NonZeroU32::new(1).expect("1 is non-zero"), - end_epoch, - event, - } - } - - /// Updates `self` with the `change_info`, increasing the count if `increase_count == true`. - /// - /// # Panics - /// - /// Panics if the change info has `deletable == true`. - pub(crate) fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { - assert!(!change_info.deletable); - - if increase_count { - self.count = self.count.saturating_add(1) - }; - if change_info.end_epoch > self.end_epoch { - *self = PermanentBlobInfoV1 { - count: self.count, - end_epoch: change_info.end_epoch, - event: change_info.status_event, - }; - } - } - - /// Updates `existing_info` with the change info or creates a new `Self` if the input is `None`. - /// - /// # Panics - /// - /// Panics if the change info has `deletable == true`. - fn update_optional(existing_info: &mut Option, change_info: &BlobStatusChangeInfo) { - let BlobStatusChangeInfo { - epoch: _, - end_epoch: new_end_epoch, - status_event: new_status_event, - deletable, - blob_id: _, - } = change_info; - assert!(!deletable); - - match existing_info { - None => { - *existing_info = Some(PermanentBlobInfoV1::new_first( - *new_end_epoch, - *new_status_event, - )) - } - Some(permanent_blob_info) => permanent_blob_info.update(change_info, true), - } - } - - #[cfg(test)] - pub(crate) fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { - Self { - count: NonZeroU32::new(count).expect("count must be non-zero"), - end_epoch, - event: walrus_sui::test_utils::fixed_event_id_for_testing(event_seq), - } - } - - #[cfg(test)] - pub(crate) fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { - Self { - count: NonZeroU32::new(count).expect("count must be non-zero"), - end_epoch, - event: walrus_sui::test_utils::event_id_for_testing(), - } - } -} - impl CertifiedBlobInfoApi for BlobInfoV1 { fn is_certified(&self, current_epoch: Epoch) -> bool { if let Self::Valid(valid_blob_info) = self { @@ -694,7 +609,7 @@ impl Mergeable for BlobInfoV1 { } } else { ValidBlobInfoV1 { - permanent_total: Some(PermanentBlobInfoV1::new_first( + permanent_total: Some(PermanentBlobInfo::new_first( end_epoch, status_event, )), @@ -737,9 +652,14 @@ impl Mergeable for BlobInfoV1 { ); None } + BlobInfoMergeOperand::PoolExpired { .. } => { + // if the first event is a pool expired before the blob is registered, there may + // be a race between GC and pool event processing. Don't need to create a blob info + // for this case. + None + } // Pool operands should never reach V1 — they are intercepted at the BlobInfo level. - BlobInfoMergeOperand::PooledBlobChangeStatus { .. } - | BlobInfoMergeOperand::PoolExpired { .. } => { + BlobInfoMergeOperand::PooledBlobChangeStatus { .. } => { unreachable!("pool operands should be handled in BlobInfoV2") } } @@ -1025,12 +945,12 @@ mod tests { ..Default::default() }), permanent: (ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), ..Default::default() }), permanent_certified: (ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), initial_certified_epoch: Some(1), ..Default::default() }), @@ -1067,12 +987,12 @@ mod tests { ..Default::default() }), permanent: (ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), ..Default::default() }), permanent_certified: (ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), initial_certified_epoch: Some(1), ..Default::default() }), @@ -1200,7 +1120,7 @@ mod tests { BlobStatusChangeType::Register, false, 1, 2, fixed_event_id_for_testing(0) ), ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, ), @@ -1228,8 +1148,8 @@ mod tests { extend_permanent: ( ValidBlobInfoV1{ initial_certified_epoch: Some(0), - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 4, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 4, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 4, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 4, 1)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( @@ -1237,8 +1157,8 @@ mod tests { ), ValidBlobInfoV1{ initial_certified_epoch: Some(0), - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 42, 2)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 42, 2)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 42, 2)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 42, 2)), ..Default::default() }, ), @@ -1266,8 +1186,8 @@ mod tests { certify_outdated_permanent: ( ValidBlobInfoV1{ initial_certified_epoch: Some(2), - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 42, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_for_testing(1, 5)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 42, 0)), + permanent_certified: Some(PermanentBlobInfo::new_for_testing(1, 5)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( @@ -1275,28 +1195,28 @@ mod tests { ), ValidBlobInfoV1{ initial_certified_epoch: Some(7), - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 42, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 42, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 42, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(2, 42, 1)), ..Default::default() }, ), register_additional_permanent: ( ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( BlobStatusChangeType::Register, false, 2, 3, fixed_event_id_for_testing(1) ), ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 1)), ..Default::default() }, ), expire_permanent_blob: ( ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(3, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(3, 5, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 1)), initial_certified_epoch: Some(1), ..Default::default() }, @@ -1304,16 +1224,16 @@ mod tests { was_certified: true, }, ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 1)), initial_certified_epoch: Some(1), ..Default::default() }, ), expire_last_permanent_blob: ( ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 1)), initial_certified_epoch: Some(1), ..Default::default() }, @@ -1321,7 +1241,7 @@ mod tests { was_certified: true, }, ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 0)), permanent_certified: None, initial_certified_epoch: None, ..Default::default() @@ -1421,20 +1341,20 @@ mod tests { ), expire_uncertified_permanent_blob: ( ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(3, 5, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(3, 5, 0)), ..Default::default() }, BlobInfoMergeOperand::PermanentExpired { was_certified: false, }, ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 0)), ..Default::default() }, ), expire_last_uncertified_permanent_blob: ( ValidBlobInfoV1{ - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 0)), ..Default::default() }, BlobInfoMergeOperand::PermanentExpired { @@ -1540,7 +1460,7 @@ mod tests { test_blob_status_is_inexistent_for_expired_blobs: [ expired_permanent_registered_0: ( ValidBlobInfoV1 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, 1, @@ -1548,7 +1468,7 @@ mod tests { ), expired_permanent_registered_1: ( ValidBlobInfoV1 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), ..Default::default() }, 2, @@ -1556,8 +1476,8 @@ mod tests { ), expired_permanent_certified: ( ValidBlobInfoV1 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 2, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 2, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, 1, diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs index 642645dd61..3503aec6cb 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs @@ -15,8 +15,9 @@ use super::{ BlobStatusChangeType, CertifiedBlobInfoApi, Mergeable, + PermanentBlobInfo, ToBytes, - blob_info_v1::{BlobInfoV1, PermanentBlobInfoV1, ValidBlobInfoV1}, + blob_info_v1::{BlobInfoV1, ValidBlobInfoV1}, per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, }; @@ -42,8 +43,8 @@ pub(crate) struct ValidBlobInfoV2 { // Regular blob fields (same as V1). pub count_deletable_total: u32, pub count_deletable_certified: u32, - pub permanent_total: Option, - pub permanent_certified: Option, + pub permanent_total: Option, + pub permanent_certified: Option, // Storage pool references counters. pub count_pooled_refs_total: u32, @@ -83,7 +84,7 @@ impl ValidBlobInfoV2 { .saturating_add(self.count_pooled_refs_certified), }; - if let Some(PermanentBlobInfoV1 { + if let Some(PermanentBlobInfo { end_epoch, event, .. }) = self.permanent_certified.as_ref() && *end_epoch > current_epoch @@ -96,7 +97,7 @@ impl ValidBlobInfoV2 { initial_certified_epoch, }; } - if let Some(PermanentBlobInfoV1 { + if let Some(PermanentBlobInfo { end_epoch, event, .. }) = self.permanent_total.as_ref() && *end_epoch > current_epoch @@ -374,7 +375,7 @@ impl BlobInfoApi for BlobInfoV2 { ) } - fn is_registered(&self, _current_epoch: Epoch) -> bool { + fn is_registered(&self, current_epoch: Epoch) -> bool { let Self::Valid(v) = self else { return false; }; @@ -382,7 +383,7 @@ impl BlobInfoApi for BlobInfoV2 { let exists_registered_permanent_blob = v .permanent_total .as_ref() - .is_some_and(|p| p.end_epoch > _current_epoch); + .is_some_and(|p| p.end_epoch > current_epoch); // Note that at the beginning of the epoch before GC runs, newly expired deletable blob or // pooled blob's registered counter may still be non-zero, but the blob is already expired. @@ -470,7 +471,6 @@ impl Mergeable for BlobInfoV2 { BlobInfoMergeOperand::PooledBlobChangeStatus { change_type, change_info, - .. }, ) => match change_type { BlobStatusChangeType::Register => { @@ -539,7 +539,7 @@ impl Mergeable for BlobInfoV2 { } } else { ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_first(end_epoch, status_event)), + permanent_total: Some(PermanentBlobInfo::new_first(end_epoch, status_event)), ..Default::default() } })), @@ -786,8 +786,8 @@ mod tests { initial_certified_epoch: Some(0), ..Default::default() }), permanent_certified: (ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), initial_certified_epoch: Some(1), ..Default::default() }), pool_refs: (ValidBlobInfoV2 { @@ -833,8 +833,8 @@ mod tests { initial_certified_epoch: Some(0), ..Default::default() }), permanent_certified: (ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), initial_certified_epoch: Some(1), ..Default::default() }), pool_refs: (ValidBlobInfoV2 { @@ -963,13 +963,13 @@ mod tests { fixed_event_id_for_testing(0) ), ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, ), register_additional_permanent: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 2, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 2, 0)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( @@ -977,15 +977,15 @@ mod tests { fixed_event_id_for_testing(1) ), ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 3, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 3, 1)), ..Default::default() }, ), extend_permanent: ( ValidBlobInfoV2 { initial_certified_epoch: Some(0), - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 4, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 4, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 4, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 4, 1)), ..Default::default() }, BlobInfoMergeOperand::new_change_for_testing( @@ -994,33 +994,33 @@ mod tests { ), ValidBlobInfoV2 { initial_certified_epoch: Some(0), - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 42, 2)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 42, 2)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 42, 2)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 42, 2)), ..Default::default() }, ), expire_permanent_blob: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(3, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(3, 5, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 1)), initial_certified_epoch: Some(1), ..Default::default() }, BlobInfoMergeOperand::PermanentExpired { was_certified: true }, ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 1)), initial_certified_epoch: Some(1), ..Default::default() }, ), expire_last_certified_permanent_clears_epoch: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 5, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 5, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 1)), initial_certified_epoch: Some(1), ..Default::default() }, BlobInfoMergeOperand::PermanentExpired { was_certified: true }, ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 5, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 5, 0)), permanent_certified: None, initial_certified_epoch: None, ..Default::default() }, @@ -1112,8 +1112,8 @@ mod tests { is_metadata_stored: true, count_deletable_total: 3, count_deletable_certified: 1, - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(2, 10, 0)), - permanent_certified: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 10, 1)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(2, 10, 0)), + permanent_certified: Some(PermanentBlobInfo::new_fixed_for_testing(1, 10, 1)), initial_certified_epoch: Some(5), latest_seen_deletable_registered_end_epoch: Some(8), latest_seen_deletable_certified_end_epoch: Some(7), @@ -1282,13 +1282,13 @@ mod tests { ), permanent_before_expiry: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 10, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 10, 0)), ..Default::default() }, 9, true, ), permanent_at_expiry: ( ValidBlobInfoV2 { - permanent_total: Some(PermanentBlobInfoV1::new_fixed_for_testing(1, 10, 0)), + permanent_total: Some(PermanentBlobInfo::new_fixed_for_testing(1, 10, 0)), ..Default::default() }, 10, false, ), From ac87a22a9f6658378d1e491c1b5a1ee6fc7e7516 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 12 Mar 2026 15:15:25 -0700 Subject: [PATCH 5/7] move register_permanent, certify_permanent, and extend_permanent to PermanentBlobInfo, move PermanentBlobInfo to its own module --- .../src/node/storage/blob_info.rs | 88 +-------- .../node/storage/blob_info/blob_info_v1.rs | 88 +-------- .../node/storage/blob_info/blob_info_v2.rs | 19 +- .../node/storage/blob_info/perm_blob_info.rs | 176 ++++++++++++++++++ 4 files changed, 194 insertions(+), 177 deletions(-) create mode 100644 crates/walrus-service/src/node/storage/blob_info/perm_blob_info.rs diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index f7a5a1ae21..8e6db23ffe 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -5,11 +5,11 @@ mod blob_info_v1; mod blob_info_v2; +mod perm_blob_info; use std::{ collections::HashSet, fmt::Debug, - num::NonZeroU32, ops::Bound::{self, Unbounded}, sync::{Arc, Mutex}, }; @@ -1029,91 +1029,7 @@ impl ChangeTypeAndInfo for BlobDeleted { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct PermanentBlobInfo { - /// The total number of `Blob` objects for that blob ID with the given status. - pub count: NonZeroU32, - /// The latest expiration epoch among these objects. - pub end_epoch: Epoch, - /// The ID of the first blob event that led to the status with the given `end_epoch`. - pub event: EventID, -} - -impl PermanentBlobInfo { - /// Creates a new `PermanentBlobInfo` object for the first blob with the given `end_epoch` and - /// `event`. - pub(crate) fn new_first(end_epoch: Epoch, event: EventID) -> Self { - Self { - count: NonZeroU32::new(1).expect("1 is non-zero"), - end_epoch, - event, - } - } - - /// Updates `self` with the `change_info`, increasing the count if `increase_count == true`. - /// - /// # Panics - /// - /// Panics if the change info has `deletable == true`. - pub(crate) fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { - assert!(!change_info.deletable); - - if increase_count { - self.count = self.count.saturating_add(1) - }; - if change_info.end_epoch > self.end_epoch { - *self = PermanentBlobInfo { - count: self.count, - end_epoch: change_info.end_epoch, - event: change_info.status_event, - }; - } - } - - /// Updates `existing_info` with the change info or creates a new `Self` if the input is `None`. - /// - /// # Panics - /// - /// Panics if the change info has `deletable == true`. - fn update_optional(existing_info: &mut Option, change_info: &BlobStatusChangeInfo) { - let BlobStatusChangeInfo { - epoch: _, - end_epoch: new_end_epoch, - status_event: new_status_event, - deletable, - blob_id: _, - } = change_info; - assert!(!deletable); - - match existing_info { - None => { - *existing_info = Some(PermanentBlobInfo::new_first( - *new_end_epoch, - *new_status_event, - )) - } - Some(permanent_blob_info) => permanent_blob_info.update(change_info, true), - } - } - - #[cfg(test)] - pub(crate) fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { - Self { - count: NonZeroU32::new(count).expect("count must be non-zero"), - end_epoch, - event: walrus_sui::test_utils::fixed_event_id_for_testing(event_seq), - } - } - - #[cfg(test)] - pub(crate) fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { - Self { - count: NonZeroU32::new(count).expect("count must be non-zero"), - end_epoch, - event: walrus_sui::test_utils::event_id_for_testing(), - } - } -} +pub(crate) use self::perm_blob_info::PermanentBlobInfo; #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) enum BlobInfoMergeOperand { diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs index 8a2538d737..3c65d1db8e 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs @@ -3,8 +3,6 @@ //! V1 blob info types and merge logic. -use std::num::NonZeroU32; - use serde::{Deserialize, Serialize}; use sui_types::{base_types::ObjectID, event::EventID}; use walrus_core::{BlobId, Epoch}; @@ -202,10 +200,10 @@ impl ValidBlobInfoV1 { } else { match change_type { BlobStatusChangeType::Register => { - Self::register_permanent(&mut self.permanent_total, &change_info); + PermanentBlobInfo::register(&mut self.permanent_total, &change_info); } BlobStatusChangeType::Certify => { - if !Self::certify_permanent( + if !PermanentBlobInfo::certify( &self.permanent_total, &mut self.permanent_certified, &change_info, @@ -215,8 +213,8 @@ impl ValidBlobInfoV1 { } } BlobStatusChangeType::Extend => { - Self::extend_permanent(&mut self.permanent_total, &change_info); - Self::extend_permanent(&mut self.permanent_certified, &change_info); + PermanentBlobInfo::extend(&mut self.permanent_total, &change_info); + PermanentBlobInfo::extend(&mut self.permanent_certified, &change_info); } BlobStatusChangeType::Delete { .. } => { tracing::error!("attempt to delete a permanent blob"); @@ -309,92 +307,18 @@ impl ValidBlobInfoV1 { } } - /// Processes a register status change on the [`Option`] object - /// representing all permanent blobs. - pub(crate) fn register_permanent( - permanent_total: &mut Option, - change_info: &BlobStatusChangeInfo, - ) { - PermanentBlobInfo::update_optional(permanent_total, change_info) - } - - /// Processes a certify status change on the [`PermanentBlobInfo`] objects representing all - /// and the certified permanent blobs. - /// - /// Returns whether the update was successful. - pub(crate) fn certify_permanent( - permanent_total: &Option, - permanent_certified: &mut Option, - change_info: &BlobStatusChangeInfo, - ) -> bool { - let Some(permanent_total) = permanent_total else { - tracing::error!("attempt to certify a permanent blob when none is tracked"); - return false; - }; - - let registered_end_epoch = permanent_total.end_epoch; - let certified_end_epoch = change_info.end_epoch; - if certified_end_epoch > registered_end_epoch { - tracing::error!( - registered_end_epoch, - certified_end_epoch, - "attempt to certify a permanent blob with later end epoch than any registered blob", - ); - return false; - } - if permanent_total.count.get() - <= permanent_certified - .as_ref() - .map(|p| p.count.get()) - .unwrap_or_default() - { - tracing::error!("attempt to certify a permanent blob before corresponding register"); - return false; - } - PermanentBlobInfo::update_optional(permanent_certified, change_info); - true - } - - /// Processes an extend status change on the [`PermanentBlobInfo`] object representing the - /// certified permanent blobs. - pub(crate) fn extend_permanent( - permanent_info: &mut Option, - change_info: &BlobStatusChangeInfo, - ) { - let Some(permanent_info) = permanent_info else { - tracing::error!("attempt to extend a permanent blob when none is tracked"); - return; - }; - - permanent_info.update(change_info, false); - } - /// Processes a delete status change on the [`PermanentBlobInfo`] objects representing all and /// the certified permanent blobs. /// /// This is called when blobs expire at the end of an epoch. fn permanent_expired(&mut self, was_certified: bool) { - Self::decrement_blob_info_inner(&mut self.permanent_total); + PermanentBlobInfo::decrement(&mut self.permanent_total); if was_certified { - Self::decrement_blob_info_inner(&mut self.permanent_certified); + PermanentBlobInfo::decrement(&mut self.permanent_certified); } self.maybe_unset_initial_certified_epoch(); } - pub(crate) fn decrement_blob_info_inner(blob_info_inner: &mut Option) { - match blob_info_inner { - None => tracing::error!("attempt to delete a permanent blob when none is tracked"), - Some(PermanentBlobInfo { count, .. }) => { - if count.get() == 1 { - *blob_info_inner = None; - } else { - *count = NonZeroU32::new(count.get() - 1) - .expect("we just checked that `count` is at least 2") - } - } - } - } - /// Checks the invariants of the aggregate blob info, returning an error if any invariant is /// violated. pub(crate) fn check_invariants(&self) -> anyhow::Result<()> { diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs index 3503aec6cb..fe0f55c9cc 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs @@ -188,10 +188,10 @@ impl ValidBlobInfoV2 { // These should be the same as V1. match change_type { BlobStatusChangeType::Register => { - ValidBlobInfoV1::register_permanent(&mut self.permanent_total, &change_info); + PermanentBlobInfo::register(&mut self.permanent_total, &change_info); } BlobStatusChangeType::Certify => { - if !ValidBlobInfoV1::certify_permanent( + if !PermanentBlobInfo::certify( &self.permanent_total, &mut self.permanent_certified, &change_info, @@ -201,8 +201,8 @@ impl ValidBlobInfoV2 { } } BlobStatusChangeType::Extend => { - ValidBlobInfoV1::extend_permanent(&mut self.permanent_total, &change_info); - ValidBlobInfoV1::extend_permanent(&mut self.permanent_certified, &change_info); + PermanentBlobInfo::extend(&mut self.permanent_total, &change_info); + PermanentBlobInfo::extend(&mut self.permanent_certified, &change_info); } BlobStatusChangeType::Delete { .. } => { tracing::error!("attempt to delete a permanent blob"); @@ -256,9 +256,9 @@ impl ValidBlobInfoV2 { } fn permanent_expired(&mut self, was_certified: bool) { - ValidBlobInfoV1::decrement_blob_info_inner(&mut self.permanent_total); + PermanentBlobInfo::decrement(&mut self.permanent_total); if was_certified { - ValidBlobInfoV1::decrement_blob_info_inner(&mut self.permanent_certified); + PermanentBlobInfo::decrement(&mut self.permanent_certified); } self.maybe_unset_initial_certified_epoch(); } @@ -302,7 +302,7 @@ impl ValidBlobInfoV2 { /// Registers a blob in a storage pool. fn pool_register(&mut self) { - self.count_pooled_refs_total += 1; + self.count_pooled_refs_total = self.count_pooled_refs_total.saturating_add(1); } /// Certifies a blob in a storage pool. @@ -313,7 +313,7 @@ impl ValidBlobInfoV2 { tracing::error!("attempt to certify a pool blob before corresponding register"); return; } - self.count_pooled_refs_certified += 1; + self.count_pooled_refs_certified = self.count_pooled_refs_certified.saturating_add(1); self.update_initial_certified_epoch(epoch, !was_certified); } @@ -387,7 +387,8 @@ impl BlobInfoApi for BlobInfoV2 { // Note that at the beginning of the epoch before GC runs, newly expired deletable blob or // pooled blob's registered counter may still be non-zero, but the blob is already expired. - // This will make the blob appears to be registered until GC finishes. + // This will make the blob appears to be registered until per object blob info GC finishes. + // The time window is expected to be very short (few seconds). exists_registered_permanent_blob || v.count_deletable_total > 0 || v.count_pooled_refs_total > 0 diff --git a/crates/walrus-service/src/node/storage/blob_info/perm_blob_info.rs b/crates/walrus-service/src/node/storage/blob_info/perm_blob_info.rs new file mode 100644 index 0000000000..0af730f367 --- /dev/null +++ b/crates/walrus-service/src/node/storage/blob_info/perm_blob_info.rs @@ -0,0 +1,176 @@ +// Copyright (c) Walrus Foundation +// SPDX-License-Identifier: Apache-2.0 + +//! Permanent blob info types and operations. + +use std::num::NonZeroU32; + +use serde::{Deserialize, Serialize}; +use sui_types::event::EventID; +use walrus_core::Epoch; + +use super::BlobStatusChangeInfo; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct PermanentBlobInfo { + /// The total number of `Blob` objects for that blob ID with the given status. + pub count: NonZeroU32, + /// The latest expiration epoch among these objects. + pub end_epoch: Epoch, + /// The ID of the first blob event that led to the status with the given `end_epoch`. + pub event: EventID, +} + +impl PermanentBlobInfo { + /// Creates a new `PermanentBlobInfo` object for the first blob with the given `end_epoch` and + /// `event`. + pub(super) fn new_first(end_epoch: Epoch, event: EventID) -> Self { + Self { + count: NonZeroU32::new(1).expect("1 is non-zero"), + end_epoch, + event, + } + } + + /// Processes a register status change on the [`Option`] object + /// representing all permanent blobs. + pub(super) fn register( + permanent_total: &mut Option, + change_info: &BlobStatusChangeInfo, + ) { + PermanentBlobInfo::update_optional(permanent_total, change_info) + } + + /// Processes a certify status change on the [`PermanentBlobInfo`] objects representing all + /// and the certified permanent blobs. + /// + /// Returns whether the update was successful. + pub(super) fn certify( + permanent_total: &Option, + permanent_certified: &mut Option, + change_info: &BlobStatusChangeInfo, + ) -> bool { + let Some(permanent_total) = permanent_total else { + tracing::error!("attempt to certify a permanent blob when none is tracked"); + return false; + }; + + let registered_end_epoch = permanent_total.end_epoch; + let certified_end_epoch = change_info.end_epoch; + if certified_end_epoch > registered_end_epoch { + tracing::error!( + registered_end_epoch, + certified_end_epoch, + "attempt to certify a permanent blob with later end epoch than any registered blob", + ); + return false; + } + if permanent_total.count.get() + <= permanent_certified + .as_ref() + .map(|p| p.count.get()) + .unwrap_or_default() + { + tracing::error!("attempt to certify a permanent blob before corresponding register"); + return false; + } + PermanentBlobInfo::update_optional(permanent_certified, change_info); + true + } + + /// Processes an extend status change on the [`PermanentBlobInfo`] object representing the + /// certified permanent blobs. + pub(super) fn extend( + permanent_info: &mut Option, + change_info: &BlobStatusChangeInfo, + ) { + let Some(permanent_info) = permanent_info else { + tracing::error!("attempt to extend a permanent blob when none is tracked"); + return; + }; + + permanent_info.update(change_info, false); + } + + /// Decrements the count of the given [`PermanentBlobInfo`], removing it if the count reaches + /// zero. + /// + /// This is called when blobs expire at the end of an epoch. + pub(super) fn decrement(blob_info_inner: &mut Option) { + match blob_info_inner { + None => tracing::error!("attempt to delete a permanent blob when none is tracked"), + Some(PermanentBlobInfo { count, .. }) => { + if count.get() == 1 { + *blob_info_inner = None; + } else { + *count = NonZeroU32::new(count.get() - 1) + .expect("we just checked that `count` is at least 2") + } + } + } + } + + /// Updates `self` with the `change_info`, increasing the count if `increase_count == true`. + /// + /// # Panics + /// + /// Panics if the change info has `deletable == true`. + fn update(&mut self, change_info: &BlobStatusChangeInfo, increase_count: bool) { + assert!(!change_info.deletable); + + if increase_count { + self.count = self.count.saturating_add(1) + }; + if change_info.end_epoch > self.end_epoch { + *self = PermanentBlobInfo { + count: self.count, + end_epoch: change_info.end_epoch, + event: change_info.status_event, + }; + } + } + + /// Updates `existing_info` with the change info or creates a new `Self` if the input is `None`. + /// + /// # Panics + /// + /// Panics if the change info has `deletable == true`. + fn update_optional(existing_info: &mut Option, change_info: &BlobStatusChangeInfo) { + let BlobStatusChangeInfo { + epoch: _, + end_epoch: new_end_epoch, + status_event: new_status_event, + deletable, + blob_id: _, + } = change_info; + assert!(!deletable); + + match existing_info { + None => { + *existing_info = Some(PermanentBlobInfo::new_first( + *new_end_epoch, + *new_status_event, + )) + } + Some(permanent_blob_info) => permanent_blob_info.update(change_info, true), + } + } + + #[cfg(test)] + pub(super) fn new_fixed_for_testing(count: u32, end_epoch: Epoch, event_seq: u64) -> Self { + Self { + count: NonZeroU32::new(count).expect("count must be non-zero"), + end_epoch, + event: walrus_sui::test_utils::fixed_event_id_for_testing(event_seq), + } + } + + #[cfg(test)] + pub(super) fn new_for_testing(count: u32, end_epoch: Epoch) -> Self { + Self { + count: NonZeroU32::new(count).expect("count must be non-zero"), + end_epoch, + event: walrus_sui::test_utils::event_id_for_testing(), + } + } +} From e0ca6c1faae59f827e46bc0a30851605263c1e18 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 12 Mar 2026 15:58:42 -0700 Subject: [PATCH 6/7] remove PerObjectBlobInfoV2 --- .../src/node/storage/blob_info.rs | 218 +++++++++++++----- .../node/storage/blob_info/blob_info_v1.rs | 157 +------------ .../node/storage/blob_info/blob_info_v2.rs | 216 +---------------- 3 files changed, 171 insertions(+), 420 deletions(-) diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index 8e6db23ffe..32e8c350e5 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -37,6 +37,7 @@ pub(crate) use self::{ blob_info_v1::BlobInfoV1, blob_info_v2::BlobInfoV2, per_object_blob_info::{PerObjectBlobInfo, PerObjectBlobInfoApi}, + perm_blob_info::PermanentBlobInfo, }; use super::{DatabaseTableOptionsFactory, constants}; use crate::{ @@ -606,13 +607,10 @@ impl BlobInfoTable { .safe_iter_with_snapshot(&snapshot) .context("failed to create per-object blob info snapshot iterator")? { - let (object_id, per_object_blob_info) = match result { - Ok(v) => v, - Err(e) => { - return Err(anyhow::anyhow!( - "error encountered while iterating over per-object blob info: {e:?}" - )); - } + let Ok((object_id, PerObjectBlobInfo::V1(per_object_blob_info))) = result else { + return Err(anyhow::anyhow!( + "error encountered while iterating over per-object blob info: {result:?}" + )); }; let blob_id = per_object_blob_info.blob_id(); per_object_table_blob_ids.insert(blob_id); @@ -661,15 +659,7 @@ impl BlobInfoTable { // Below checks the invariants of last seen epochs on deletable blobs, which is only // relevant for V1 per-object entries. if v1_blob && per_object_blob_info.is_deletable() { - let per_object_end_epoch = match &per_object_blob_info { - PerObjectBlobInfo::V1(v) => v.end_epoch, - PerObjectBlobInfo::V2(v) => match v.end_epoch_info { - per_object_blob_info::EndEpochInfo::Individual(e) => e, - per_object_blob_info::EndEpochInfo::StoragePool(_) => { - unreachable!("storage_pool_id() returned None above") - } - }, - }; + let per_object_end_epoch = per_object_blob_info.end_epoch; anyhow::ensure!( count_deletable_total > 0, "count_deletable_total is 0 for blob ID {blob_id}, even though a deletable \ @@ -683,13 +673,7 @@ impl BlobInfoTable { end epoch of a deletable blob object: {per_object_end_epoch} (object ID: \ {object_id})" ); - let certified_epoch = match &per_object_blob_info { - PerObjectBlobInfo::V1(v) => v.certified_epoch, - PerObjectBlobInfo::V2(_) => { - unreachable!("v2 blob should not check latest seen epochs") - } - }; - if certified_epoch.is_some() { + if per_object_blob_info.certified_epoch.is_some() { anyhow::ensure!( count_deletable_certified > 0, "count_deletable_certified is 0 for blob ID {blob_id}, even though a \ @@ -860,7 +844,7 @@ where } } -pub(crate) trait ToBytes: Serialize + Sized { +pub(super) trait ToBytes: Serialize + Sized { /// Converts the value to a `Vec`. /// /// Uses BCS encoding (which is assumed to succeed) by default. @@ -868,7 +852,7 @@ pub(crate) trait ToBytes: Serialize + Sized { bcs::to_bytes(self).expect("value must be BCS-serializable") } } -pub(crate) trait Mergeable: ToBytes + Debug + DeserializeOwned + Serialize + Sized { +trait Mergeable: ToBytes + Debug + DeserializeOwned + Serialize + Sized { type MergeOperand: Debug + DeserializeOwned + ToBytes; type Key: Debug + DeserializeOwned + std::fmt::Display; @@ -942,16 +926,16 @@ pub(crate) trait BlobInfoApi: CertifiedBlobInfoApi { } #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(crate) struct BlobStatusChangeInfo { - pub(crate) blob_id: BlobId, - pub(crate) deletable: bool, - pub(crate) epoch: Epoch, - pub(crate) end_epoch: Epoch, - pub(crate) status_event: EventID, +pub(super) struct BlobStatusChangeInfo { + pub(super) blob_id: BlobId, + pub(super) deletable: bool, + pub(super) epoch: Epoch, + pub(super) end_epoch: Epoch, + pub(super) status_event: EventID, } #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Copy)] -pub(crate) enum BlobStatusChangeType { +pub(super) enum BlobStatusChangeType { Register, Certify, // INV: Can only be applied to a certified blob. @@ -965,9 +949,9 @@ pub(crate) enum BlobStatusChangeType { /// blobs is determined by the pool's end_epoch (tracked in `storage_pool_end_epochs`), not by /// the individual blob's end_epoch at registration time. #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(crate) struct PooledBlobChangeInfo { - pub(crate) epoch: Epoch, - pub(crate) storage_pool_id: ObjectID, +pub(super) struct PooledBlobChangeInfo { + pub(super) epoch: Epoch, + pub(super) storage_pool_id: ObjectID, } trait ChangeTypeAndInfo { @@ -1029,10 +1013,8 @@ impl ChangeTypeAndInfo for BlobDeleted { } } -pub(crate) use self::perm_blob_info::PermanentBlobInfo; - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(crate) enum BlobInfoMergeOperand { +pub(super) enum BlobInfoMergeOperand { MarkMetadataStored(bool), MarkInvalid { epoch: Epoch, @@ -1251,10 +1233,6 @@ impl Mergeable for BlobInfo { mod per_object_blob_info { use super::*; - pub(crate) use super::{ - blob_info_v1::PerObjectBlobInfoV1, - blob_info_v2::{EndEpochInfo, PerObjectBlobInfoV2}, - }; #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) struct PerObjectBlobInfoMergeOperand { @@ -1305,8 +1283,6 @@ mod per_object_blob_info { fn is_registered(&self, current_epoch: Epoch) -> bool; /// Returns true iff the object is already deleted. fn is_deleted(&self) -> bool; - /// Returns the storage pool ID if this is a storage pool blob. - fn storage_pool_id(&self) -> Option; } #[enum_dispatch(CertifiedBlobInfoApi)] @@ -1314,7 +1290,6 @@ mod per_object_blob_info { #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub(crate) enum PerObjectBlobInfo { V1(PerObjectBlobInfoV1), - V2(PerObjectBlobInfoV2), } impl PerObjectBlobInfo { @@ -1349,20 +1324,159 @@ mod per_object_blob_info { fn merge_with(self, operand: Self::MergeOperand) -> Self { match self { Self::V1(value) => Self::V1(value.merge_with(operand)), - Self::V2(value) => Self::V2(value.merge_with(operand)), } } fn merge_new(operand: Self::MergeOperand) -> Option { - // We never create PerObjectBlobInfoV2 via merge operator. This is because the - // PerObjectBlobInfoMergeOperand struct can only used for V1 for registration. - // So any newly created PerObjectBlobInfoV2 is directly inserted into the table. - // - // The certify and delete operation can be used in both V1 and V2, so merge_with above - // works for both. PerObjectBlobInfoV1::merge_new(operand).map(Self::from) } } + + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] + pub(crate) struct PerObjectBlobInfoV1 { + /// The blob ID. + pub blob_id: BlobId, + /// The epoch in which the blob has been registered. + pub registered_epoch: Epoch, + /// The epoch in which the blob was first certified, `None` if the blob is uncertified. + pub certified_epoch: Option, + /// The epoch in which the blob expires. + pub end_epoch: Epoch, + /// Whether the blob is deletable. + pub deletable: bool, + /// The ID of the last blob event related to this object. + pub event: EventID, + /// Whether the blob has been deleted. + pub deleted: bool, + } + + impl CertifiedBlobInfoApi for PerObjectBlobInfoV1 { + fn is_certified(&self, current_epoch: Epoch) -> bool { + self.is_registered(current_epoch) + && self + .certified_epoch + .is_some_and(|epoch| epoch <= current_epoch) + } + + fn initial_certified_epoch(&self) -> Option { + self.certified_epoch + } + } + + impl PerObjectBlobInfoApi for PerObjectBlobInfoV1 { + fn blob_id(&self) -> BlobId { + self.blob_id + } + + fn is_deletable(&self) -> bool { + self.deletable + } + + fn is_registered(&self, current_epoch: Epoch) -> bool { + self.end_epoch > current_epoch && !self.deleted + } + + fn is_deleted(&self) -> bool { + self.deleted + } + } + + impl ToBytes for PerObjectBlobInfoV1 {} + + impl Mergeable for PerObjectBlobInfoV1 { + type MergeOperand = PerObjectBlobInfoMergeOperand; + type Key = ObjectID; + + fn merge_with( + mut self, + PerObjectBlobInfoMergeOperand { + change_type, + change_info, + }: PerObjectBlobInfoMergeOperand, + ) -> Self { + assert_eq!( + self.blob_id, change_info.blob_id, + "blob ID mismatch in merge operand" + ); + assert_eq!( + self.deletable, change_info.deletable, + "deletable mismatch in merge operand" + ); + assert!( + !self.deleted, + "attempt to update an already deleted blob {}", + self.blob_id + ); + self.event = change_info.status_event; + match change_type { + // We ensure that the blob info is only updated a single time for each event. So if + // we see a duplicated registered or certified event for the some object, this is a + // serious bug somewhere. + BlobStatusChangeType::Register => { + panic!( + "cannot register an already registered blob {}", + self.blob_id + ); + } + BlobStatusChangeType::Certify => { + assert!( + self.certified_epoch.is_none(), + "cannot certify an already certified blob {}", + self.blob_id + ); + self.certified_epoch = Some(change_info.epoch); + } + BlobStatusChangeType::Extend => { + assert!( + self.certified_epoch.is_some(), + "cannot extend an uncertified blob {}", + self.blob_id + ); + self.end_epoch = change_info.end_epoch; + } + BlobStatusChangeType::Delete { was_certified } => { + assert_eq!(self.certified_epoch.is_some(), was_certified); + self.deleted = true; + } + } + self + } + + fn merge_new(operand: Self::MergeOperand) -> Option { + let PerObjectBlobInfoMergeOperand { + change_type: BlobStatusChangeType::Register, + change_info: + BlobStatusChangeInfo { + blob_id, + deletable, + epoch, + end_epoch, + status_event, + }, + } = operand + else { + tracing::error!( + ?operand, + "encountered an update other than 'register' for an untracked blob object" + ); + debug_assert!( + false, + "encountered an update other than 'register' for an untracked blob object: \ + {operand:?}" + ); + return None; + }; + Some(Self { + blob_id, + registered_epoch: epoch, + certified_epoch: None, + end_epoch, + deletable, + event: status_event, + deleted: false, + }) + } + } } fn deserialize_from_db<'de, T>(data: &'de [u8]) -> Option diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs index 3c65d1db8e..3a5129ad29 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs @@ -4,7 +4,7 @@ //! V1 blob info types and merge logic. use serde::{Deserialize, Serialize}; -use sui_types::{base_types::ObjectID, event::EventID}; +use sui_types::event::EventID; use walrus_core::{BlobId, Epoch}; use walrus_storage_node_client::api::{BlobStatus, DeletableCounts}; @@ -17,7 +17,6 @@ use super::{ Mergeable, PermanentBlobInfo, ToBytes, - per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, }; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -590,160 +589,6 @@ impl Mergeable for BlobInfoV1 { } } -// ============================================================================= -// Per-object blob info V1 -// ============================================================================= - -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(crate) struct PerObjectBlobInfoV1 { - /// The blob ID. - pub blob_id: BlobId, - /// The epoch in which the blob has been registered. - pub registered_epoch: Epoch, - /// The epoch in which the blob was first certified, `None` if the blob is uncertified. - pub certified_epoch: Option, - /// The epoch in which the blob expires. - pub end_epoch: Epoch, - /// Whether the blob is deletable. - pub deletable: bool, - /// The ID of the last blob event related to this object. - pub event: EventID, - /// Whether the blob has been deleted. - pub deleted: bool, -} - -impl CertifiedBlobInfoApi for PerObjectBlobInfoV1 { - fn is_certified(&self, current_epoch: Epoch) -> bool { - self.is_registered(current_epoch) - && self - .certified_epoch - .is_some_and(|epoch| epoch <= current_epoch) - } - - fn initial_certified_epoch(&self) -> Option { - self.certified_epoch - } -} - -impl PerObjectBlobInfoApi for PerObjectBlobInfoV1 { - fn blob_id(&self) -> BlobId { - self.blob_id - } - - fn is_deletable(&self) -> bool { - self.deletable - } - - fn is_registered(&self, current_epoch: Epoch) -> bool { - self.end_epoch > current_epoch && !self.deleted - } - - fn is_deleted(&self) -> bool { - self.deleted - } - - fn storage_pool_id(&self) -> Option { - None - } -} - -impl ToBytes for PerObjectBlobInfoV1 {} - -impl Mergeable for PerObjectBlobInfoV1 { - type MergeOperand = PerObjectBlobInfoMergeOperand; - type Key = ObjectID; - - fn merge_with( - mut self, - PerObjectBlobInfoMergeOperand { - change_type, - change_info, - }: PerObjectBlobInfoMergeOperand, - ) -> Self { - assert_eq!( - self.blob_id, change_info.blob_id, - "blob ID mismatch in merge operand" - ); - assert_eq!( - self.deletable, change_info.deletable, - "deletable mismatch in merge operand" - ); - assert!( - !self.deleted, - "attempt to update an already deleted blob {}", - self.blob_id - ); - self.event = change_info.status_event; - match change_type { - // We ensure that the blob info is only updated a single time for each event. So if - // we see a duplicated registered or certified event for the some object, this is a - // serious bug somewhere. - BlobStatusChangeType::Register => { - panic!( - "cannot register an already registered blob {}", - self.blob_id - ); - } - BlobStatusChangeType::Certify => { - assert!( - self.certified_epoch.is_none(), - "cannot certify an already certified blob {}", - self.blob_id - ); - self.certified_epoch = Some(change_info.epoch); - } - BlobStatusChangeType::Extend => { - assert!( - self.certified_epoch.is_some(), - "cannot extend an uncertified blob {}", - self.blob_id - ); - self.end_epoch = change_info.end_epoch; - } - BlobStatusChangeType::Delete { was_certified } => { - assert_eq!(self.certified_epoch.is_some(), was_certified); - self.deleted = true; - } - } - self - } - - fn merge_new(operand: Self::MergeOperand) -> Option { - let PerObjectBlobInfoMergeOperand { - change_type: BlobStatusChangeType::Register, - change_info: - BlobStatusChangeInfo { - blob_id, - deletable, - epoch, - end_epoch, - status_event, - }, - } = operand - else { - tracing::error!( - ?operand, - "encountered an update other than 'register' for an untracked blob object" - ); - debug_assert!( - false, - "encountered an update other than 'register' for an untracked blob object: \ - {operand:?}" - ); - return None; - }; - Some(Self { - blob_id, - registered_epoch: epoch, - certified_epoch: None, - end_epoch, - deletable, - event: status_event, - deleted: false, - }) - } -} - #[cfg(test)] mod tests { use walrus_sui::test_utils::{event_id_for_testing, fixed_event_id_for_testing}; diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs index fe0f55c9cc..e907062c8b 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v2.rs @@ -4,7 +4,7 @@ //! V2 blob info types and merge logic, supporting both regular blobs and storage pool blobs. use serde::{Deserialize, Serialize}; -use sui_types::{base_types::ObjectID, event::EventID}; +use sui_types::event::EventID; use walrus_core::{BlobId, Epoch}; use walrus_storage_node_client::api::{BlobStatus, DeletableCounts}; @@ -18,7 +18,6 @@ use super::{ PermanentBlobInfo, ToBytes, blob_info_v1::{BlobInfoV1, ValidBlobInfoV1}, - per_object_blob_info::{PerObjectBlobInfoApi, PerObjectBlobInfoMergeOperand}, }; /// V2 aggregate blob info that supports both regular blobs and storage pool blobs. @@ -559,175 +558,15 @@ impl Mergeable for BlobInfoV2 { } } -// ============================================================================= -// Per-object blob info V2 -// ============================================================================= - -/// How the end epoch of a per-object blob is determined. -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(crate) enum EndEpochInfo { - /// Regular blob with an individual end epoch. - Individual(Epoch), - /// Pooled blob whose lifetime is determined by the storage pool's end epoch. - StoragePool(ObjectID), -} - -/// V2 per-object blob info that supports both regular blobs and storage pool blobs. -/// -/// Unlike V1, the end epoch and storage pool ID are combined into a single `EndEpochInfo` -/// enum: regular blobs have `EndEpochInfo::Individual(end_epoch)`, while pool blobs have -/// `EndEpochInfo::StoragePool(pool_id)`. -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] -pub(crate) struct PerObjectBlobInfoV2 { - /// The blob ID. - pub blob_id: BlobId, - /// The epoch in which the blob has been registered. - pub registered_epoch: Epoch, - /// The epoch in which the blob was first certified, `None` if the blob is uncertified. - pub certified_epoch: Option, - /// How the blob's end epoch is determined. - pub end_epoch_info: EndEpochInfo, - /// Whether the blob is deletable. - pub deletable: bool, - /// The ID of the last blob event related to this object. - pub event: EventID, - /// Whether the blob has been deleted. - pub deleted: bool, -} - -impl CertifiedBlobInfoApi for PerObjectBlobInfoV2 { - fn is_certified(&self, current_epoch: Epoch) -> bool { - self.is_registered(current_epoch) - && self - .certified_epoch - .is_some_and(|epoch| epoch <= current_epoch) - } - - fn initial_certified_epoch(&self) -> Option { - self.certified_epoch - } -} - -impl PerObjectBlobInfoApi for PerObjectBlobInfoV2 { - fn blob_id(&self) -> BlobId { - self.blob_id - } - - fn is_deletable(&self) -> bool { - self.deletable - } - - fn is_registered(&self, current_epoch: Epoch) -> bool { - if self.deleted { - return false; - } - match self.end_epoch_info { - EndEpochInfo::Individual(end_epoch) => end_epoch > current_epoch, - // Pool blob liveness depends on the pool, not the blob. - EndEpochInfo::StoragePool(_) => true, - } - } - - fn is_deleted(&self) -> bool { - self.deleted - } - - fn storage_pool_id(&self) -> Option { - match &self.end_epoch_info { - EndEpochInfo::StoragePool(id) => Some(*id), - EndEpochInfo::Individual(_) => None, - } - } -} - -impl ToBytes for PerObjectBlobInfoV2 {} - -impl Mergeable for PerObjectBlobInfoV2 { - type MergeOperand = PerObjectBlobInfoMergeOperand; - type Key = ObjectID; - - fn merge_with( - mut self, - PerObjectBlobInfoMergeOperand { - change_type, - change_info, - }: PerObjectBlobInfoMergeOperand, - ) -> Self { - assert_eq!( - self.blob_id, change_info.blob_id, - "blob ID mismatch in merge operand" - ); - assert_eq!( - self.deletable, change_info.deletable, - "deletable mismatch in merge operand" - ); - assert!( - !self.deleted, - "attempt to update an already deleted blob {}", - self.blob_id - ); - self.event = change_info.status_event; - match change_type { - // We ensure that the blob info is only updated a single time for each event. So if - // we see a duplicated registered or certified event for the some object, this is a - // serious bug somewhere. - BlobStatusChangeType::Register => { - panic!( - "cannot register an already registered blob {}", - self.blob_id - ); - } - BlobStatusChangeType::Certify => { - assert!( - self.certified_epoch.is_none(), - "cannot certify an already certified blob {}", - self.blob_id - ); - self.certified_epoch = Some(change_info.epoch); - } - BlobStatusChangeType::Extend => { - assert!( - self.certified_epoch.is_some(), - "cannot extend an uncertified blob {}", - self.blob_id - ); - self.end_epoch_info = EndEpochInfo::Individual(change_info.end_epoch); - } - BlobStatusChangeType::Delete { was_certified } => { - assert_eq!(self.certified_epoch.is_some(), was_certified); - self.deleted = true; - } - } - self - } - - fn merge_new(operand: Self::MergeOperand) -> Option { - // V2 entries are created via direct insert, not merge_new. - tracing::error!( - ?operand, - "PerObjectBlobInfoV2::merge_new should not be called; V2 entries are created via \ - direct insert" - ); - debug_assert!( - false, - "PerObjectBlobInfoV2::merge_new should not be called: {operand:?}" - ); - None - } -} - +// TODO(zhewu): revisit these tests to make sure that they are thorough and complete. #[cfg(test)] mod tests { use sui_types::base_types::ObjectID; use walrus_sui::test_utils::{event_id_for_testing, fixed_event_id_for_testing}; use walrus_test_utils::param_test; - use super::{EndEpochInfo, PerObjectBlobInfoV2, *}; - use crate::node::storage::blob_info::{ - BlobInfo, - PooledBlobChangeInfo, - per_object_blob_info::PerObjectBlobInfoMergeOperand, - }; + use super::*; + use crate::node::storage::blob_info::{BlobInfo, PooledBlobChangeInfo}; fn pool_id() -> ObjectID { walrus_sui::test_utils::object_id_for_testing() @@ -1340,51 +1179,4 @@ mod tests { fn test_v2_invariant_violations(info: ValidBlobInfoV2) { assert!(info.check_invariants().is_err()); } - - // --- PerObjectBlobInfoV2 --- - fn make_per_object_v2(end_epoch_info: EndEpochInfo) -> PerObjectBlobInfoV2 { - PerObjectBlobInfoV2 { - blob_id: walrus_core::test_utils::blob_id_from_u64(42), - registered_epoch: 1, - certified_epoch: None, - end_epoch_info, - deletable: true, - event: event_id_for_testing(), - deleted: false, - } - } - - #[test] - fn per_object_v2_certify_and_delete_merge() { - let info = make_per_object_v2(EndEpochInfo::StoragePool(pool_id())); - let cert_operand = PerObjectBlobInfoMergeOperand { - change_type: BlobStatusChangeType::Certify, - change_info: BlobStatusChangeInfo { - blob_id: walrus_core::test_utils::blob_id_from_u64(42), - deletable: true, - epoch: 3, - end_epoch: 0, - status_event: event_id_for_testing(), - }, - }; - let certified = info.merge_with(cert_operand); - assert_eq!(certified.certified_epoch, Some(3)); - assert!(!certified.deleted); - - let del_operand = PerObjectBlobInfoMergeOperand { - change_type: BlobStatusChangeType::Delete { - was_certified: true, - }, - change_info: BlobStatusChangeInfo { - blob_id: walrus_core::test_utils::blob_id_from_u64(42), - deletable: true, - epoch: 5, - end_epoch: 0, - status_event: event_id_for_testing(), - }, - }; - let deleted = certified.merge_with(del_operand); - assert!(deleted.deleted); - assert!(!deleted.is_registered(0)); - } } From 59d96e0117d6446efea0ac60f36ddc56dddb4c48 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Sun, 15 Mar 2026 21:39:52 -0700 Subject: [PATCH 7/7] address comments --- .../walrus-service/src/node/storage/blob_info.rs | 16 ++++++++++++++-- .../src/node/storage/blob_info/blob_info_v1.rs | 9 ++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/walrus-service/src/node/storage/blob_info.rs b/crates/walrus-service/src/node/storage/blob_info.rs index 32e8c350e5..f330698afb 100644 --- a/crates/walrus-service/src/node/storage/blob_info.rs +++ b/crates/walrus-service/src/node/storage/blob_info.rs @@ -1208,7 +1208,13 @@ impl Mergeable for BlobInfo { Self::V2(BlobInfoV2::from(v1).merge_with(operand)) } // Regular operands keep V1 as V1. - _ => Self::V1(v1.merge_with(operand)), + BlobInfoMergeOperand::MarkMetadataStored(_) + | BlobInfoMergeOperand::MarkInvalid { .. } + | BlobInfoMergeOperand::ChangeStatus { .. } + | BlobInfoMergeOperand::DeletableExpired { .. } + | BlobInfoMergeOperand::PermanentExpired { .. } => { + Self::V1(v1.merge_with(operand)) + } }, // V2 handles all operand types (regular AND pool). Self::V2(v2) => Self::V2(v2.merge_with(operand)), @@ -1226,7 +1232,13 @@ impl Mergeable for BlobInfo { // blob delete event. BlobInfoMergeOperand::PoolExpired { .. } => None, // First event is a regular event → create V1 (as before). - _ => BlobInfoV1::merge_new(operand).map(Self::V1), + BlobInfoMergeOperand::MarkMetadataStored(_) + | BlobInfoMergeOperand::MarkInvalid { .. } + | BlobInfoMergeOperand::ChangeStatus { .. } + | BlobInfoMergeOperand::DeletableExpired { .. } + | BlobInfoMergeOperand::PermanentExpired { .. } => { + BlobInfoV1::merge_new(operand).map(Self::V1) + } } } } diff --git a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs index 3a5129ad29..8db847b8f3 100644 --- a/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs +++ b/crates/walrus-service/src/node/storage/blob_info/blob_info_v1.rs @@ -575,14 +575,9 @@ impl Mergeable for BlobInfoV1 { ); None } - BlobInfoMergeOperand::PoolExpired { .. } => { - // if the first event is a pool expired before the blob is registered, there may - // be a race between GC and pool event processing. Don't need to create a blob info - // for this case. - None - } // Pool operands should never reach V1 — they are intercepted at the BlobInfo level. - BlobInfoMergeOperand::PooledBlobChangeStatus { .. } => { + BlobInfoMergeOperand::PooledBlobChangeStatus { .. } + | BlobInfoMergeOperand::PoolExpired { .. } => { unreachable!("pool operands should be handled in BlobInfoV2") } }