feat: delta-encode BasicBlockNode metadata (2/3)

CHANGELOG.md

@@ -27,6 +27,7 @@
 - [BREAKING] Change backend from winterfell to Plonky3 ([#2472](https://github.com/0xMiden/miden-vm/pull/2472)).
 - Added validation of `core_trace_fragment_size` in `ExecutionOptions` ([#2528](https://github.com/0xMiden/miden-vm/pull/2528)).
 - [BREAKING] Change serialization of `BasicBlockNode`s to use padded indices ([#2466](https://github.com/0xMiden/miden-vm/pull/2466/)).
+- Change padded serialization of `BasicBlockNode`s to use delta-encoded metadata ([#2469](https://github.com/0xMiden/miden-vm/pull/2469/)).
 
 ## 0.20.2 (TBD)
 - Fix issue where decorator access was not bypassed properly in release mode ([#2529](https://github.com/0xMiden/miden-vm/pull/2529)).

core/Cargo.toml

@@ -66,5 +66,6 @@ criterion = { workspace = true }
 insta.workspace = true
 miden-test-serde-macros.workspace = true
 proptest.workspace = true
+rstest = { version = "0.26" }
 serde_json = { workspace = true }
 miden-utils-testing.workspace = true

core/src/mast/node/basic_block_node/mod.rs

@@ -526,7 +526,7 @@ impl BasicBlockNode {
             let indptr = batch.indptr();
             let ops = batch.ops();
 
-            // indptr should be monotonic non-decreasing
+            // indptr should be monotonic non-decreasing in valid prefix
             for i in 0..batch.num_groups() {
                 if indptr[i] > indptr[i + 1] {
                     return Err(format!(
@@ -540,8 +540,32 @@ impl BasicBlockNode {
                 }
             }
 
+            // Full array must be monotonic for serialization (delta encoding)
+            for i in 0..indptr.len() - 1 {
+                if indptr[i] > indptr[i + 1] {
+                    return Err(format!(
+                        "Batch {}: indptr[{}] {} > indptr[{}] {} - full array not monotonic (required for serialization)",
+                        batch_idx,
+                        i,
+                        indptr[i],
+                        i + 1,
+                        indptr[i + 1]
+                    ));
+                }
+            }
+
             // All indptr values should be within ops bounds
             let ops_len = ops.len();
+
+            // Final indptr value must equal ops.len()
+            if indptr[indptr.len() - 1] != ops_len {
+                return Err(format!(
+                    "Batch {}: final indptr value {} doesn't match ops.len() {}",
+                    batch_idx,
+                    indptr[indptr.len() - 1],
+                    ops_len
+                ));
+            }
             for (i, &indptr_val) in indptr.iter().enumerate().take(batch.num_groups() + 1) {
                 if indptr_val > ops_len {
                     return Err(format!(

core/src/mast/node/basic_block_node/op_batch.rs

@@ -17,22 +17,20 @@ use super::{BATCH_SIZE, Felt, GROUP_SIZE, Operation, ZERO};
 pub struct OpBatch {
     /// A list of operations in this batch, including padding noops.
     pub(super) ops: Vec<Operation>,
-    /// An array of indexes in the ops array, marking the beginning and end of each group.
+    /// Indexes marking the start and end of each group in the ops array.
     ///
-    /// The array maintains the invariant that the i-th group (i <= BATCH_SIZE-1) is at
-    /// `self.ops[self.indptr[i]..self.indptr[i+1]]`.
+    /// Group i is at `self.ops[self.indptr[i]..self.indptr[i+1]]`. Groups with immediate values
+    /// have zero-length slices.
     ///
-    /// By convention, the groups containing immediate values have a zero-length slice of the ops
-    /// array.
+    /// Only `[0..num_groups+1]` is valid data. The tail is undefined but must be monotonic
+    /// (filled with final ops count) for delta encoding during serialization.
     pub(super) indptr: [usize; Self::BATCH_SIZE_PLUS_ONE],
-    /// An array of bits representing whether a group had undergone padding
+    /// Whether each group had padding added. Only `[0..num_groups]` is valid.
     pub(super) padding: [bool; BATCH_SIZE],
-    /// Value of groups in the batch, which includes operations and immediate values.
+    /// Group hashes and immediate values. Only `[0..num_groups]` is valid.
     pub(super) groups: [Felt; BATCH_SIZE],
-    /// Number of groups in this batch.
-    ///
-    /// The arrays above are meaningful in their [0..self.num_groups] prefix
-    /// (or [0..self.num_groups + 1] in the case of the indptr array).
+    /// Number of groups. Determines valid prefix sizes: indptr `[0..num_groups+1]`, padding and
+    /// groups `[0..num_groups]`.
     pub(super) num_groups: usize,
 }
 
@@ -134,7 +132,62 @@ impl OpBatch {
         groups: [Felt; BATCH_SIZE],
         num_groups: usize,
     ) -> Self {
-        Self { ops, indptr, padding, groups, num_groups }
+        let batch = Self { ops, indptr, padding, groups, num_groups };
+        #[cfg(debug_assertions)]
+        batch.validate_invariants();
+        batch
+    }
+
+    /// Validates invariants in debug builds: num_groups in range, indptr monotonic (full array),
+    /// final indptr equals ops.len().
+    #[cfg(debug_assertions)]
+    fn validate_invariants(&self) {
+        // Validate num_groups is in valid range
+        assert!(
+            self.num_groups <= BATCH_SIZE,
+            "num_groups {} exceeds BATCH_SIZE {}",
+            self.num_groups,
+            BATCH_SIZE
+        );
+
+        // Validate indptr starts at 0
+        assert_eq!(self.indptr[0], 0, "indptr must start at 0, got {}", self.indptr[0]);
+
+        // Validate monotonicity in the semantically valid prefix [0..num_groups+1]
+        for i in 0..self.num_groups {
+            assert!(
+                self.indptr[i] <= self.indptr[i + 1],
+                "indptr not monotonic in valid prefix: indptr[{}]={} > indptr[{}]={}",
+                i,
+                self.indptr[i],
+                i + 1,
+                self.indptr[i + 1]
+            );
+        }
+
+        // Validate monotonicity across ENTIRE array (required for serialization)
+        for i in 0..Self::BATCH_SIZE_PLUS_ONE - 1 {
+            assert!(
+                self.indptr[i] <= self.indptr[i + 1],
+                "indptr not monotonic at index {}: indptr[{}]={} > indptr[{}]={} \
+                 (full array monotonicity required for delta encoding)",
+                i,
+                i,
+                self.indptr[i],
+                i + 1,
+                self.indptr[i + 1]
+            );
+        }
+
+        // Validate final indptr value matches ops length
+        let final_indptr = self.indptr[Self::BATCH_SIZE_PLUS_ONE - 1];
+        assert_eq!(
+            final_indptr,
+            self.ops.len(),
+            "final indptr value {} doesn't match ops.len() {}",
+            final_indptr,
+            self.ops.len()
+        );
     }
 
     /// Returns the (op_group_idx, op_idx_in_group) given an operation index in the batch. Returns
@@ -327,13 +380,26 @@ impl OpBatchAccumulator {
         self.pad_if_needed();
         self.finalize_indptr();
 
-        OpBatch {
+        // Fill the unused tail of indptr array with the final value to maintain monotonicity
+        // This is required for delta encoding which expects indptr to be monotonically
+        // non-decreasing
+        let final_ops_count = self.ops.len();
+        for i in self.next_group_idx..OpBatch::BATCH_SIZE_PLUS_ONE {
+            self.indptr[i] = final_ops_count;
+        }
+
+        let batch = OpBatch {
             ops: self.ops,
             indptr: self.indptr,
             padding: self.padding,
             groups: self.groups,
             num_groups: self.next_group_idx,
-        }
+        };
+
+        #[cfg(debug_assertions)]
+        batch.validate_invariants();
+
+        batch
     }
 
     // HELPER METHODS

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_1.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 16
 expression: batches
 ---
 [
@@ -10,13 +11,13 @@ expression: batches
         indptr: [
             0,
             1,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
+            1,
         ],
         padding: [
             false,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_2.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 30
 expression: batches
 ---
 [
@@ -11,13 +12,13 @@ expression: batches
         indptr: [
             0,
             2,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
+            2,
+            2,
+            2,
+            2,
+            2,
+            2,
+            2,
         ],
         padding: [
             false,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_3.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 44
 expression: batches
 ---
 [
@@ -15,12 +16,12 @@ expression: batches
             0,
             3,
             3,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
+            3,
+            3,
+            3,
+            3,
+            3,
+            3,
         ],
         padding: [
             true,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_5.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 101
 expression: batches
 ---
 [
@@ -72,12 +73,12 @@ expression: batches
             0,
             2,
             2,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
+            2,
+            2,
+            2,
+            2,
+            2,
+            2,
         ],
         padding: [
             true,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_6.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 139
 expression: batches
 ---
 [
@@ -26,10 +27,10 @@ expression: batches
             9,
             9,
             10,
-            0,
-            0,
-            0,
-            0,
+            10,
+            10,
+            10,
+            10,
         ],
         padding: [
             false,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_7.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 171
 expression: batches
 ---
 [
@@ -25,10 +26,10 @@ expression: batches
             10,
             10,
             11,
-            0,
-            0,
-            0,
-            0,
+            11,
+            11,
+            11,
+            11,
         ],
         padding: [
             false,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_8.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 203
 expression: batches
 ---
 [
@@ -27,10 +28,10 @@ expression: batches
             9,
             11,
             11,
-            0,
-            0,
-            0,
-            0,
+            11,
+            11,
+            11,
+            11,
         ],
         padding: [
             true,

core/src/mast/node/basic_block_node/snapshots/miden_core__mast__node__basic_block_node__tests__batch_ops_9.snap

@@ -1,5 +1,6 @@
 ---
 source: core/src/mast/node/basic_block_node/tests.rs
+assertion_line: 246
 expression: batches
 ---
 [
@@ -78,12 +79,12 @@ expression: batches
             0,
             2,
             2,
-            0,
-            0,
-            0,
-            0,
-            0,
-            0,
+            2,
+            2,
+            2,
+            2,
+            2,
+            2,
         ],
         padding: [
             false,