diff --git a/Cargo.lock b/Cargo.lock index aa0fb49b9..b8c7ebf9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1263,6 +1263,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "blstrs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a8a8ed6fefbeef4a8c7b460e4110e12c5e22a5b7cf32621aae6ad650c4dcf29" +dependencies = [ + "blst", + "byte-slice-cast", + "ff", + "group", + "pairing", + "rand_core 0.6.4", + "serde", + "subtle", +] + [[package]] name = "borsh" version = "1.6.0" @@ -1648,6 +1664,53 @@ dependencies = [ "libc", ] +[[package]] +name = "crate_crypto_internal_eth_kzg_bls12_381" +version = "0.5.4" +source = "git+https://github.com/crate-crypto/rust-eth-kzg.git?tag=v0.5.4#ca7a9e4002c1328abf80ba66838daefaa825dd89" +dependencies = [ + "blst", + "blstrs", + "ff", + "group", + "pairing", + "subtle", +] + +[[package]] +name = "crate_crypto_internal_eth_kzg_erasure_codes" +version = "0.5.4" +source = "git+https://github.com/crate-crypto/rust-eth-kzg.git?tag=v0.5.4#ca7a9e4002c1328abf80ba66838daefaa825dd89" +dependencies = [ + "crate_crypto_internal_eth_kzg_bls12_381", + "crate_crypto_internal_eth_kzg_polynomial", +] + +[[package]] +name = "crate_crypto_internal_eth_kzg_maybe_rayon" +version = "0.5.4" +source = "git+https://github.com/crate-crypto/rust-eth-kzg.git?tag=v0.5.4#ca7a9e4002c1328abf80ba66838daefaa825dd89" + +[[package]] +name = "crate_crypto_internal_eth_kzg_polynomial" +version = "0.5.4" +source = "git+https://github.com/crate-crypto/rust-eth-kzg.git?tag=v0.5.4#ca7a9e4002c1328abf80ba66838daefaa825dd89" +dependencies = [ + "crate_crypto_internal_eth_kzg_bls12_381", +] + +[[package]] +name = "crate_crypto_kzg_multi_open_fk20" +version = "0.5.4" +source = "git+https://github.com/crate-crypto/rust-eth-kzg.git?tag=v0.5.4#ca7a9e4002c1328abf80ba66838daefaa825dd89" +dependencies = [ + "crate_crypto_internal_eth_kzg_bls12_381", + "crate_crypto_internal_eth_kzg_maybe_rayon", + "crate_crypto_internal_eth_kzg_polynomial", + "hex", + "sha2", +] + [[package]] name = "crc" version = "3.4.0" @@ -2778,7 +2841,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", + "rand 0.8.5", "rand_core 0.6.4", + "rand_xorshift 0.3.0", "subtle", ] @@ -5453,7 +5518,7 @@ dependencies = [ "num-traits", "rand 0.9.2", "rand_chacha 0.9.0", - "rand_xorshift", + "rand_xorshift 0.4.0", "regex-syntax", "rusty-fork", "tempfile", @@ -5655,6 +5720,15 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -5918,6 +5992,7 @@ dependencies = [ "alloy-rlp", "anyhow", "async-trait", + "discv5", "ethereum_hashing 0.7.0", "ethereum_serde_utils", "ethereum_ssz", @@ -5929,6 +6004,7 @@ dependencies = [ "ream-execution-rpc-types", "ream-merkle", "ream-network-spec", + "rust_eth_kzg", "serde", "serde_json", "serde_yaml", @@ -6891,6 +6967,19 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust_eth_kzg" +version = "0.5.4" +source = "git+https://github.com/crate-crypto/rust-eth-kzg.git?tag=v0.5.4#ca7a9e4002c1328abf80ba66838daefaa825dd89" +dependencies = [ + "crate_crypto_internal_eth_kzg_bls12_381", + "crate_crypto_internal_eth_kzg_erasure_codes", + "crate_crypto_kzg_multi_open_fk20", + "hex", + "serde", + "serde_json", +] + [[package]] name = "rustc-hash" version = "2.1.1" diff --git a/Cargo.toml b/Cargo.toml index 8d099a65c..908d568a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ redb = "3.1.0" reqwest = { version = "0.12.24", features = ["native-tls-vendored", "json"] } rstest = "0.26.1" rust-kzg-blst = { git = 'https://github.com/grandinetech/rust-kzg.git' } +rust_eth_kzg = { git = "https://github.com/crate-crypto/rust-eth-kzg.git", tag = "v0.5.4" } serde = { version = '1.0', features = ['derive', "rc"] } serde_json = "1.0.145" serde_yaml = "0.9" diff --git a/crates/common/consensus/beacon/Cargo.toml b/crates/common/consensus/beacon/Cargo.toml index a2d18c00d..4dd8a4034 100644 --- a/crates/common/consensus/beacon/Cargo.toml +++ b/crates/common/consensus/beacon/Cargo.toml @@ -19,11 +19,13 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true anyhow.workspace = true async-trait.workspace = true +discv5.workspace = true ethereum_hashing.workspace = true ethereum_serde_utils.workspace = true ethereum_ssz.workspace = true ethereum_ssz_derive.workspace = true itertools.workspace = true +rust_eth_kzg.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true diff --git a/crates/common/consensus/beacon/src/custody_group.rs b/crates/common/consensus/beacon/src/custody_group.rs new file mode 100644 index 000000000..d08334fe0 --- /dev/null +++ b/crates/common/consensus/beacon/src/custody_group.rs @@ -0,0 +1,59 @@ +use anyhow::{Ok, Result, ensure}; +use discv5::enr::NodeId; +use ream_consensus_misc::constants::beacon::NUM_CUSTODY_GROUPS; +use sha2::{Digest, Sha256}; + +use crate::data_column_sidecar::NUMBER_OF_COLUMNS; + +pub fn get_custody_group_indices(node_id: NodeId, custody_group_count: u64) -> Result> { + ensure!( + custody_group_count <= NUM_CUSTODY_GROUPS, + "Custody group count more than number of custody groups" + ); + + if custody_group_count == NUM_CUSTODY_GROUPS { + return Ok((0..NUM_CUSTODY_GROUPS).collect()); + } + + let mut custody_indices = Vec::new(); + let mut current_id = node_id.raw(); + + while custody_indices.len() < custody_group_count as usize { + let hash = Sha256::digest(current_id); + + let mut array = [0u8; 8]; + array.copy_from_slice(&hash[0..8]); + let index = u64::from_le_bytes(array) % NUM_CUSTODY_GROUPS; + + if !custody_indices.contains(&index) { + custody_indices.push(index); + } + + let mut carry = true; + for byte in current_id.iter_mut().rev() { + if carry { + let (new_byte, overflow) = byte.overflowing_add(1); + *byte = new_byte; + carry = overflow; + } + } + } + custody_indices.sort(); + Ok(custody_indices) +} + +pub fn compute_columns_for_custody_group(custody_group_index: u64) -> Result> { + ensure!( + custody_group_index < NUM_CUSTODY_GROUPS, + "Custody group index is greater than total custody groups" + ); + + let mut column_indices = Vec::new(); + for column in 0..NUMBER_OF_COLUMNS { + if column % NUM_CUSTODY_GROUPS == custody_group_index { + column_indices.push(column); + } + } + + Ok(column_indices) +} diff --git a/crates/common/consensus/beacon/src/data_column_sidecar.rs b/crates/common/consensus/beacon/src/data_column_sidecar.rs index 7d82ba2ee..4a8aed5bd 100644 --- a/crates/common/consensus/beacon/src/data_column_sidecar.rs +++ b/crates/common/consensus/beacon/src/data_column_sidecar.rs @@ -7,6 +7,7 @@ use ream_consensus_misc::{ use ream_merkle::is_valid_merkle_branch; use ream_network_spec::networks::beacon_network_spec; use serde::{Deserialize, Serialize}; +use ssz::DecodeError; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList, typenum}; use tree_hash::TreeHash; @@ -102,6 +103,51 @@ impl DataColumnSidecar { } } +pub fn get_column_data_sidecars( + signed_block_header: SignedBeaconBlockHeader, + kzg_commitments: VariableList, + kzg_commitments_inclusion_proof: FixedVector, + cells_and_kzg_proofs: Vec<(Vec, Vec)>, +) -> Result, DecodeError> { + if cells_and_kzg_proofs.len() != kzg_commitments.len() { + return Err(DecodeError::InvalidByteLength { + len: kzg_commitments.len(), + expected: cells_and_kzg_proofs.len(), + }); + } + + let mut sidecars = Vec::new(); + for column_index in 0..NUMBER_OF_COLUMNS { + let mut column_cells = Vec::new(); + let mut column_proofs = Vec::new(); + for (cells, proofs) in &cells_and_kzg_proofs { + if column_index as usize >= cells.len() || column_index as usize >= proofs.len() { + return Err(DecodeError::OffsetOutOfBounds(column_index as usize)); + } + column_cells.push(cells[column_index as usize].clone()); + column_proofs.push(proofs[column_index as usize]); + } + + sidecars.push(DataColumnSidecar { + index: column_index, + column: VariableList::try_from(column_cells).map_err(|err| { + DecodeError::BytesInvalid(format!( + "Creating column VariableList for index {column_index} failed: {err}", + )) + })?, + kzg_commitments: kzg_commitments.clone(), + kzg_proofs: VariableList::try_from(column_proofs).map_err(|err| { + DecodeError::BytesInvalid(format!( + "Creating proofs VariableList for index {column_index} failed: {err}", + )) + })?, + signed_block_header: signed_block_header.clone(), + kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), + }); + } + Ok(sidecars) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/common/consensus/beacon/src/lib.rs b/crates/common/consensus/beacon/src/lib.rs index d0b386ac0..fad1fc62c 100644 --- a/crates/common/consensus/beacon/src/lib.rs +++ b/crates/common/consensus/beacon/src/lib.rs @@ -5,6 +5,7 @@ pub mod attester_slashing; pub mod beacon_committee_selection; pub mod blob_sidecar; pub mod bls_to_execution_change; +pub mod custody_group; pub mod data_column_sidecar; pub mod electra; pub mod eth_1_block; @@ -12,6 +13,7 @@ pub mod fork_choice; pub mod genesis; pub mod helpers; pub mod historical_summary; +pub mod matrix_entry; pub mod pending_consolidation; pub mod pending_deposit; pub mod pending_partial_withdrawal; diff --git a/crates/common/consensus/beacon/src/matrix_entry.rs b/crates/common/consensus/beacon/src/matrix_entry.rs new file mode 100644 index 000000000..ccf44d3f4 --- /dev/null +++ b/crates/common/consensus/beacon/src/matrix_entry.rs @@ -0,0 +1,134 @@ +use anyhow::{Ok, Result, anyhow, ensure}; +use ream_consensus_misc::polynomial_commitments::kzg_proof::KZGProof; +use ream_execution_rpc_types::get_blobs::Blob; +use rust_eth_kzg::{Cell as KZGCell, DASContext, KZGProof as Proof}; +use ssz_types::FixedVector; + +use crate::data_column_sidecar::Cell; + +#[derive(Debug, Clone)] +pub struct MatrixEntry { + cell: Cell, + #[allow(dead_code)] + kzg_proof: KZGProof, + column_index: u64, + row_index: u64, +} + +pub fn compute_matrix(blobs: Vec, das_context: &DASContext) -> Result> { + let mut matrix = Vec::new(); + + for (blob_index, blob) in blobs.iter().enumerate() { + let (cells, proofs) = compute_cells_and_kzg_proofs(blob, das_context)?; + for (cell_index, (cell, kzg_proof)) in cells.into_iter().zip(proofs.into_iter()).enumerate() + { + matrix.push(MatrixEntry { + cell, + kzg_proof, + column_index: cell_index as u64, + row_index: blob_index as u64, + }); + } + } + + Ok(matrix) +} + +pub fn recover_matrix( + partial_matrix: Vec, + blob_count: u64, + das_context: &DASContext, +) -> Result> { + let mut matrix = Vec::new(); + + for blob_index in 0..blob_count { + let (cell_indices, cells): (Vec, Vec) = partial_matrix + .iter() + .filter(|entry| entry.row_index == blob_index) + .map(|entry| (entry.column_index, entry.cell.clone())) + .unzip(); + + let (recovered_cells, recovered_proofs) = + recover_cells_and_kzg_proofs(cell_indices, cells, das_context)?; + + for (cell_index, (cell, kzg_proof)) in recovered_cells + .into_iter() + .zip(recovered_proofs.into_iter()) + .enumerate() + { + matrix.push(MatrixEntry { + cell, + kzg_proof, + column_index: blob_index, + row_index: cell_index as u64, + }); + } + } + + Ok(matrix) +} + +fn compute_cells_and_kzg_proofs( + blob: &Blob, + das_context: &DASContext, +) -> Result<(Vec, Vec)> { + let blob_data: Vec = blob.inner.clone().into(); + ensure!( + blob_data.len() == 131072, + "Blob inner length {}, expected 131072", + blob_data.len() + ); + let blob_bytes: &[u8; 131072] = blob_data + .as_slice() + .try_into() + .map_err(|err| anyhow!("Failed to convert blob inner to &[u8; 131072]: {err}"))?; + let (kzg_cells, kzg_proofs) = das_context + .compute_cells_and_kzg_proofs(blob_bytes) + .map_err(|err| anyhow!("KZG error: {err:?}"))?; + + let cells = kzg_cells.into_iter().map(convert_cell).collect(); + let proofs = kzg_proofs.into_iter().map(convert_kzg_proof).collect(); + + Ok((cells, proofs)) +} + +fn recover_cells_and_kzg_proofs( + cell_indices: Vec, + cells: Vec, + das_context: &DASContext, +) -> Result<(Vec, Vec)> { + let kzg_cells_result: Result> = cells + .into_iter() + .map(|cell_fixed_vector| { + let cell_vec: Vec = cell_fixed_vector.into(); + ensure!( + cell_vec.len() == 2048, + "Cell length {}, expected 2048", + cell_vec.len() + ); + let cell_array: [u8; 2048] = cell_vec + .try_into() + .map_err(|err| anyhow!("Failed to convert Cell to [u8; 2048]: {err:?}"))?; + Ok(cell_array) + }) + .collect(); + + let kzg_cells = kzg_cells_result?; + let kzg_cells_refs = kzg_cells.iter().collect(); + let (new_kzg_cells, new_kzg_proofs) = das_context + .recover_cells_and_kzg_proofs(cell_indices, kzg_cells_refs) + .map_err(|err| anyhow!("KZG recovery error: {err:?}"))?; + + let cells = new_kzg_cells.into_iter().map(convert_cell).collect(); + let proofs = new_kzg_proofs.into_iter().map(convert_kzg_proof).collect(); + + Ok((cells, proofs)) +} + +fn convert_cell(kzg_cell: KZGCell) -> Cell { + FixedVector::try_from(kzg_cell.to_vec()).expect("Cell conversion failed") +} + +fn convert_kzg_proof(kzg_proof: Proof) -> KZGProof { + KZGProof::from(kzg_proof) +} diff --git a/crates/common/consensus/misc/src/constants/beacon.rs b/crates/common/consensus/misc/src/constants/beacon.rs index 0e1bdd845..7c7245bb4 100644 --- a/crates/common/consensus/misc/src/constants/beacon.rs +++ b/crates/common/consensus/misc/src/constants/beacon.rs @@ -7,6 +7,7 @@ pub const BASE_REWARDS_PER_EPOCH: u64 = 4; pub const BASE_REWARD_FACTOR: u64 = 64; pub const BEACON_STATE_MERKLE_DEPTH: u64 = 6; pub const BLOB_KZG_COMMITMENTS_INDEX: u64 = 11; +pub const BLOB_SIDECAR_KZG_PROOF_DEPTH: u64 = 17; pub const BLOCK_BODY_MERKLE_DEPTH: u64 = 4; pub const BYTES_PER_BLOB: usize = BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB; pub const BYTES_PER_COMMITMENT: usize = 48; @@ -15,6 +16,8 @@ pub const BYTES_PER_PROOF: usize = 48; pub const CAPELLA_FORK_VERSION: B32 = fixed_bytes!("0x03000000"); pub const CHURN_LIMIT_QUOTIENT: u64 = 65536; pub const CURRENT_SYNC_COMMITTEE_INDEX: u64 = 22; +pub const CUSTODY_REQUIREMENT: u64 = 4; +pub const DATA_COLUMN_SIDECAR_KZG_PROOF_DEPTH: u64 = 4; pub const DEPOSIT_CONTRACT_TREE_DEPTH: u64 = 32; pub const DOMAIN_AGGREGATE_AND_PROOF: B32 = fixed_bytes!("0x06000000"); pub const DOMAIN_BEACON_ATTESTER: B32 = fixed_bytes!("0x01000000"); @@ -46,8 +49,6 @@ pub const INTERVALS_PER_SLOT: u64 = 3; pub const INACTIVITY_SCORE_BIAS: u64 = 4; pub const INACTIVITY_SCORE_RECOVERY_RATE: u64 = 16; pub const JUSTIFICATION_BITS_LENGTH: usize = 4; -pub const BLOB_SIDECAR_KZG_PROOF_DEPTH: u64 = 17; -pub const DATA_COLUMN_SIDECAR_KZG_PROOF_DEPTH: u64 = 4; pub const KZG_COMMITMENTS_MERKLE_DEPTH: u64 = 12; pub const MAX_BLOBS_PER_BLOCK: usize = 4096; pub const MAX_COMMITTEES_PER_SLOT: u64 = 64; @@ -66,12 +67,14 @@ pub const MIN_PER_EPOCH_CHURN_LIMIT: u64 = 4; pub const MIN_SEED_LOOKAHEAD: u64 = 1; pub const MIN_VALIDATOR_WITHDRAWABILITY_DELAY: u64 = 256; pub const NEXT_SYNC_COMMITTEE_INDEX: u64 = 23; +pub const NUM_CUSTODY_GROUPS: u64 = 128; pub const NUM_FLAG_INDICES: usize = 3; pub const PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: u64 = 3; pub const PROPOSER_REWARD_QUOTIENT: u64 = 8; pub const PROPOSER_WEIGHT: u64 = 8; pub const REORG_PARENT_WEIGHT_THRESHOLD: u64 = 160; pub const SAFETY_DECAY: u64 = 10; +pub const SAMPLES_PER_SLOT: u64 = 8; pub const SECONDS_PER_ETH1_BLOCK: u64 = 14; pub const SHARD_COMMITTEE_PERIOD: u64 = 256; pub const SHUFFLE_ROUND_COUNT: u8 = 90;