Merge pull request #830 from sethdusek/headerserialization6.0

sethdusek · web-flow · commit 4d25b1abdefa · 2025-05-27T01:15:04.000+05:00
Add Header serialization
diff --git a/ergo-chain-generation/src/chain_generation.rs b/ergo-chain-generation/src/chain_generation.rs
@@ -191,6 +191,7 @@ fn prove_block(
         extension_root,
         autolykos_solution: dummy_autolykos_solution,
         votes,
+        unparsed_bytes: Box::new([]),
     };
     let msg = blake2b256_hash(&header.serialize_without_pow().unwrap())
         .0
diff --git a/ergo-chain-generation/src/fake_pow_scheme.rs b/ergo-chain-generation/src/fake_pow_scheme.rs
@@ -141,6 +141,7 @@ mod tests {
             extension_root,
             autolykos_solution: dummy_autolykos_solution,
             votes,
+            unparsed_bytes: Box::new([]),
         };
 
         let x = DlogProverInput::random();
diff --git a/ergo-chain-types/src/header.rs b/ergo-chain-types/src/header.rs
@@ -56,6 +56,17 @@ pub struct Header {
     /// 3 bytes in accordance to Scala implementation, but will use `Vec` until further improvements
     #[cfg_attr(feature = "json", serde(rename = "votes"))]
     pub votes: Votes,
+    /// Unparsed bytes that encode new possible fields
+    #[cfg_attr(
+        feature = "json",
+        serde(
+            rename = "unparsedBytes",
+            default,
+            serialize_with = "crate::json::autolykos_solution::as_base16_string",
+            deserialize_with = "crate::json::autolykos_solution::from_base16_string"
+        )
+    )]
+    pub unparsed_bytes: Box<[u8]>,
 }
 
 impl Header {
@@ -82,7 +93,8 @@ impl Header {
         // For block version >= 2, this new byte encodes length of possible new fields.
         // Set to 0 for now, so no new fields.
         if self.version > 1 {
-            w.put_i8(0)?;
+            w.put_u8(self.unparsed_bytes.len().try_into()?)?;
+            w.write_all(&self.unparsed_bytes)?;
         }
         Ok(data)
     }
@@ -127,13 +139,18 @@ impl ScorexSerializable for Header {
 
         // For block version >= 2, a new byte encodes length of possible new fields.  If this byte >
         // 0, we read new fields but do nothing, as semantics of the fields is not known.
-        if version > 1 {
+        let unparsed_bytes: Box<[u8]> = if version > 1 {
             let new_field_size = r.get_u8()?;
             if new_field_size > 0 {
                 let mut field_bytes: Vec<u8> = vec![0; new_field_size as usize];
                 r.read_exact(&mut field_bytes)?;
+                field_bytes.into()
+            } else {
+                Box::new([])
             }
-        }
+        } else {
+            Box::new([])
+        };
 
         // Parse `AutolykosSolution`
         let autolykos_solution = if version == 1 {
@@ -182,6 +199,7 @@ impl ScorexSerializable for Header {
             extension_root,
             autolykos_solution: autolykos_solution.clone(),
             votes,
+            unparsed_bytes,
         };
 
         let mut id_bytes = header.serialize_without_pow()?;
@@ -296,6 +314,7 @@ mod arbitrary {
                 prop::sample::select(vec![1_u8, 2]),
                 any::<Box<AutolykosSolution>>(),
                 uniform3(1u8..),
+                proptest::collection::vec(any::<u8>(), 0..=255),
             )
                 .prop_map(
                     |(
@@ -309,6 +328,7 @@ mod arbitrary {
                         version,
                         autolykos_solution,
                         votes,
+                        unparsed_bytes,
                     )| {
                         let parent_id = BlockId(Digest(parent_id));
                         let ad_proofs_root = Digest(ad_proofs_root);
@@ -332,6 +352,11 @@ mod arbitrary {
                             extension_root,
                             autolykos_solution: *autolykos_solution.clone(),
                             votes,
+                            unparsed_bytes: if version > 1 {
+                                unparsed_bytes.into()
+                            } else {
+                                Box::new([])
+                            },
                         };
                         let mut id_bytes = header.serialize_without_pow().unwrap();
                         let mut data = Vec::new();
diff --git a/ergo-chain-types/src/json/autolykos_solution.rs b/ergo-chain-types/src/json/autolykos_solution.rs
@@ -16,13 +16,14 @@ where
     serializer.serialize_str(&base16::encode_lower(value))
 }
 
-pub(crate) fn from_base16_string<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+pub(crate) fn from_base16_string<'de, D, T: From<Vec<u8>>>(deserializer: D) -> Result<T, D::Error>
 where
     D: Deserializer<'de>,
 {
     use serde::de::Error;
     String::deserialize(deserializer)
         .and_then(|string| base16::decode(&string).map_err(|err| Error::custom(err.to_string())))
+        .map(From::from)
 }
 
 /// Serialize `BigInt` as a string
diff --git a/ergotree-interpreter/src/eval/sglobal.rs b/ergotree-interpreter/src/eval/sglobal.rs
@@ -250,7 +250,7 @@ pub(crate) static DECODE_NBITS_EVAL_FN: EvalFn = |_mc, _env, _ctx, _obj, args| {
 #[cfg(test)]
 #[cfg(feature = "arbitrary")]
 mod tests {
-    use ergo_chain_types::EcPoint;
+    use ergo_chain_types::{EcPoint, Header};
     use ergotree_ir::bigint256::BigInt256;
     use ergotree_ir::ergo_tree::ErgoTreeVersion;
     use ergotree_ir::mir::constant::Constant;
@@ -682,5 +682,9 @@ mod tests {
         fn serialize_unsigned_bigint(v in any::<UnsignedBigInt>()) {
             assert_eq!(deserialize(&serialize(v), SType::SUnsignedBigInt), Constant::from(v));
         }
+        #[test]
+        fn serialize_header(h in any::<Header>()) {
+            assert_eq!(deserialize(&serialize(h.clone()), SType::SHeader), Constant::from(h));
+        }
     }
 }
diff --git a/ergotree-ir/src/mir/constant.rs b/ergotree-ir/src/mir/constant.rs
@@ -31,6 +31,7 @@ use ergo_chain_types::ADDigest;
 use ergo_chain_types::Base16DecodedBytes;
 use ergo_chain_types::Digest32;
 use ergo_chain_types::EcPoint;
+use ergo_chain_types::Header;
 use impl_trait_for_tuples::impl_for_tuples;
 use sigma_util::AsVecI8;
 use sigma_util::AsVecU8;
@@ -83,6 +84,8 @@ pub enum Literal {
     GroupElement(Arc<EcPoint>),
     /// AVL tree
     AvlTree(Box<AvlTreeData>),
+    /// Block Header type
+    Header(Box<Header>),
     /// Ergo box
     CBox(Ref<'static, ErgoBox>),
     /// Collection
@@ -165,6 +168,7 @@ impl core::fmt::Debug for Literal {
             Literal::GroupElement(v) => v.fmt(f),
             Literal::UnsignedBigInt(v) => v.fmt(f),
             Literal::AvlTree(v) => v.fmt(f),
+            Literal::Header(v) => v.fmt(f),
             Literal::CBox(v) => v.fmt(f),
             Literal::String(v) => v.fmt(f),
         }
@@ -224,6 +228,7 @@ impl core::fmt::Display for Literal {
             Literal::UnsignedBigInt(v) => v.fmt(f),
             Literal::GroupElement(v) => v.fmt(f),
             Literal::AvlTree(v) => write!(f, "AvlTree({:?})", v),
+            Literal::Header(v) => write!(f, "Header({:?})", v),
             Literal::CBox(v) => write!(f, "ErgoBox({:?})", v),
             Literal::String(v) => write!(f, "String({v})"),
         }
@@ -422,9 +427,9 @@ impl<'ctx> TryFrom<Value<'ctx>> for Constant {
                 }
             }
             Value::AvlTree(a) => Ok(Constant::from(*a)),
+            Value::Header(h) => Ok(Constant::from(*h)),
             Value::String(s) => Ok(Constant::from(s)),
             Value::Context => Err("Cannot convert Value::Context into Constant".into()),
-            Value::Header(_) => Err("Cannot convert Value::Header(_) into Constant".into()),
             Value::PreHeader(_) => Err("Cannot convert Value::PreHeader(_) into Constant".into()),
             Value::Global => Err("Cannot convert Value::Global into Constant".into()),
             Value::Lambda(_) => Err("Cannot convert Value::Lambda(_) into Constant".into()),
@@ -643,6 +648,15 @@ impl From<AvlTreeData> for Constant {
     }
 }
 
+impl From<Header> for Constant {
+    fn from(h: Header) -> Self {
+        Constant {
+            tpe: SType::SHeader,
+            v: Literal::Header(h.into()),
+        }
+    }
+}
+
 impl From<AvlTreeFlags> for Constant {
     fn from(a: AvlTreeFlags) -> Self {
         Constant {
@@ -1209,6 +1223,8 @@ pub(crate) mod arbitrary {
 #[cfg(feature = "arbitrary")]
 #[allow(clippy::panic)]
 mod tests {
+    use crate::{ergo_tree::ErgoTreeVersion, serialization::roundtrip_new_feature};
+
     use super::*;
     use core::fmt;
     use proptest::prelude::*;
@@ -1424,5 +1440,10 @@ mod tests {
             test_constant_roundtrip(v);
         }
 
+        #[test]
+        fn header_ser_roundtrip(h in any::<Header>()) {
+            roundtrip_new_feature(&Constant::from(h), ErgoTreeVersion::V3);
+        }
+
     }
 }
diff --git a/ergotree-ir/src/mir/value.rs b/ergotree-ir/src/mir/value.rs
@@ -386,6 +386,7 @@ impl From<Literal> for Value<'static> {
                 Value::Coll(converted_coll)
             }
             Literal::AvlTree(a) => Value::AvlTree(a),
+            Literal::Header(h) => Value::Header(h),
             Literal::Opt(lit) => Value::Opt(lit.map(|boxed| *boxed).map(Value::from).map(Box::new)),
             Literal::Tup(t) => Value::Tup(t.mapped(Value::from)),
         }
diff --git a/ergotree-ir/src/serialization/data.rs b/ergotree-ir/src/serialization/data.rs
@@ -1,4 +1,6 @@
 use alloc::boxed::Box;
+use ergo_chain_types::Header;
+use sigma_ser::ScorexSerializable;
 
 use alloc::string::String;
 use alloc::string::ToString;
@@ -93,13 +95,21 @@ impl DataSerializer {
             Literal::Tup(items) => items
                 .iter()
                 .try_for_each(|i| DataSerializer::sigma_serialize(i, w))?,
+            Literal::Header(h) if w.tree_version() >= ErgoTreeVersion::V3 => {
+                h.scorex_serialize(w)?;
+            }
             // unsupported, see
             // https://github.com/ScorexFoundation/sigmastate-interpreter/issues/659
             Literal::Opt(_) => {
                 return Err(SigmaSerializationError::NotSupported(
                     "Option serialization is not supported".to_string(),
                 ));
             }
+            Literal::Header(_) => {
+                return Err(SigmaSerializationError::NotSupported(
+                    "Header serialization is not supported".to_string(),
+                ));
+            }
         })
     }
 
@@ -174,6 +184,9 @@ impl DataSerializer {
             }
             SBox => Literal::CBox(Arc::new(ErgoBox::sigma_parse(r)?).into()),
             SAvlTree => Literal::AvlTree(Box::new(AvlTreeData::sigma_parse(r)?)),
+            SHeader if r.tree_version() >= ErgoTreeVersion::V3 => {
+                Literal::Header(Box::new(Header::scorex_parse(r)?))
+            }
             STypeVar(_) => return Err(SigmaParsingError::NotSupported("TypeVar data")),
             SAny => return Err(SigmaParsingError::NotSupported("SAny data")),
             SOption(_) => return Err(SigmaParsingError::NotSupported("SOption data")),

Original file line number	Diff line number	Diff line change
`@@ -16,13 +16,14 @@ where`
`16`	`16`	`serializer.serialize_str(&base16::encode_lower(value))`
`17`	`17`	`}`
`18`	`18`
`19`		`-pub(crate) fn from_base16_string<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>`
	`19`	`+pub(crate) fn from_base16_string<'de, D, T: From<Vec<u8>>>(deserializer: D) -> Result<T, D::Error>`
`20`	`20`	`where`
`21`	`21`	`D: Deserializer<'de>,`
`22`	`22`	`{`
`23`	`23`	`use serde::de::Error;`
`24`	`24`	`String::deserialize(deserializer)`
`25`	`25`	`.and_then(\|string\| base16::decode(&string).map_err(\|err\| Error::custom(err.to_string())))`
	`26`	`+ .map(From::from)`
`26`	`27`	`}`
`27`	`28`
`28`	`29`	/// Serialize `BigInt` as a string
Original file line number	Diff line number	Diff line change
`@@ -250,7 +250,7 @@ pub(crate) static DECODE_NBITS_EVAL_FN: EvalFn = \|_mc, _env, _ctx, _obj, args\| {`
`250`	`250`	`#[cfg(test)]`
`251`	`251`	`#[cfg(feature = "arbitrary")]`
`252`	`252`	`mod tests {`
`253`		`- use ergo_chain_types::EcPoint;`
	`253`	`+ use ergo_chain_types::{EcPoint, Header};`
`254`	`254`	`use ergotree_ir::bigint256::BigInt256;`
`255`	`255`	`use ergotree_ir::ergo_tree::ErgoTreeVersion;`
`256`	`256`	`use ergotree_ir::mir::constant::Constant;`
`@@ -682,5 +682,9 @@ mod tests {`
`682`	`682`	`fn serialize_unsigned_bigint(v in any::<UnsignedBigInt>()) {`
`683`	`683`	`assert_eq!(deserialize(&serialize(v), SType::SUnsignedBigInt), Constant::from(v));`
`684`	`684`	`}`
	`685`	`+ #[test]`
	`686`	`+ fn serialize_header(h in any::<Header>()) {`
	`687`	`+ assert_eq!(deserialize(&serialize(h.clone()), SType::SHeader), Constant::from(h));`
	`688`	`+ }`
`685`	`689`	`}`
`686`	`690`	`}`
Original file line number	Diff line number	Diff line change
`@@ -386,6 +386,7 @@ impl From<Literal> for Value<'static> {`
`386`	`386`	`Value::Coll(converted_coll)`
`387`	`387`	`}`
`388`	`388`	`Literal::AvlTree(a) => Value::AvlTree(a),`
	`389`	`+ Literal::Header(h) => Value::Header(h),`
`389`	`390`	`Literal::Opt(lit) => Value::Opt(lit.map(\|boxed\| *boxed).map(Value::from).map(Box::new)),`
`390`	`391`	`Literal::Tup(t) => Value::Tup(t.mapped(Value::from)),`
`391`	`392`	`}`