feat(drive-abci): state sync - faster sync of new nodes

.github/actions/sccache/action.yaml

@@ -34,7 +34,7 @@ inputs:
     default: "true"
   version:
     description: "sccache version"
-    default: "0.8.2"
+    default: "0.9.1"
     required: false
 outputs:
   env_vars:

Dockerfile

@@ -173,7 +173,7 @@ ENV NODE_ENV=${NODE_ENV}
 FROM deps-base AS deps-sccache
 
 # SCCACHE_VERSION must be the same as in github actions, to avoid cache incompatibility
-ARG SCCHACHE_VERSION=0.8.2
+ARG SCCHACHE_VERSION=0.9.1
 
 # Install sccache for caching
 RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else export SCC_ARCH=x86_64; fi; \
@@ -584,19 +584,24 @@ LABEL description="Drive ABCI Rust"
 RUN apk add --no-cache libgcc libstdc++
 
 ENV DB_PATH=/var/lib/dash/rs-drive-abci/db
+ENV CHECKPOINTS_PATH=/var/lib/dash/rs-drive-abci/db-checkpoints
 ENV REJECTIONS_PATH=/var/log/dash/rejected
 
-RUN mkdir -p /var/log/dash \
-    /var/lib/dash/rs-drive-abci/db \
-    ${REJECTIONS_PATH}
-
 COPY --from=build-drive-abci /artifacts/drive-abci /usr/bin/drive-abci
 COPY packages/rs-drive-abci/.env.mainnet /var/lib/dash/rs-drive-abci/.env
 
 # Create a volume
 VOLUME /var/lib/dash/rs-drive-abci/db
 VOLUME /var/log/dash
 
+# Ensure required paths do exist
+# TODO: remove /var/lib/dash-platform/data/checkpoints when drive-abci is fixed
+RUN mkdir -p /var/log/dash \
+    ${DB_PATH} \
+    ${CHECKPOINTS_PATH} \
+    ${REJECTIONS_PATH} \
+    /var/lib/dash-platform/data/checkpoints
+
 # Double-check that we don't have missing deps
 RUN ldd /usr/bin/drive-abci
 
@@ -606,9 +611,10 @@ RUN ldd /usr/bin/drive-abci
 ARG USERNAME=dash
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
+# TODO: remove /var/lib/dash-platform/data/checkpoints when drive-abci is fixed
 RUN addgroup -g $USER_GID $USERNAME && \
     adduser -D -u $USER_UID -G $USERNAME -h /var/lib/dash/rs-drive-abci $USERNAME && \
-    chown -R $USER_UID:$USER_GID /var/lib/dash/rs-drive-abci /var/log/dash
+    chown -R $USER_UID:$USER_GID /var/lib/dash/rs-drive-abci /var/log/dash /var/lib/dash-platform/data/checkpoints
 
 USER $USERNAME
 

packages/dapi-grpc/Cargo.toml

@@ -42,6 +42,10 @@ tonic = { version = "0.12.3", features = [
 serde = { version = "1.0.197", optional = true, features = ["derive"] }
 serde_bytes = { version = "0.11.12", optional = true }
 serde_json = { version = "1.0", optional = true }
+tenderdash-proto = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.3.0", tag = "v1.3.0+1.3.0", default-features = false, features = [
+  "grpc",
+  "serde",
+] }
 dapi-grpc-macros = { path = "../rs-dapi-grpc-macros" }
 platform-version = { path = "../rs-platform-version" }
 

packages/dashmate/configs/defaults/getBaseConfigFactory.js

@@ -309,7 +309,7 @@ export default function getBaseConfigFactory() {
           tenderdash: {
             mode: 'full',
             docker: {
-              image: 'dashpay/tenderdash:1',
+              image: 'dashpay/tenderdash:feat-statesync-improvements',
             },
             p2p: {
               host: '0.0.0.0',

packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot

@@ -81,7 +81,7 @@ filter-peers = false
 # Example for routed multi-app setup:
 #   abci = "routed"
 #   address = "Info:socket:unix:///tmp/socket.1,Info:socket:unix:///tmp/socket.2,CheckTx:socket:unix:///tmp/socket.1,*:socket:unix:///tmp/socket.3"
-address = "CheckTx:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658"
+address = "ListSnapshots:grpc:drive_abci:26670,LoadSnapshotChunk:grpc:drive_abci:26670,CheckTx:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658"
 # Transport mechanism to connect to the ABCI application: socket | grpc | routed
 transport = "routed"
 # Maximum number of simultaneous connections to the ABCI application
@@ -97,6 +97,10 @@ transport = "routed"
 #]
 grpc-concurrency = [
   { "check_tx" = {{= it.platform.drive.tenderdash.mempool.maxConcurrentCheckTx }} },
+  { "list_snapshots" = {{= it.platform.drive.tenderdash.mempool.maxConcurrentCheckTx }} },
+  { "load_snapshot_chunk" = {{= it.platform.drive.tenderdash.mempool.maxConcurrentCheckTx }} },
+  { "offer_snapshot" = 1 },
+  { "apply_snapshot_chunk" = 1 },
 ]
 
 
@@ -414,26 +418,17 @@ ttl-num-blocks = {{=it.platform.drive.tenderdash.mempool.ttlNumBlocks}}
 # the network to take and serve state machine snapshots. State sync is not attempted if the node
 # has any local state (LastBlockHeight > 0). The node will have a truncated block history,
 # starting from the height of the snapshot.
-enable = false
+enable = true
 
 # State sync uses light client verification to verify state. This can be done either through the
 # P2P layer or RPC layer. Set this to true to use the P2P layer. If false (default), RPC layer
 # will be used.
-use-p2p = false
+use-p2p = true
 
 # If using RPC, at least two addresses need to be provided. They should be compatible with net.Dial,
 # for example: "host.example.com:2125"
 rpc-servers = ""
 
-# The hash and height of a trusted block. Must be within the trust-period.
-trust-height = 0
-trust-hash = ""
-
-# The trust period should be set so that Tendermint can detect and gossip misbehavior before
-# it is considered expired. For chains based on the Cosmos SDK, one day less than the unbonding
-# period should suffice.
-trust-period = "168h0m0s"
-
 # Time to spend discovering snapshots before initiating a restore.
 discovery-time = "15s"
 

packages/rs-dapi-client/Cargo.toml

@@ -55,7 +55,7 @@ sha2 = { version = "0.10", optional = true }
 hex = { version = "0.4.3", optional = true }
 lru = { version = "0.12.3" }
 serde = { version = "1.0.197", optional = true, features = ["derive"] }
-serde_json = { version = "1.0.120", optional = true }
+serde_json = { version = "1.0", optional = true }
 chrono = { version = "0.4.38", features = ["serde"] }
 
 [dev-dependencies]

packages/rs-dpp/src/lib.rs

@@ -59,6 +59,7 @@ pub mod voting;
 pub mod core_types;
 
 pub mod group;
+pub mod reduced_platform_state;
 pub mod withdrawal;
 
 pub use async_trait;

packages/rs-dpp/src/reduced_platform_state/mod.rs

@@ -0,0 +1,38 @@
+use crate::reduced_platform_state::v0::ReducedPlatformStateForSavingV0;
+use crate::serialization::{PlatformSerializable, ReducedPlatformDeserializable};
+use crate::ProtocolError;
+use bincode::{config, Decode, Encode};
+use derive_more::From;
+use platform_version::version::PlatformVersion;
+use platform_version::TryIntoPlatformVersioned;
+use serde::{Deserialize, Serialize};
+
+pub mod v0;
+
+/// Reduced Platform State For Saving
+#[derive(Clone, Debug, Encode, Decode)]
+pub enum ReducedPlatformStateForSaving {
+    V0(ReducedPlatformStateForSavingV0),
+}
+
+impl ReducedPlatformDeserializable for ReducedPlatformStateForSaving {
+    fn versioned_deserialize(
+        data: &[u8],
+        platform_version: &PlatformVersion,
+    ) -> Result<Self, ProtocolError>
+    where
+        Self: Sized,
+    {
+        let config = config::standard().with_big_endian().with_no_limit();
+        let reduced_platform_state_in_save_format: ReducedPlatformStateForSaving =
+            bincode::decode_from_slice(data, config)
+                .map_err(|e| {
+                    ProtocolError::PlatformDeserializationError(format!(
+                        "unable to deserialize ReducedPlatformStateForSaving: {}",
+                        e
+                    ))
+                })?
+                .0;
+        Ok(reduced_platform_state_in_save_format)
+    }
+}

packages/rs-dpp/src/reduced_platform_state/v0/mod.rs

@@ -0,0 +1,21 @@
+use crate::block::block_info::BlockInfo;
+use crate::block::extended_block_info::ExtendedBlockInfo;
+use crate::fee::default_costs::EpochIndexFeeVersionsForStorage;
+use crate::util::deserializer::ProtocolVersion;
+use bincode::{Decode, Encode};
+use platform_value::Bytes32;
+
+/// Reduced Platform State for Saving V0
+#[derive(Clone, Debug, Encode, Decode)]
+pub struct ReducedPlatformStateForSavingV0 {
+    /// Current Version
+    pub current_protocol_version_in_consensus: ProtocolVersion,
+    /// upcoming protocol version
+    pub next_epoch_protocol_version: ProtocolVersion,
+    /// current quorum
+    pub current_validator_set_quorum_hash: Bytes32,
+    /// next quorum
+    pub next_validator_set_quorum_hash: Option<Bytes32>,
+    /// previous Fee Versions
+    pub previous_fee_versions: EpochIndexFeeVersionsForStorage,
+}

packages/rs-dpp/src/serialization/serialization_traits.rs

@@ -31,6 +31,20 @@ pub trait PlatformSerializable {
     }
 }
 
+pub trait ReducedPlatformSerializable {
+    type Error;
+    fn reduced_serialize_to_bytes(&self) -> Result<Vec<u8>, Self::Error>;
+}
+
+pub trait ReducedPlatformDeserializable {
+    fn versioned_deserialize(
+        data: &[u8],
+        platform_version: &PlatformVersion,
+    ) -> Result<Self, ProtocolError>
+    where
+        Self: Sized;
+}
+
 pub trait PlatformSerializableWithPlatformVersion {
     type Error;
     /// Version based serialization is done based on the desired structure version.

packages/rs-drive-abci/.env.local

@@ -12,6 +12,12 @@ ABCI_LOG_STDOUT_FORMAT=pretty
 ABCI_LOG_STDOUT_COLOR=true
 
 DB_PATH=/tmp/db
+
+CHECKPOINTS_PATH=${DB_PATH}/checkpoints
+
+# GroveDB database file
+GROVEDB_LATEST_FILE=${DB_PATH}/latest_state
+
 REJECTIONS_PATH=/tmp/rejected
 
 # Cache size for Data Contracts

packages/rs-drive-abci/Cargo.toml

@@ -53,7 +53,9 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features =
   "tracing-log",
 ], optional = false }
 tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.3.0", tag = "v1.3.0+1.3.0", features = [
+  "crypto",
   "grpc",
+  "serde",
 ] }
 
 lazy_static = "1.4.0"
@@ -77,7 +79,9 @@ tokio-util = { version = "0.7" }
 derive_more = { version = "1.0", features = ["from", "deref", "deref_mut"] }
 async-trait = "0.1.77"
 console-subscriber = { version = "0.4", optional = true }
-bls-signatures = { git = "https://github.com/dashpay/bls-signatures", tag = "1.3.3", optional = true }
+bls-signatures = { git = "https://github.com/dashpay/bls-signatures", tag = "1.3.3", optional = true}
+crc = "3.2.1"
+
 
 [dev-dependencies]
 bs58 = { version = "0.5.0" }