feat(drive-abci): state sync - faster sync of new nodes

.github/actions/sccache/action.yaml

@@ -34,7 +34,7 @@ inputs:
     default: "true"
   version:
     description: "sccache version"
-    default: "0.8.2"
+    default: "0.9.1"
     required: false
 outputs:
   env_vars:

Dockerfile

@@ -175,7 +175,7 @@ ENV NODE_ENV=${NODE_ENV}
 FROM deps-base AS deps-sccache
 
 # SCCACHE_VERSION must be the same as in github actions, to avoid cache incompatibility
-ARG SCCHACHE_VERSION=0.8.2
+ARG SCCHACHE_VERSION=0.9.1
 
 # Install sccache for caching
 RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else export SCC_ARCH=x86_64; fi; \
@@ -600,19 +600,24 @@ LABEL description="Drive ABCI Rust"
 RUN apk add --no-cache libgcc libstdc++
 
 ENV DB_PATH=/var/lib/dash/rs-drive-abci/db
+ENV CHECKPOINTS_PATH=/var/lib/dash/rs-drive-abci/db-checkpoints
 ENV REJECTIONS_PATH=/var/log/dash/rejected
-
-RUN mkdir -p /var/log/dash \
-    /var/lib/dash/rs-drive-abci/db \
-    ${REJECTIONS_PATH}
+ENV SNAPSHOTS_ENABLED=true
 
 COPY --from=build-drive-abci /artifacts/drive-abci /usr/bin/drive-abci
 COPY packages/rs-drive-abci/.env.mainnet /var/lib/dash/rs-drive-abci/.env
 
 # Create a volume
 VOLUME /var/lib/dash/rs-drive-abci/db
+VOLUME /var/lib/dash/rs-drive-abci/db-checkpoints
 VOLUME /var/log/dash
 
+# Ensure required paths do exist
+RUN mkdir -p /var/log/dash \
+    ${DB_PATH} \
+    ${CHECKPOINTS_PATH} \
+    ${REJECTIONS_PATH}
+
 # Double-check that we don't have missing deps
 RUN ldd /usr/bin/drive-abci
 
@@ -622,6 +627,7 @@ RUN ldd /usr/bin/drive-abci
 ARG USERNAME=dash
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
+
 RUN addgroup -g $USER_GID $USERNAME && \
     adduser -D -u $USER_UID -G $USERNAME -h /var/lib/dash/rs-drive-abci $USERNAME && \
     chown -R $USER_UID:$USER_GID /var/lib/dash/rs-drive-abci /var/log/dash

packages/dapi-grpc/Cargo.toml

@@ -39,8 +39,6 @@ serde = ["dep:serde", "dep:serde_bytes", "tenderdash-proto/serde"]
 mocks = ["serde", "dep:serde_json"]
 
 [dependencies]
-tenderdash-proto = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.4.0", tag = "v1.4.0", default-features = false }
-
 prost = { version = "0.13" }
 futures-core = "0.3.30"
 tonic = { version = "0.13.0", features = [
@@ -51,6 +49,7 @@ tonic = { version = "0.13.0", features = [
 serde = { version = "1.0.219", optional = true, features = ["derive"] }
 serde_bytes = { version = "0.11.12", optional = true }
 serde_json = { version = "1.0", optional = true }
+tenderdash-proto = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.4.0", tag = "v1.4.0", default-features = false }
 dapi-grpc-macros = { path = "../rs-dapi-grpc-macros" }
 platform-version = { path = "../rs-platform-version" }
 

packages/dashmate/configs/defaults/getBaseConfigFactory.js

@@ -305,11 +305,18 @@ export default function getBaseConfigFactory() {
               txProcessingTimeLimit: null,
             },
             epochTime: 788400,
+            stateSync: {
+              snapshots: {
+                enabled: true,
+                frequency: 5,
+                maxLimit: 100,
+              },
+            },
           },
           tenderdash: {
             mode: 'full',
             docker: {
-              image: 'dashpay/tenderdash:1',
+              image: 'dashpay/tenderdash:feat-statesync-improvements',
             },
             p2p: {
               host: '0.0.0.0',
@@ -412,6 +419,14 @@ export default function getBaseConfigFactory() {
               },
             },
             moniker: null,
+            stateSync: {
+              enabled: true,
+              retries: 3,
+              chunkRequestTimeout: '15s',
+              fetchersCount: 4,
+              maxConcurrentListSnapshots: 100,
+              maxConcurrentSnapshotChunk: 100,
+            },
           },
         },
         sourcePath: null,

packages/dashmate/configs/defaults/getLocalConfigFactory.js

@@ -63,6 +63,9 @@ export default function getLocalConfigFactory(getBaseConfig) {
             metrics: {
               port: 46660,
             },
+            stateSync: {
+              enabled: false,
+            },
           },
           abci: {
             epochTime: 1200,

packages/dashmate/configs/getConfigFileMigrationsFactory.js

@@ -20,7 +20,7 @@
    * @typedef {function} getConfigFileMigrations
    * @returns {Object}
    */
   function getConfigFileMigrations() {
     const base = defaultConfigs.get('base');
     const testnet = defaultConfigs.get('testnet');
     const mainnet = defaultConfigs.get('mainnet');
@@ -1096,8 +1096,33 @@
           });
         return configFile;
       },
+      '2.1.0-dev.1': (configFile) => {
+        Object.entries(configFile.configs)
+          .forEach(([, options]) => {
+            options.platform.drive.tenderdash.stateSync = {
+              enabled: true,
+              retries: 3,
+              chunkRequestTimeout: '15s',
+              fetchersCount: 4,
+              maxConcurrentListSnapshots: 100,
+              maxConcurrentSnapshotChunk: 100,
+            };
+
+            options.platform.drive.abci.stateSync = {
+              snapshots: {
+                enabled: true,
+                frequency: 5,
+                maxLimit: 100,
+              },
+            };
+          });
+
+        configFile.configs.local.platform.drive.abci.stateSync.snapshots.enabled = false;
+        configFile.configs.local.platform.drive.tenderdash.stateSync.enabled = false;
+
+        return configFile;
+      },
     };
   }
 
-  return getConfigFileMigrations;
 }

packages/dashmate/docker-compose.yml

@@ -58,6 +58,7 @@ services:
     logging: *default-logging
     volumes:
       - drive_abci_data:/var/lib/dash/rs-drive-abci/db
+      - drive_abci_checkpoints:/var/lib/dash/rs-drive-abci/db-checkpoints
     environment:
       - CHAIN_ID=${PLATFORM_DRIVE_TENDERDASH_GENESIS_CHAIN_ID:-devnet}
       - CORE_CONSENSUS_JSON_RPC_USERNAME=drive_consensus
@@ -82,6 +83,7 @@ services:
       - INSTANT_LOCK_QUORUM_ACTIVE_SIGNERS=${PLATFORM_DRIVE_ABCI_INSTANT_LOCK_QUORUM_ACTIVE_SIGNERS:?err}
       - INSTANT_LOCK_QUORUM_ROTATION=${PLATFORM_DRIVE_ABCI_INSTANT_LOCK_QUORUM_ROTATION:?err}
       - DB_PATH=/var/lib/dash/rs-drive-abci/db
+      - GROVEDB_LATEST_FILE=/var/lib/dash/rs-drive-abci/db/latest_state
       - ABCI_CONSENSUS_BIND_ADDRESS=tcp://0.0.0.0:26658
       - GRPC_BIND_ADDRESS=0.0.0.0:26670
       - PROMETHEUS_BIND_ADDRESS=${PLATFORM_DRIVE_ABCI_METRICS_URL}
@@ -92,6 +94,10 @@ services:
       - GROVEDB_VISUALIZER_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_GROVEDB_VISUALIZER_PORT:?err}
       - PROPOSER_TX_PROCESSING_TIME_LIMIT=${PLATFORM_DRIVE_ABCI_PROPOSER_TX_PROCESSING_TIME_LIMIT}
       - NETWORK=${NETWORK:?err}
+      - CHECKPOINTS_PATH=/var/lib/dash/rs-drive-abci/db-checkpoints
+      - SNAPSHOTS_ENABLED=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_ENABLED:?err}
+      - SNAPSHOTS_FREQUENCY=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_FREQUENCY:?err}
+      - MAX_NUM_SNAPSHOTS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_MAX_LIMIT:?err}
     stop_grace_period: 30s
     expose:
       - 26658
@@ -226,6 +232,7 @@ services:
 volumes:
   core_data:
   drive_abci_data:
+  drive_abci_checkpoints:
   drive_tenderdash:
 
 networks:

packages/dashmate/src/config/configJsonSchema.js

@@ -949,6 +949,31 @@ export default {
                   required: ['txProcessingTimeLimit'],
                   additionalProperties: false,
                 },
+                stateSync: {
+                  type: 'object',
+                  properties: {
+                    snapshots: {
+                      type: 'object',
+                      properties: {
+                        enabled: {
+                          type: 'boolean',
+                        },
+                        frequency: {
+                          type: 'integer',
+                          minimum: 1,
+                        },
+                        maxLimit: {
+                          type: 'integer',
+                          minimum: 1,
+                        },
+                      },
+                      required: ['enabled', 'frequency', 'maxLimit'],
+                      additionalProperties: false,
+                    },
+                  },
+                  required: ['snapshots'],
+                  additionalProperties: false,
+                },
               },
               additionalProperties: false,
               required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime', 'metrics', 'grovedbVisualizer', 'proposer'],
@@ -1208,8 +1233,37 @@ export default {
                 genesis: {
                   type: 'object',
                 },
+                stateSync: {
+                  type: 'object',
+                  properties: {
+                    enabled: {
+                      type: 'boolean',
+                    },
+                    retries: {
+                      type: 'integer',
+                      minimum: 1,
+                    },
+                    chunkRequestTimeout: {
+                      $ref: '#/definitions/duration',
+                    },
+                    fetchersCount: {
+                      type: 'integer',
+                      minimum: 1,
+                    },
+                    maxConcurrentListSnapshots: {
+                      type: 'integer',
+                      minimum: 1,
+                    },
+                    maxConcurrentSnapshotChunk: {
+                      type: 'integer',
+                      minimum: 1,
+                    },
+                  },
+                  required: ['enabled', 'retries', 'chunkRequestTimeout', 'fetchersCount', 'maxConcurrentListSnapshots', 'maxConcurrentSnapshotChunk'],
+                  additionalProperties: false,
+                },
               },
-              required: ['mode', 'docker', 'p2p', 'mempool', 'consensus', 'log', 'rpc', 'pprof', 'node', 'moniker', 'genesis', 'metrics'],
+              required: ['mode', 'docker', 'p2p', 'mempool', 'consensus', 'log', 'rpc', 'pprof', 'node', 'moniker', 'genesis', 'metrics', 'stateSync'],
               additionalProperties: false,
             },
           },

packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot

@@ -81,7 +81,7 @@ filter-peers = false
 # Example for routed multi-app setup:
 #   abci = "routed"
 #   address = "Info:socket:unix:///tmp/socket.1,Info:socket:unix:///tmp/socket.2,CheckTx:socket:unix:///tmp/socket.1,*:socket:unix:///tmp/socket.3"
-address = "CheckTx:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658"
+address = "ListSnapshots:grpc:drive_abci:26670,LoadSnapshotChunk:grpc:drive_abci:26670,CheckTx:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658"
 # Transport mechanism to connect to the ABCI application: socket | grpc | routed
 transport = "routed"
 # Maximum number of simultaneous connections to the ABCI application
@@ -97,6 +97,10 @@ transport = "routed"
 #]
 grpc-concurrency = [
   { "check_tx" = {{= it.platform.drive.tenderdash.mempool.maxConcurrentCheckTx }} },
+  { "list_snapshots" = {{= it.platform.drive.tenderdash.stateSync.maxConcurrentListSnapshots }} },
+  { "load_snapshot_chunk" = {{= it.platform.drive.tenderdash.stateSync.maxConcurrentSnapshotChunk }} },
+  { "offer_snapshot" = 1 },
+  { "apply_snapshot_chunk" = 1 },
 ]
 
 
@@ -414,40 +418,37 @@ ttl-num-blocks = {{=it.platform.drive.tenderdash.mempool.ttlNumBlocks}}
 # the network to take and serve state machine snapshots. State sync is not attempted if the node
 # has any local state (LastBlockHeight > 0). The node will have a truncated block history,
 # starting from the height of the snapshot.
-enable = false
+enable = {{= it.platform.drive.tenderdash.stateSync.enabled.toString() }}
 
 # State sync uses light client verification to verify state. This can be done either through the
 # P2P layer or RPC layer. Set this to true to use the P2P layer. If false (default), RPC layer
 # will be used.
-use-p2p = false
+use-p2p = true
 
 # If using RPC, at least two addresses need to be provided. They should be compatible with net.Dial,
 # for example: "host.example.com:2125"
 rpc-servers = ""
 
-# The hash and height of a trusted block. Must be within the trust-period.
-trust-height = 0
-trust-hash = ""
-
-# The trust period should be set so that Tendermint can detect and gossip misbehavior before
-# it is considered expired. For chains based on the Cosmos SDK, one day less than the unbonding
-# period should suffice.
-trust-period = "168h0m0s"
-
 # Time to spend discovering snapshots before initiating a restore.
 discovery-time = "15s"
 
+# Number of times to retry state sync. When retries are exhausted, the node will
+# fall back to the regular block sync. Set to 0 to disable retries. Default is 3.
+# Note that in pessimistic case, it will take at least (discovery-time * retries) before
+# falling back to block sync.
+retries = {{= it.platform.drive.tenderdash.stateSync.retries }}
+
 # Temporary directory for state sync snapshot chunks, defaults to os.TempDir().
 # The synchronizer will create a new, randomly named directory within this directory
 # and remove it when the sync is complete.
 temp-dir = ""
 
 # The timeout duration before re-requesting a chunk, possibly from a different
 # peer (default: 15 seconds).
-chunk-request-timeout = "15s"
+chunk-request-timeout = "{{= it.platform.drive.tenderdash.stateSync.chunkRequestTimeout }}"
 
 # The number of concurrent chunk and block fetchers to run (default: 4).
-fetchers = "4"
+fetchers = "{{= it.platform.drive.tenderdash.stateSync.fetchersCount }}"
 
 #######################################################
 ###         Consensus Configuration Options         ###

packages/rs-dpp/src/lib.rs

@@ -59,6 +59,7 @@ pub mod voting;
 pub mod core_types;
 
 pub mod group;
+pub mod reduced_platform_state;
 pub mod withdrawal;
 
 pub use async_trait;