fix: Use separate snapshots for sequencer and validator workloads

Author: tobidae-cb

runner/service.go

@@ -80,7 +80,7 @@ func readBenchmarkConfig(path string) (*benchmark.BenchmarkConfig, error) {
 	return config, err
 }
 
-func (s *service) setupInternalDirectories(testDir string, params types.RunParams, genesis *core.Genesis, snapshot *benchmark.SnapshotDefinition, role string) (*config.InternalClientOptions, error) {
+func (s *service) setupInternalDirectories(testDir string, params types.RunParams, genesis *core.Genesis, snapshot *benchmark.SnapshotDefinition, role string, dataDirOverride string) (*config.InternalClientOptions, error) {
 	err := os.MkdirAll(testDir, 0755)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create test directory")
@@ -105,52 +105,61 @@ func (s *service) setupInternalDirectories(testDir string, params types.RunParam
 	}
 
 	var dataDirPath string
-	isSnapshot := snapshot != nil && snapshot.Command != ""
-	if isSnapshot {
-		dataDirPath = path.Join(testDir, "data")
+	var isSnapshot bool
 
-		initialSnapshotPath := s.dataDirState.GetInitialSnapshotPath(params.NodeType)
-
-		if initialSnapshotPath != "" && s.fileExists(initialSnapshotPath) {
-			snapshotMethod := snapshot.GetSnapshotMethod()
-
-			switch snapshotMethod {
-			case benchmark.SnapshotMethodReuseExisting:
-				dataDirPath = initialSnapshotPath
-				s.log.Info("Reusing existing snapshot", "snapshotPath", initialSnapshotPath, "method", snapshotMethod)
-			case benchmark.SnapshotMethodHeadRollback:
-				// For head_rollback, copy the snapshot but mark it for rollback later
-				err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, dataDirPath)
-				if err != nil {
-					return nil, errors.Wrap(err, "failed to copy from initial snapshot for head rollback")
+	// If dataDirOverride is provided, use it (already set up by caller)
+	if dataDirOverride != "" {
+		dataDirPath = dataDirOverride
+		isSnapshot = true // dataDirOverride is only set when using snapshots
+		s.log.Info("Using pre-configured datadir", "path", dataDirPath, "role", role)
+	} else {
+		isSnapshot = snapshot != nil && snapshot.Command != ""
+		if isSnapshot {
+			dataDirPath = path.Join(testDir, "data")
+
+			initialSnapshotPath := s.dataDirState.GetInitialSnapshotPath(params.NodeType)
+
+			if initialSnapshotPath != "" && s.fileExists(initialSnapshotPath) {
+				snapshotMethod := snapshot.GetSnapshotMethod()
+
+				switch snapshotMethod {
+				case benchmark.SnapshotMethodReuseExisting:
+					dataDirPath = initialSnapshotPath
+					s.log.Info("Reusing existing snapshot", "snapshotPath", initialSnapshotPath, "method", snapshotMethod)
+				case benchmark.SnapshotMethodHeadRollback:
+					// For head_rollback, copy the snapshot but mark it for rollback later
+					err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, dataDirPath)
+					if err != nil {
+						return nil, errors.Wrap(err, "failed to copy from initial snapshot for head rollback")
+					}
+					s.log.Info("Copied from initial snapshot for head rollback", "initialSnapshotPath", initialSnapshotPath, "dataDirPath", dataDirPath, "method", snapshotMethod)
+				default:
+					// Default chain_copy behavior
+					err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, dataDirPath)
+					if err != nil {
+						return nil, errors.Wrap(err, "failed to copy from initial snapshot")
+					}
+					s.log.Info("Copied from initial snapshot", "initialSnapshotPath", initialSnapshotPath, "dataDirPath", dataDirPath)
 				}
-				s.log.Info("Copied from initial snapshot for head rollback", "initialSnapshotPath", initialSnapshotPath, "dataDirPath", dataDirPath, "method", snapshotMethod)
-			default:
-				// Default chain_copy behavior
-				err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, dataDirPath)
+			} else {
+				// Fallback to direct snapshot creation
+				if initialSnapshotPath != "" {
+					s.log.Warn("Initial snapshot path registered but doesn't exist, falling back to direct snapshot creation",
+						"path", initialSnapshotPath, "nodeType", params.NodeType)
+				}
+				snapshotDir, err := s.dataDirState.EnsureSnapshot(*snapshot, params.NodeType, role)
 				if err != nil {
-					return nil, errors.Wrap(err, "failed to copy from initial snapshot")
+					return nil, errors.Wrap(err, "failed to ensure snapshot")
 				}
-				s.log.Info("Copied from initial snapshot", "initialSnapshotPath", initialSnapshotPath, "dataDirPath", dataDirPath)
+				dataDirPath = snapshotDir
 			}
 		} else {
-			// Fallback to direct snapshot creation
-			if initialSnapshotPath != "" {
-				s.log.Warn("Initial snapshot path registered but doesn't exist, falling back to direct snapshot creation",
-					"path", initialSnapshotPath, "nodeType", params.NodeType)
-			}
-			snapshotDir, err := s.dataDirState.EnsureSnapshot(*snapshot, params.NodeType, role)
+			// if no snapshot, just create a new datadir
+			dataDirPath = path.Join(testDir, "data")
+			err = os.Mkdir(dataDirPath, 0755)
 			if err != nil {
-				return nil, errors.Wrap(err, "failed to ensure snapshot")
+				return nil, errors.Wrap(err, "failed to create data directory")
 			}
-			dataDirPath = snapshotDir
-		}
-	} else {
-		// if no snapshot, just create a new datadir
-		dataDirPath = path.Join(testDir, "data")
-		err = os.Mkdir(dataDirPath, 0755)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create data directory")
 		}
 	}
 
@@ -326,12 +335,45 @@ func (s *service) setupDataDirs(workingDir string, params types.RunParams, genes
 	sequencerTestDir := path.Join(workingDir, fmt.Sprintf("%s-sequencer", testName))
 	validatorTestDir := path.Join(workingDir, fmt.Sprintf("%s-validator", testName))
 
-	sequencerOptions, err := s.setupInternalDirectories(sequencerTestDir, params, genesis, snapshot, "sequencer")
+	// Special handling for SnapshotMethodReuseExisting to avoid both nodes using the same datadir
+	var sequencerDataDirOverride, validatorDataDirOverride string
+	if snapshot != nil && snapshot.GetSnapshotMethod() == benchmark.SnapshotMethodReuseExisting {
+		initialSnapshotPath := s.dataDirState.GetInitialSnapshotPath(params.NodeType)
+		if initialSnapshotPath != "" && s.fileExists(initialSnapshotPath) {
+			validatorDataDirOverride = path.Join(validatorTestDir, "data")
+			sequencerDataDirOverride = path.Join(sequencerTestDir, "data")
+
+			s.log.Info("Setting up reuse_existing snapshot: copying to validator, moving to sequencer",
+				"initialSnapshot", initialSnapshotPath,
+				"sequencerDataDir", sequencerDataDirOverride,
+				"validatorDataDir", validatorDataDirOverride)
+
+			// First: copy from initial snapshot to validator directory
+			err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, validatorDataDirOverride)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "failed to copy initial snapshot to validator directory")
+			}
+			s.log.Info("Copied initial snapshot to validator directory", "path", validatorDataDirOverride)
+
+			err = os.MkdirAll(sequencerTestDir, 0755)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "failed to create sequencer test directory")
+			}
+
+			err = os.Rename(initialSnapshotPath, sequencerDataDirOverride)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "failed to move initial snapshot to sequencer directory")
+			}
+			s.log.Info("Moved initial snapshot to sequencer directory", "from", initialSnapshotPath, "to", sequencerDataDirOverride)
+		}
+	}
+
+	sequencerOptions, err := s.setupInternalDirectories(sequencerTestDir, params, genesis, snapshot, "sequencer", sequencerDataDirOverride)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to setup internal directories")
 	}
 
-	validatorOptions, err := s.setupInternalDirectories(validatorTestDir, params, genesis, snapshot, "validator")
+	validatorOptions, err := s.setupInternalDirectories(validatorTestDir, params, genesis, snapshot, "validator", validatorDataDirOverride)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to setup internal directories")
 	}