snap batching

deps/config/types.go

@@ -30,6 +30,10 @@ func DefaultCurioConfig() *CurioConfig {
 				Base:      types.MustParseFIL("0"),
 				PerSector: types.MustParseFIL("0.03"), // enough for 6 agg and 1nFIL base fee
 			},
+			MaxUpdateBatchGasFee: BatchFeeConfig{
+				Base:      types.MustParseFIL("0"),
+				PerSector: types.MustParseFIL("0.03"),
+			},
 
 			MaxTerminateGasFee:         types.MustParseFIL("0.5"),
 			MaxWindowPoStGasFee:        types.MustParseFIL("5"),
@@ -84,6 +88,11 @@ func DefaultCurioConfig() *CurioConfig {
 				Timeout:          Duration(1 * time.Hour),
 				Slack:            Duration(1 * time.Hour),
 			},
+			Update: UpdateBatchingConfig{
+				BaseFeeThreshold: types.MustParseFIL("0.005"),
+				Timeout:          Duration(1 * time.Hour),
+				Slack:            Duration(1 * time.Hour),
+			},
 		},
 	}
 }
@@ -308,6 +317,7 @@ type CurioFees struct {
 	// maxBatchFee = maxBase + maxPerSector * nSectors
 	MaxPreCommitBatchGasFee BatchFeeConfig
 	MaxCommitBatchGasFee    BatchFeeConfig
+	MaxUpdateBatchGasFee    BatchFeeConfig
 
 	MaxTerminateGasFee types.FIL
 	// WindowPoSt is a high-value operation, so the default fee should be high.
@@ -527,6 +537,9 @@ type CurioBatchingConfig struct {
 
 	// Commit batching configuration
 	Commit CommitBatchingConfig
+
+	// Snap Deals batching configuration
+	Update UpdateBatchingConfig
 }
 
 type PreCommitBatchingConfig struct {
@@ -550,3 +563,14 @@ type CommitBatchingConfig struct {
 	// Time buffer for forceful batch submission before sectors/deals in batch would start expiring
 	Slack Duration
 }
+
+type UpdateBatchingConfig struct {
+	// Base fee value below which we should try to send Commit messages immediately
+	BaseFeeThreshold types.FIL
+
+	// Maximum amount of time any given sector in the batch can wait for the batch to accumulate
+	Timeout Duration
+
+	// Time buffer for forceful batch submission before sectors/deals in batch would start expiring
+	Slack Duration
+}

documentation/en/configuration/default-curio-configuration.md

@@ -302,6 +302,13 @@ description: The default curio configuration
     # type: types.FIL
     #PerSector = "0.03 FIL"
 
+  [Fees.MaxUpdateBatchGasFee]
+    # type: types.FIL
+    #Base = "0 FIL"
+
+    # type: types.FIL
+    #PerSector = "0.03 FIL"
+
 
 [[Addresses]]
   #PreCommitControl = []
@@ -536,4 +543,20 @@ description: The default curio configuration
     # type: Duration
     #Slack = "1h0m0s"
 
+  [Batching.Update]
+    # Base fee value below which we should try to send Commit messages immediately
+    #
+    # type: types.FIL
+    #BaseFeeThreshold = "0.005 FIL"
+
+    # Maximum amount of time any given sector in the batch can wait for the batch to accumulate
+    #
+    # type: Duration
+    #Timeout = "1h0m0s"
+
+    # Time buffer for forceful batch submission before sectors/deals in batch would start expiring
+    #
+    # type: Duration
+    #Slack = "1h0m0s"
+
 ```

harmony/harmonydb/sql/20240611-snap-pipeline.sql

@@ -26,6 +26,8 @@ CREATE TABLE sectors_snap_pipeline (
     task_id_prove BIGINT,
     after_prove BOOLEAN NOT NULL DEFAULT FALSE,
 
+    -- update_ready_at TIMESTAMP, // Added in 20241210-sdr-batching
+
     -- submit
     prove_msg_cid TEXT,
 

harmony/harmonydb/sql/20241210-sdr-batching.sql

@@ -43,4 +43,24 @@ CREATE TRIGGER update_commit_ready_at
     AFTER INSERT OR UPDATE OR DELETE ON sectors_sdr_pipeline
     FOR EACH ROW EXECUTE FUNCTION set_commit_ready_at();
 
+ALTER TABLE sectors_snap_pipeline ADD COLUMN update_ready_at TIMESTAMPTZ;
 
+-- Function to precommit_ready_at value. Used by the trigger
+CREATE OR REPLACE FUNCTION set_update_ready_at()
+RETURNS TRIGGER AS $$
+BEGIN
+    -- Check if after_prove column is changing from FALSE to TRUE
+    IF OLD.after_prove = FALSE AND NEW.after_prove = TRUE THEN
+        -- Explicitly set update_ready_at to the current UTC timestamp
+        UPDATE sectors_snap_pipeline SET update_ready_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
+        WHERE sp_id = NEW.sp_id AND sector_number = NEW.sector_number;
+    END IF;
+
+    -- Return the modified row
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER update_update_ready_at
+    AFTER INSERT OR UPDATE OR DELETE ON sectors_snap_pipeline
+    FOR EACH ROW EXECUTE FUNCTION set_update_ready_at();

tasks/seal/task_submit_commit.go

@@ -274,7 +274,7 @@ func (s *SubmitCommitTask) Do(taskID harmonytask.TaskID, stillOwned func() bool)
 						return false, xerrors.Errorf("allocation check failed with an unrecoverable issue: %w", multierr.Combine(err, err2))
 					}
 					log.Errorw("allocation check failed with an unrecoverable issue", "sp", sectorParams.SpID, "sector", sectorParams.SectorNumber, "err", err)
-					sectorFailed = false
+					sectorFailed = true
 					break
 				}
 			}
@@ -286,7 +286,7 @@ func (s *SubmitCommitTask) Do(taskID harmonytask.TaskID, stillOwned func() bool)
 		}
 
 		if sectorFailed {
-			break
+			continue // Skip this sector
 		}
 
 		ssize, err := pci.Info.SealProof.SectorSize()