Improve legacy sync behaviour and logs

services/blockvalidation/Server.go

@@ -1161,7 +1161,7 @@ func (u *Server) ValidateBlock(ctx context.Context, request *blockvalidation_api
 	defer deferFn()
 
 	// Wait for block assembly to be ready before processing the block
-	if err = blockassemblyutil.WaitForBlockAssemblyReady(ctx, u.logger, u.blockAssemblyClient, block.Height); err != nil {
+	if err = blockassemblyutil.WaitForBlockAssemblyReady(ctx, u.logger, u.blockAssemblyClient, block.Height, u.settings.BlockValidation.MaxBlocksBehindBlockAssembly); err != nil {
 		// block-assembly is still behind, so we cannot process this block
 		return nil, errors.WrapGRPC(err)
 	}
@@ -1275,7 +1275,7 @@ func (u *Server) processBlockFound(ctx context.Context, hash *chainhash.Hash, ba
 	}
 
 	// Wait for block assembly to be ready before processing the block
-	if err = blockassemblyutil.WaitForBlockAssemblyReady(ctx, u.logger, u.blockAssemblyClient, block.Height); err != nil {
+	if err = blockassemblyutil.WaitForBlockAssemblyReady(ctx, u.logger, u.blockAssemblyClient, block.Height, u.settings.BlockValidation.MaxBlocksBehindBlockAssembly); err != nil {
 		// block-assembly is still behind, so we cannot process this block
 		return err
 	}

services/blockvalidation/catchup.go

@@ -791,7 +791,7 @@ func (u *Server) validateBlocksOnChannel(validateBlocksChan chan *model.Block, g
 			}
 
 			// Wait for block assembly to be ready if needed
-			if err := blockassemblyutil.WaitForBlockAssemblyReady(gCtx, u.logger, u.blockAssemblyClient, block.Height); err != nil {
+			if err := blockassemblyutil.WaitForBlockAssemblyReady(gCtx, u.logger, u.blockAssemblyClient, block.Height, u.settings.BlockValidation.MaxBlocksBehindBlockAssembly); err != nil {
 				return errors.NewProcessingError("[catchup:validateBlocksOnChannel][%s] failed to wait for block assembly for block %s: %v", blockUpTo.Hash().String(), block.Hash().String(), err)
 			}
 

services/legacy/netsync/handle_block.go

@@ -109,7 +109,7 @@ func (sm *SyncManager) HandleBlockDirect(ctx context.Context, peer *peer.Peer, b
 	}()
 
 	// Wait for block assembly to be ready
-	if err = blockassemblyutil.WaitForBlockAssemblyReady(ctx, sm.logger, sm.blockAssembly, blockHeight); err != nil {
+	if err = blockassemblyutil.WaitForBlockAssemblyReady(ctx, sm.logger, sm.blockAssembly, blockHeight, sm.settings.BlockValidation.MaxBlocksBehindBlockAssembly); err != nil {
 		// block-assembly is still behind, so we cannot process this block
 		return err
 	}

services/legacy/netsync/manager.go

@@ -649,16 +649,21 @@ func (sm *SyncManager) handleCheckSyncPeer() {
 	validNetworkSpeed := sm.syncPeerState.validNetworkSpeed(sm.minSyncPeerNetworkSpeed)
 	lastBlockSince := time.Since(sm.syncPeerState.getLastBlockTime())
 
-	sm.logger.Debugf("[CheckSyncPeer] sync peer %s check, network violations: %v (limit %v), time since last block: %v (limit %v)", sm.syncPeer.String(), validNetworkSpeed, maxNetworkViolations, lastBlockSince, maxLastBlockTime)
+	sm.logger.Debugf("[CheckSyncPeer] sync peer %s check, network violations: %v (limit %v), time since last block: %v (limit %v), headers-first mode: %v", sm.syncPeer.String(), validNetworkSpeed, maxNetworkViolations, lastBlockSince, maxLastBlockTime, sm.headersFirstMode)
+
+	// Don't check network speed during headers-first mode, as we're intentionally
+	// downloading small headers (80 bytes each) rather than full blocks. The peer
+	// may appear slow because we're not requesting much data, not because it's actually slow.
+	isNetworkSpeedViolation := !sm.headersFirstMode && (validNetworkSpeed >= maxNetworkViolations)
 
 	// Check network speed of the sync peer and its last block time. If we're currently
 	// flushing the cache skip this round.
-	if (validNetworkSpeed < maxNetworkViolations) && (lastBlockSince <= maxLastBlockTime) {
+	if !isNetworkSpeedViolation && (lastBlockSince <= maxLastBlockTime) {
 		return
 	}
 
 	var reason string
-	if validNetworkSpeed >= maxNetworkViolations {
+	if isNetworkSpeedViolation {
 		reason = "network speed violation"
 	} else if lastBlockSince > maxLastBlockTime {
 		reason = "last block time out of range"
@@ -1129,8 +1134,6 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockQueueMsg) error {
 			// TODO TEMPORARY: we should not panic here, but return the error
 			panic(err)
 		}
-	} else {
-		sm.logger.Infof("accepted block %v", bmsg.blockHash)
 	}
 
 	// Meta-data about the new block this peer is reporting. We use this
@@ -1177,6 +1180,8 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockQueueMsg) error {
 		}
 	}
 
+	sm.logger.Infof("accepted block %v at height %d", bmsg.blockHash, heightUpdate)
+
 	// Clear the rejected transactions.
 	sm.rejectedTxns.Clear()
 

settings/interface.go

@@ -281,6 +281,7 @@ type BlockValidationSettings struct {
 	CheckSubtreeFromBlockRetryBackoffDuration time.Duration
 	SecretMiningThreshold                     uint32
 	PreviousBlockHeaderCount                  uint64
+	MaxBlocksBehindBlockAssembly              int
 	// Catchup configuration
 	CatchupMaxRetries            int // Maximum number of retries for catchup operations
 	CatchupIterationTimeout      int // Timeout in seconds for each catchup iteration

settings/settings.go

@@ -272,6 +272,7 @@ func NewSettings(alternativeContext ...string) *Settings {
 			CheckSubtreeFromBlockRetryBackoffDuration: getDuration("blockvalidation_check_subtree_from_block_retry_backoff_duration", 30*time.Second),
 			SecretMiningThreshold:                     getUint32("blockvalidation_secret_mining_threshold", uint32(params.CoinbaseMaturity-1), alternativeContext...), // golint:nolint
 			PreviousBlockHeaderCount:                  getUint64("blockvalidation_previous_block_header_count", 100, alternativeContext...),
+			MaxBlocksBehindBlockAssembly:              getInt("blockvalidation_maxBlocksBehindBlockAssembly", 20, alternativeContext...),
 			// Catchup configuration
 			CatchupMaxRetries:            getInt("blockvalidation_catchup_max_retries", 3, alternativeContext...),
 			CatchupIterationTimeout:      getInt("blockvalidation_catchup_iteration_timeout", 30, alternativeContext...),

util/blockassemblyutil/blockassembly_wait.go

@@ -16,8 +16,8 @@ import (
 // a block at the given height. This ensures that all necessary data (such as coinbase
 // transactions) has been processed before allowing block validation to proceed.
 //
-// The function implements a retry mechanism with exponential backoff, checking if the
-// block assembly service is at most 1 block behind the target height. This prevents the
+// The function implements a retry mechanism with linear backoff, checking if the
+// block assembly service is not too far behind the target height. This prevents the
 // blockchain state from running too far ahead of block assembly, which would cause
 // coinbase maturity checks to fail incorrectly in the UTXO store.
 //
@@ -26,6 +26,7 @@ import (
 //   - logger: Logger for recording operations
 //   - blockAssemblyClient: Client interface to the block assembly service
 //   - blockHeight: The height of the block to be processed
+//   - maxBlocksBehind: Maximum number of blocks block assembly can be behind
 //
 // Returns:
 //   - error: nil if block assembly is ready, error if timeout or other failure
@@ -34,33 +35,37 @@ func WaitForBlockAssemblyReady(
 	logger ulogger.Logger,
 	blockAssemblyClient blockassembly.ClientI,
 	blockHeight uint32,
+	maxBlocksBehind int,
 ) error {
 	// Skip if block assembly client is not available (e.g., in tests)
 	if blockAssemblyClient == nil {
 		return nil
 	}
 
-	// Block assembly must be at most 1 block behind to prevent UTXO store
-	// coinbase maturity checks from failing with incorrect "current height"
-	const maxBlocksBehind uint32 = 1
+	// Validate maxBlocksBehind parameter to prevent uint32 wraparound
+	if maxBlocksBehind < 0 {
+		return errors.NewInvalidArgumentError("maxBlocksBehind must be non-negative, got %d", maxBlocksBehind)
+	}
 
-	// Check that block assembly is not more than 1 block behind
-	// This is to make sure all the coinbases have been processed in the block assembly
+	// Check that block assembly is not more than maxBlocksBehind blocks behind.
+	// We allow block assembly to run slightly behind as a performance optimization, but must ensure
+	// it stays within the coinbase maturity window to prevent block assembly state resets.
+	// This ensures all coinbase transactions have been properly processed before validation proceeds.
 	_, err := retry.Retry(ctx, logger, func() (uint32, error) {
 		blockAssemblyStatus, err := blockAssemblyClient.GetBlockAssemblyState(ctx)
 		if err != nil {
 			return 0, errors.NewProcessingError("failed to get block assembly state", err)
 		}
 
-		if blockAssemblyStatus.CurrentHeight+maxBlocksBehind < blockHeight {
+		if blockAssemblyStatus.CurrentHeight+uint32(maxBlocksBehind) < blockHeight {
 			return 0, errors.NewProcessingError("block assembly is behind, block height %d, block assembly height %d", blockHeight, blockAssemblyStatus.CurrentHeight)
 		}
 
 		return blockAssemblyStatus.CurrentHeight, nil
 	},
 		retry.WithRetryCount(100),
-		retry.WithBackoffDurationType(time.Millisecond),
-		retry.WithBackoffMultiplier(10),
+		retry.WithBackoffDurationType(20*time.Millisecond),
+		retry.WithBackoffMultiplier(4),
 		retry.WithMessage(fmt.Sprintf("[WaitForBlockAssemblyReady] block assembly block height %d is behind, waiting", blockHeight)),
 	)
 

util/blockassemblyutil/blockassembly_wait_test.go

@@ -52,19 +52,19 @@ func TestWaitForBlockAssemblyReady(t *testing.T) {
 			blockHeight: 100,
 			blockHash:   "test-hash",
 			setupMock: func(m *blockassembly.Mock) {
-				// First call - behind
+				// First call - behind (CurrentHeight: 88 + maxBlocksBehind: 10 = 98, which is < 100)
 				m.On("GetBlockAssemblyState", mock.Anything).Return(
-					&blockassembly_api.StateMessage{CurrentHeight: 98},
+					&blockassembly_api.StateMessage{CurrentHeight: 88},
 					nil,
 				).Once()
 				// Second call - still behind
 				m.On("GetBlockAssemblyState", mock.Anything).Return(
-					&blockassembly_api.StateMessage{CurrentHeight: 98},
+					&blockassembly_api.StateMessage{CurrentHeight: 88},
 					nil,
 				).Once()
-				// Third call - caught up
+				// Third call - caught up (88 + 10 = 98, still behind, so need 90+)
 				m.On("GetBlockAssemblyState", mock.Anything).Return(
-					&blockassembly_api.StateMessage{CurrentHeight: 99},
+					&blockassembly_api.StateMessage{CurrentHeight: 90},
 					nil,
 				).Once()
 			},
@@ -75,9 +75,9 @@ func TestWaitForBlockAssemblyReady(t *testing.T) {
 			blockHeight: 100,
 			blockHash:   "test-hash",
 			setupMock: func(m *blockassembly.Mock) {
-				// Always return behind height
+				// Always return behind height (CurrentHeight: 88 + maxBlocksBehind: 10 = 98, which is < 100)
 				m.On("GetBlockAssemblyState", mock.Anything).Return(
-					&blockassembly_api.StateMessage{CurrentHeight: 98},
+					&blockassembly_api.StateMessage{CurrentHeight: 88},
 					nil,
 				)
 			},
@@ -128,6 +128,7 @@ func TestWaitForBlockAssemblyReady(t *testing.T) {
 				logger,
 				mockClient,
 				tt.blockHeight,
+				10, // maxBlocksBehind
 			)
 
 			if tt.expectedError {
@@ -155,9 +156,9 @@ func TestWaitForBlockAssemblyReady(t *testing.T) {
 func TestWaitForBlockAssemblyReadyContextCancellation(t *testing.T) {
 	mockClient := &blockassembly.Mock{}
 
-	// Make block assembly always behind
+	// Make block assembly always behind (CurrentHeight: 88 + maxBlocksBehind: 10 = 98, which is < 100)
 	mockClient.On("GetBlockAssemblyState", mock.Anything).Return(
-		&blockassembly_api.StateMessage{CurrentHeight: 98},
+		&blockassembly_api.StateMessage{CurrentHeight: 88},
 		nil,
 	)
 
@@ -171,6 +172,7 @@ func TestWaitForBlockAssemblyReadyContextCancellation(t *testing.T) {
 		logger,
 		mockClient,
 		100,
+		10, // maxBlocksBehind
 	)
 
 	assert.Error(t, err)

util/retry/retry.go

@@ -78,8 +78,14 @@ func Retry[T any](ctx context.Context, logger ulogger.Logger, f func() (T, error
 			}
 
 			// Log the retry message with wait time
-			logger.Warnf(setOptions.Message+" (attempt %d): %v, trying again in %.1f seconds",
-				i+1, err, waitTime.Seconds())
+			// Use DEBUG level for first 5 attempts to reduce noise in logs during normal transient failures,
+			// then switch to WARN level for later attempts to highlight persistent issues that need attention
+			logMessage := setOptions.Message + " (attempt %d): %v, trying again in %.1f seconds"
+			if i < 5 {
+				logger.Debugf(logMessage, i+1, err, waitTime.Seconds())
+			} else {
+				logger.Warnf(logMessage, i+1, err, waitTime.Seconds())
+			}
 
 			// Wait before retrying
 			if setOptions.ExponentialBackoff {