Count cluster client timeouts

CHANGELOG.md

@@ -2,6 +2,7 @@
 ### Implementation changes and bug fixes
 - [PR #1884](https://github.com/rqlite/rqlite/pull/1884): Refactor code to further use common utilities.
 - [PR #1887](https://github.com/rqlite/rqlite/pull/1887): Improve stats and error message for Queued Wait timeouts.
+- [PR #1888](https://github.com/rqlite/rqlite/pull/1888): Count cluster client read and write timeouts
 
 ## v8.30.1 (September 9th 2024)
 ### Implementation changes and bug fixes

cluster/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"os"
 	"sync"
 	"time"
 
@@ -533,6 +534,9 @@ func writeCommand(conn net.Conn, c *proto.Command, timeout time.Duration) error
 	binary.LittleEndian.PutUint64(b[0:], uint64(len(p)))
 	_, err = conn.Write(b)
 	if err != nil {
+		if errors.Is(err, os.ErrDeadlineExceeded) {
+			stats.Add(numClientWriteTimeouts, 1)
+		}
 		return fmt.Errorf("write length: %w", err)
 	}
 	// Write actual protobuf.
@@ -541,6 +545,9 @@ func writeCommand(conn net.Conn, c *proto.Command, timeout time.Duration) error
 	}
 	_, err = conn.Write(p)
 	if err != nil {
+		if errors.Is(err, os.ErrDeadlineExceeded) {
+			stats.Add(numClientWriteTimeouts, 1)
+		}
 		return fmt.Errorf("write protobuf bytes: %w", err)
 	}
 	return nil
@@ -562,6 +569,9 @@ func readResponse(conn net.Conn, timeout time.Duration) (buf []byte, retErr erro
 	b := make([]byte, protoBufferLengthSize)
 	_, err := io.ReadFull(conn, b)
 	if err != nil {
+		if errors.Is(err, os.ErrDeadlineExceeded) {
+			stats.Add(numClientReadTimeouts, 1)
+		}
 		return nil, fmt.Errorf("read protobuf length: %w", err)
 	}
 	sz := binary.LittleEndian.Uint64(b[0:])
@@ -573,6 +583,9 @@ func readResponse(conn net.Conn, timeout time.Duration) (buf []byte, retErr erro
 	}
 	_, err = io.ReadFull(conn, p)
 	if err != nil {
+		if errors.Is(err, os.ErrDeadlineExceeded) {
+			stats.Add(numClientReadTimeouts, 1)
+		}
 		return nil, fmt.Errorf("read protobuf bytes: %w", err)
 	}
 	return p, nil

cluster/service.go

@@ -41,6 +41,8 @@ const (
 	numClientExecuteRetries     = "num_client_execute_retries"
 	numClientQueryRetries       = "num_client_query_retries"
 	numClientRequestRetries     = "num_client_request_retries"
+	numClientReadTimeouts       = "num_client_read_timeouts"
+	numClientWriteTimeouts      = "num_client_write_timeouts"
 
 	// Client stats for this package.
 	numGetNodeAPIRequestLocal = "num_get_node_api_req_local"
@@ -73,6 +75,8 @@ func init() {
 	stats.Add(numClientExecuteRetries, 0)
 	stats.Add(numClientQueryRetries, 0)
 	stats.Add(numClientRequestRetries, 0)
+	stats.Add(numClientReadTimeouts, 0)
+	stats.Add(numClientWriteTimeouts, 0)
 }
 
 // Dialer is the interface dialers must implement.