diff --git a/docker/walrus-antithesis/build-test-config-image/docker-compose.yaml b/docker/walrus-antithesis/build-test-config-image/docker-compose.yaml index 1df37da4e2..fd2ebdd993 100644 --- a/docker/walrus-antithesis/build-test-config-image/docker-compose.yaml +++ b/docker/walrus-antithesis/build-test-config-image/docker-compose.yaml @@ -7,6 +7,7 @@ # - A setup completion service # - 4 Walrus nodes for testing # - 1 stress client for testing +# - 1 cross-node invariant observer services: # Runs a local Sui network with a single validator sui-localnet: @@ -173,6 +174,31 @@ services: command: > /bin/bash -c "sleep 15 && /root/run-staking.sh" + # Cross-node invariant observer — scrapes Prometheus metrics from all storage + # nodes and verifies data consistency across the cluster. Crashes on violation, + # which Antithesis surfaces as a test failure. + walrus-observer: + depends_on: + walrus-deploy: + condition: service_completed_successfully + networks: + testbed-network: + ipv4_address: 10.0.0.40 + image: ${WALRUS_IMAGE_NAME} + platform: ${WALRUS_PLATFORM:-linux/amd64} + hostname: walrus-observer + container_name: walrus-observer + environment: + - NO_COLOR=1 + - CHECK_INTERVAL=60 + - INITIAL_WAIT=180 + - EVENT_SOURCE_PATIENCE=3 + - FULLY_STORED_PATIENCE=3 + volumes: + - ./files/run-observer.sh:/root/run-observer.sh + command: > + /bin/bash -c "/root/run-observer.sh" + # Persistent volumes for sharing data between containers volumes: sui-bin: diff --git a/docker/walrus-antithesis/build-test-config-image/files/run-observer.sh b/docker/walrus-antithesis/build-test-config-image/files/run-observer.sh new file mode 100755 index 0000000000..bb34615da8 --- /dev/null +++ b/docker/walrus-antithesis/build-test-config-image/files/run-observer.sh @@ -0,0 +1,240 @@ +#!/bin/bash +# Copyright (c) Walrus Foundation +# SPDX-License-Identifier: Apache-2.0 + +# Cross-node invariant observer for Antithesis testing. +# +# Periodically scrapes Prometheus metrics from all storage nodes and verifies +# cross-node data consistency invariants. Crashes (exits non-zero) when an +# invariant violation is detected, which Antithesis surfaces as a test failure. +# +# Hard invariants (crash on first confirmed violation): +# - walrus_blob_info_consistency_check — same digest per epoch across all nodes +# - walrus_per_object_blob_info_consistency_check — same digest per epoch across all nodes +# +# Soft invariants (crash after persistent violation): +# - walrus_periodic_event_source_for_deterministic_events — same per bucket across all nodes +# - walrus_node_blob_data_fully_stored_ratio — must equal 1 on every node/epoch + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration (override via environment variables) +# --------------------------------------------------------------------------- +NODES=("10.0.0.10" "10.0.0.11" "10.0.0.12" "10.0.0.13") +METRICS_PORT="${METRICS_PORT:-9184}" +CHECK_INTERVAL="${CHECK_INTERVAL:-60}" +INITIAL_WAIT="${INITIAL_WAIT:-180}" +# Consecutive rounds a soft-invariant violation must persist before crashing. +EVENT_SOURCE_PATIENCE="${EVENT_SOURCE_PATIENCE:-3}" +FULLY_STORED_PATIENCE="${FULLY_STORED_PATIENCE:-3}" + +WORK_DIR="/tmp/observer" +mkdir -p "$WORK_DIR" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [observer] $*"; } +die() { log "FATAL: $*" >&2; exit 1; } + +# Scrape the /metrics endpoint of a storage node. +scrape_node() { + local ip="$1" out="$2" + curl -sf --connect-timeout 5 --max-time 10 \ + "http://${ip}:${METRICS_PORT}/metrics" > "$out" 2>/dev/null +} + +# Extract (label_value, metric_value) pairs from a Prometheus scrape file. +# Outputs: label_valuemetric_value (sorted by label). +extract_metric() { + local file="$1" metric="$2" label="$3" + { grep "^${metric}{" "$file" 2>/dev/null || true; } \ + | sed -n "s/^${metric}{.*${label}=\"\([^\"]*\)\".*} \([^ ]*\).*/\1\t\2/p" \ + | sort -t"$(printf '\t')" -k1,1 +} + +# --------------------------------------------------------------------------- +# Cross-node comparison for a labelled gauge metric. +# +# For every label value present in ALL scraped nodes, assert that the metric +# value is identical. Writes per-violation details to a file. +# +# Arguments: metric_name label_name details_file +# Echoes the number of mismatched label values (0 = clean). +# --------------------------------------------------------------------------- +check_cross_node_metric() { + local metric="$1" label="$2" details_file="$3" + local num_nodes=${#NODES[@]} + local violations=0 + + : > "$details_file" + + for i in "${!NODES[@]}"; do + extract_metric "$WORK_DIR/raw_${NODES[$i]}.prom" "$metric" "$label" \ + > "$WORK_DIR/chk_${i}.tsv" + done + + # Labels that appear in every node's output. + for i in "${!NODES[@]}"; do + cut -f1 "$WORK_DIR/chk_${i}.tsv" + done | sort | uniq -c | awk -v n="$num_nodes" '$1 == n { print $2 }' \ + > "$WORK_DIR/chk_common.txt" + + while IFS= read -r lbl; do + [ -z "$lbl" ] && continue + local ref_val="" mismatch=false + + for i in "${!NODES[@]}"; do + local val + val=$(awk -F'\t' -v l="$lbl" '$1 == l { print $2 }' "$WORK_DIR/chk_${i}.tsv") + if [ -z "$ref_val" ]; then + ref_val="$val" + elif [ "$val" != "$ref_val" ]; then + mismatch=true + fi + done + + if [ "$mismatch" = true ]; then + { + echo " ${metric}{${label}=\"${lbl}\"}:" + for i in "${!NODES[@]}"; do + local val + val=$(awk -F'\t' -v l="$lbl" '$1 == l { print $2 }' "$WORK_DIR/chk_${i}.tsv") + echo " node-${i} (${NODES[$i]}): ${val}" + done + } >> "$details_file" + violations=$((violations + 1)) + fi + done < "$WORK_DIR/chk_common.txt" + + echo "$violations" +} + +# --------------------------------------------------------------------------- +# Check walrus_node_blob_data_fully_stored_ratio == 1 for all nodes/epochs. +# Echoes the number of violations. +# --------------------------------------------------------------------------- +check_fully_stored_ratio() { + local metric="walrus_node_blob_data_fully_stored_ratio" + local violations=0 + + for i in "${!NODES[@]}"; do + while IFS="$(printf '\t')" read -r epoch ratio; do + [ -z "$epoch" ] && continue + # Numeric comparison: ratio must equal 1. + if ! awk -v r="$ratio" 'BEGIN { exit (r == 1) ? 0 : 1 }'; then + log " node-${i} (${NODES[$i]}) epoch=${epoch} ratio=${ratio}" >&2 + violations=$((violations + 1)) + fi + done < <(extract_metric "$WORK_DIR/raw_${NODES[$i]}.prom" "$metric" "epoch") + done + + echo "$violations" +} + +# --------------------------------------------------------------------------- +# Main loop +# --------------------------------------------------------------------------- +log "Cross-node invariant observer starting" +log "Nodes: ${NODES[*]}, port: ${METRICS_PORT}" +log "Check interval: ${CHECK_INTERVAL}s, initial wait: ${INITIAL_WAIT}s" +log "Event source patience: ${EVENT_SOURCE_PATIENCE} rounds" +log "Fully stored patience: ${FULLY_STORED_PATIENCE} rounds" + +log "Waiting ${INITIAL_WAIT}s for cluster stabilization..." +sleep "$INITIAL_WAIT" + +event_source_streak=0 +fully_stored_streak=0 +round=0 + +while true; do + round=$((round + 1)) + log "=== Round ${round} ===" + + # ------------------------------------------------------------------ + # Scrape all nodes; skip the round if any node is unreachable. + # ------------------------------------------------------------------ + all_ok=true + for i in "${!NODES[@]}"; do + if scrape_node "${NODES[$i]}" "$WORK_DIR/raw_${NODES[$i]}.prom"; then + log "Scraped node-${i} (${NODES[$i]})" + else + log "Cannot reach node-${i} (${NODES[$i]}), skipping round" + all_ok=false + break + fi + done + if [ "$all_ok" = false ]; then + sleep "$CHECK_INTERVAL" + continue + fi + + # ------------------------------------------------------------------ + # Hard invariant 1: certified blob digest must match across nodes. + # ------------------------------------------------------------------ + v=$(check_cross_node_metric \ + "walrus_blob_info_consistency_check" "epoch" \ + "$WORK_DIR/details_blob_info.txt") + if [ "$v" -gt 0 ]; then + log "INVARIANT VIOLATION — blob_info_consistency_check (${v} epoch(s)):" + cat "$WORK_DIR/details_blob_info.txt" + die "blob_info_consistency_check: mismatched digests across nodes" + fi + log "walrus_blob_info_consistency_check: OK" + + # ------------------------------------------------------------------ + # Hard invariant 2: per-object blob digest must match across nodes. + # ------------------------------------------------------------------ + v=$(check_cross_node_metric \ + "walrus_per_object_blob_info_consistency_check" "epoch" \ + "$WORK_DIR/details_per_object.txt") + if [ "$v" -gt 0 ]; then + log "INVARIANT VIOLATION — per_object_blob_info_consistency_check (${v} epoch(s)):" + cat "$WORK_DIR/details_per_object.txt" + die "per_object_blob_info_consistency_check: mismatched digests across nodes" + fi + log "walrus_per_object_blob_info_consistency_check: OK" + + # ------------------------------------------------------------------ + # Soft invariant 1: event source consistency (tolerate transient lag). + # Nodes may be at different event-processing positions, so bucket + # values can temporarily diverge. Crash only after a persistent + # mismatch across EVENT_SOURCE_PATIENCE consecutive rounds. + # ------------------------------------------------------------------ + v=$(check_cross_node_metric \ + "walrus_periodic_event_source_for_deterministic_events" "bucket" \ + "$WORK_DIR/details_event_source.txt") + if [ "$v" -gt 0 ]; then + event_source_streak=$((event_source_streak + 1)) + log "Event source mismatch (streak: ${event_source_streak}/${EVENT_SOURCE_PATIENCE}):" + cat "$WORK_DIR/details_event_source.txt" + if [ "$event_source_streak" -ge "$EVENT_SOURCE_PATIENCE" ]; then + die "periodic_event_source: persistent mismatch for ${EVENT_SOURCE_PATIENCE} consecutive rounds" + fi + else + event_source_streak=0 + log "walrus_periodic_event_source_for_deterministic_events: OK" + fi + + # ------------------------------------------------------------------ + # Soft invariant 2: all blobs fully stored (tolerate recovery lag). + # A node that just recovered may briefly report < 1 while syncing + # completes. Crash only after FULLY_STORED_PATIENCE rounds. + # ------------------------------------------------------------------ + v=$(check_fully_stored_ratio) + if [ "$v" -gt 0 ]; then + fully_stored_streak=$((fully_stored_streak + 1)) + log "Fully-stored-ratio violation (streak: ${fully_stored_streak}/${FULLY_STORED_PATIENCE})" + if [ "$fully_stored_streak" -ge "$FULLY_STORED_PATIENCE" ]; then + die "node_blob_data_fully_stored_ratio: persistent violation for ${FULLY_STORED_PATIENCE} consecutive rounds" + fi + else + fully_stored_streak=0 + log "walrus_node_blob_data_fully_stored_ratio: OK" + fi + + log "All checks passed for round ${round}" + sleep "$CHECK_INTERVAL" +done