STAC-23599: Restoring VictoriaMetrics (#5)

* STAC-23599: Restoring VictoriaMetrics

Author: viliakov

ARCHITECTURE.md

@@ -20,7 +20,8 @@ stackstate-backup-cli/
 │   ├── root.go              # Root command and global flags
 │   ├── version/             # Version information command
 │   ├── elasticsearch/       # Elasticsearch backup/restore commands
-│   └── stackgraph/          # Stackgraph backup/restore commands
+│   ├── stackgraph/          # Stackgraph backup/restore commands
+│   └── victoriametrics/     # VictoriaMetrics backup/restore commands
 │
 ├── internal/                # Internal packages (Layers 0-3)
 │   ├── foundation/          # Layer 0: Core utilities
@@ -35,7 +36,8 @@ stackstate-backup-cli/
 │   │
 │   ├── orchestration/       # Layer 2: Workflows
 │   │   ├── portforward/     # Port-forwarding orchestration
-│   │   └── scale/           # Deployment scaling workflows
+│   │   ├── scale/           # Deployment/StatefulSet scaling workflows
+│   │   └── restore/         # Restore job orchestration
 │   │
 │   ├── app/                 # Layer 3: Dependency Container
 │   │   └── app.go           # Application context and dependency injection
@@ -62,7 +64,8 @@ stackstate-backup-cli/
 
 **Key Packages**:
 - `cmd/elasticsearch/`: Elasticsearch snapshot/restore commands (configure, list-snapshots, list-indices, restore-snapshot)
-- `cmd/stackgraph/`: Stackgraph backup/restore commands (list, restore)
+- `cmd/stackgraph/`: Stackgraph backup/restore commands (list, restore, check-and-finalize)
+- `cmd/victoriametrics/`: VictoriaMetrics backup/restore commands (list, restore, check-and-finalize)
 - `cmd/version/`: Version information
 
 **Dependency Rules**:
@@ -117,7 +120,8 @@ appCtx.Formatter
 
 **Key Packages**:
 - `portforward/`: Manages Kubernetes port-forwarding lifecycle
-- `scale/`: Deployment scaling workflows with detailed logging
+- `scale/`: Deployment and StatefulSet scaling workflows with detailed logging
+- `restore/`: Restore job orchestration (confirmation, job lifecycle, finalization, resource management)
 
 **Dependency Rules**:
 - ✅ Can import: `internal/foundation/*`, `internal/clients/*`
@@ -167,7 +171,7 @@ appCtx.Formatter
 
 ```
 1. User invokes CLI command
-   └─> cmd/elasticsearch/restore-snapshot.go
+   └─> cmd/victoriametrics/restore.go (or stackgraph/restore.go)
        │
 2. Parse flags and validate input
    └─> Cobra command receives global flags
@@ -177,16 +181,17 @@ appCtx.Formatter
        ├─> internal/clients/k8s/ (K8s client)
        ├─> internal/foundation/config/ (Load from ConfigMap/Secret)
        ├─> internal/clients/s3/ (S3/Minio client)
-       ├─> internal/clients/elasticsearch/ (ES client)
        ├─> internal/foundation/logger/ (Logger)
        └─> internal/foundation/output/ (Formatter)
        │
 4. Execute business logic with injected dependencies
    └─> runRestore(appCtx)
-       ├─> internal/orchestration/scale/ (Scale down)
-       ├─> internal/orchestration/portforward/ (Port-forward)
-       ├─> internal/clients/elasticsearch/ (Restore snapshot)
-       └─> internal/orchestration/scale/ (Scale up)
+       ├─> internal/orchestration/restore/ (User confirmation)
+       ├─> internal/orchestration/scale/ (Scale down StatefulSets)
+       ├─> internal/orchestration/restore/ (Ensure resources: ConfigMaps, Secrets)
+       ├─> internal/clients/k8s/ (Create restore Job)
+       ├─> internal/orchestration/restore/ (Wait for completion & cleanup)
+       └─> internal/orchestration/scale/ (Scale up StatefulSets)
        │
 5. Format and display results
    └─> appCtx.Formatter.PrintTable() or PrintJSON()
@@ -262,15 +267,50 @@ defer close(pf.StopChan)  // Automatic cleanup
 
 ### 5. Scale Down/Up Pattern
 
-Deployments are scaled down before restore operations and scaled up afterward:
+Deployments and StatefulSets are scaled down before restore operations and scaled up afterward:
 
 ```go
 // Example usage
-scaledDeployments, _ := scale.ScaleDown(k8sClient, namespace, selector, log)
-defer scale.ScaleUp(k8sClient, namespace, scaledDeployments, log)
+scaledResources, _ := scale.ScaleDown(k8sClient, namespace, selector, log)
+defer scale.ScaleUpFromAnnotations(k8sClient, namespace, selector, log)
 ```
 
-### 6. Structured Logging
+**Note**: Scaling now supports both Deployments and StatefulSets through a unified interface.
+
+### 6. Restore Orchestration Pattern
+
+Common restore operations are centralized in the `restore` orchestration layer:
+
+```go
+// User confirmation
+if !restore.PromptForConfirmation() {
+return fmt.Errorf("operation cancelled")
+}
+
+// Wait for job completion and cleanup
+restore.PrintWaitingMessage(log, "service-name", jobName, namespace)
+err := restore.WaitAndCleanup(k8sClient, namespace, jobName, log, cleanupPVC)
+
+// Check and finalize background jobs
+err := restore.CheckAndFinalize(restore.CheckAndFinalizeParams{
+K8sClient:     k8sClient,
+Namespace:     namespace,
+JobName:       jobName,
+ServiceName:   "service-name",
+ScaleSelector: config.ScaleDownLabelSelector,
+CleanupPVC:    true,
+WaitForJob:    false,
+Log:           log,
+})
+```
+
+**Benefits**:
+
+- Eliminates duplicate code between Stackgraph and VictoriaMetrics restore commands
+- Consistent user experience across services
+- Centralized job lifecycle management and cleanup
+
+### 7. Structured Logging
 
 All operations use structured logging with consistent levels:
 

README.md

@@ -9,8 +9,9 @@ This CLI tool replaces the legacy Bash-based backup/restore scripts with a singl
 **Current Support:**
 - Elasticsearch snapshots and restores
 - Stackgraph backups and restores
+- VictoriaMetrics backups and restores
 
-**Planned:** VictoriaMetrics, ClickHouse, Configuration backups
+**Planned:** ClickHouse, Configuration backups
 
 ## Installation
 
@@ -112,11 +113,76 @@ sts-backup stackgraph restore --namespace <namespace> [--archive <name> | --late
 **Flags:**
 - `--archive` - Specific archive name to restore (e.g., sts-backup-20210216-0300.graph)
 - `--latest` - Restore from the most recent backup
-- `--force` - Force delete existing data during restore
 - `--background` - Run restore job in background without waiting for completion
+- `--yes, -y` - Skip confirmation prompt
 
 **Note**: Either `--archive` or `--latest` must be specified (mutually exclusive).
 
+#### check-and-finalize
+
+Check the status of a background Stackgraph restore job and clean up resources.
+
+```bash
+sts-backup stackgraph check-and-finalize --namespace <namespace> --job <job-name> [--wait]
+```
+
+**Flags:**
+
+- `--job, -j` - Stackgraph restore job name (required)
+- `--wait, -w` - Wait for job to complete before cleanup
+
+**Use Case**: This command is useful when a restore job was started with `--background` flag or was interrupted (
+Ctrl+C).
+
+### victoriametrics
+
+Manage VictoriaMetrics backups and restores.
+
+#### list
+
+List available VictoriaMetrics backups from S3/Minio.
+
+```bash
+sts-backup victoriametrics list --namespace <namespace>
+```
+
+**Note**: In HA mode, backups from both instances (victoria-metrics-0 and victoria-metrics-1) are listed. The restore
+command accepts either backup to restore both instances.
+
+#### restore
+
+Restore VictoriaMetrics from a backup archive. Automatically scales down affected StatefulSets before restore and scales
+them back up afterward.
+
+```bash
+sts-backup victoriametrics restore --namespace <namespace> [--archive <name> | --latest] [flags]
+```
+
+**Flags:**
+
+- `--archive` - Specific backup name to restore (e.g., sts-victoria-metrics-backup/victoria-metrics-0-20251030143500)
+- `--latest` - Restore from the most recent backup
+- `--background` - Run restore job in background without waiting for completion
+- `--yes, -y` - Skip confirmation prompt
+
+**Note**: Either `--archive` or `--latest` must be specified (mutually exclusive).
+
+#### check-and-finalize
+
+Check the status of a background VictoriaMetrics restore job and clean up resources.
+
+```bash
+sts-backup victoriametrics check-and-finalize --namespace <namespace> --job <job-name> [--wait]
+```
+
+**Flags:**
+
+- `--job, -j` - VictoriaMetrics restore job name (required)
+- `--wait, -w` - Wait for job to complete before cleanup
+
+**Use Case**: This command is useful when a restore job was started with `--background` flag or was interrupted (
+Ctrl+C).
+
 ## Configuration
 
 The CLI uses configuration from Kubernetes ConfigMaps and Secrets with the following precedence:
@@ -194,9 +260,14 @@ See [internal/foundation/config/testdata/validConfigMapConfig.yaml](internal/fou
 │   │   ├── list-indices.go       # List indices
 │   │   ├── list-snapshots.go     # List snapshots
 │   │   └── restore-snapshot.go   # Restore snapshot
-│   └── stackgraph/               # Stackgraph subcommands
+│   ├── stackgraph/               # Stackgraph subcommands
+│   │   ├── list.go               # List backups
+│   │   ├── restore.go            # Restore backup
+│   │   └── check-and-finalize.go # Check and finalize restore job
+│   └── victoriametrics/          # VictoriaMetrics subcommands
 │       ├── list.go               # List backups
-│       └── restore.go            # Restore backup
+│       ├── restore.go            # Restore backup
+│       └── check-and-finalize.go # Check and finalize restore job
 ├── internal/                     # Internal packages (Layers 0-3)
 │   ├── foundation/               # Layer 0: Core utilities
 │   │   ├── config/               # Configuration management
@@ -208,7 +279,12 @@ See [internal/foundation/config/testdata/validConfigMapConfig.yaml](internal/fou
 │   │   └── s3/                   # S3/Minio client
 │   ├── orchestration/            # Layer 2: Workflows
 │   │   ├── portforward/          # Port-forwarding lifecycle
-│   │   └── scale/                # Deployment scaling
+│   │   ├── scale/                # Deployment/StatefulSet scaling
+│   │   └── restore/              # Restore job orchestration
+│   │       ├── confirmation.go   # User confirmation prompts
+│   │       ├── finalize.go       # Job status check and cleanup
+│   │       ├── job.go            # Job lifecycle management
+│   │       └── resources.go      # Restore resource management
 │   ├── app/                      # Layer 3: Dependency container
 │   │   └── app.go                # Application context and DI
 │   └── scripts/                  # Embedded bash scripts

cmd/elasticsearch/list_snapshots_test.go

@@ -49,6 +49,26 @@ stackgraph:
           memory: "2Gi"
     pvc:
       size: "10Gi"
+victoriaMetrics:
+  S3Locations:
+    - bucket: vm-backup
+      prefix: victoria-metrics-0
+    - bucket: vm-backup
+      prefix: victoria-metrics-1
+  restore:
+    haMode: "mirror"
+    persistentVolumeClaimPrefix: "database-victoria-metrics-"
+    scaleDownLabelSelector: "app=victoria-metrics"
+    job:
+      image: vm-backup:latest
+      waitImage: wait:latest
+      resources:
+        limits:
+          cpu: "1"
+          memory: "2Gi"
+        requests:
+          cpu: "500m"
+          memory: "1Gi"
 `
 
 // mockESClient is a simple mock for testing commands

cmd/root.go

@@ -7,6 +7,7 @@ import (
 	"github.com/stackvista/stackstate-backup-cli/cmd/elasticsearch"
 	"github.com/stackvista/stackstate-backup-cli/cmd/stackgraph"
 	"github.com/stackvista/stackstate-backup-cli/cmd/version"
+	"github.com/stackvista/stackstate-backup-cli/cmd/victoriametrics"
 	"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
 )
 
@@ -39,6 +40,10 @@ func init() {
 	addBackupConfigFlags(stackgraphCmd)
 	rootCmd.AddCommand(stackgraphCmd)
 
+	victoriaMetricsCmd := victoriametrics.Cmd(flags)
+	addBackupConfigFlags(victoriaMetricsCmd)
+	rootCmd.AddCommand(victoriaMetricsCmd)
+
 	// Add commands that don't need backup config flags
 	rootCmd.AddCommand(version.Cmd())
 }