diff --git a/cmd/topf/apply.go b/cmd/topf/apply.go index 636c200..162fd0b 100644 --- a/cmd/topf/apply.go +++ b/cmd/topf/apply.go @@ -11,6 +11,7 @@ import ( "maps" "slices" "strings" + "time" "github.com/postfinance/topf/internal/cmd/apply" "github.com/postfinance/topf/internal/nodepool" @@ -55,6 +56,12 @@ func newApplyCmd() *cli.Command { Sources: cli.EnvVars("TOPF_ALLOW_NOT_READY"), DefaultText: defaultTextFalse, }, + &cli.DurationFlag{ + Name: "stabilization-duration", + Usage: "how long a node must stay ready after applying before it is considered stable", + Value: time.Second * 30, + Sources: cli.EnvVars("TOPF_STABILIZATION_DURATION"), + }, &cli.StringFlag{ Name: "mode", Value: "auto", @@ -83,13 +90,14 @@ func newApplyCmd() *cli.Command { } err = apply.Execute(ctx, t, apply.Options{ - DryRun: c.Bool("dry-run"), - AutoBootstrap: c.Bool("auto-bootstrap"), - SkipProblematicNodes: c.Bool("skip-problematic-nodes"), - SkipPostApplyChecks: c.Bool("skip-post-apply-checks"), - AllowNotReady: c.Bool("allow-not-ready"), - Mode: mode, - MaxParallel: maxParallel, + DryRun: c.Bool("dry-run"), + AutoBootstrap: c.Bool("auto-bootstrap"), + SkipProblematicNodes: c.Bool("skip-problematic-nodes"), + SkipPostApplyChecks: c.Bool("skip-post-apply-checks"), + StabilizationDuration: c.Duration("stabilization-duration"), + AllowNotReady: c.Bool("allow-not-ready"), + Mode: mode, + MaxParallel: maxParallel, }) if errors.Is(err, topf.ErrDryRunChangesDetected) { return cli.Exit(err.Error(), 2) diff --git a/cmd/topf/upgrade.go b/cmd/topf/upgrade.go index 5c3fc77..31ad2a0 100644 --- a/cmd/topf/upgrade.go +++ b/cmd/topf/upgrade.go @@ -11,6 +11,7 @@ import ( "maps" "slices" "strings" + "time" "github.com/postfinance/topf/internal/cmd/upgrade" "github.com/postfinance/topf/internal/nodepool" @@ -63,6 +64,12 @@ func newUpgradeCmd() *cli.Command { Sources: cli.EnvVars("TOPF_DELETE_IF_EVICTION_FAILS"), DefaultText: defaultTextFalse, }, + &cli.DurationFlag{ + Name: "stabilization-duration", + Usage: "how long a node must stay ready after rebooting before it is considered stable", + Value: time.Second * 30, + Sources: cli.EnvVars("TOPF_STABILIZATION_DURATION"), + }, &cli.BoolFlag{ Name: "force", Usage: "skip etcd health checks during upgrade; only applies to nodes running Talos < 1.13 (legacy MachineService.Upgrade RPC); has no effect on Talos >= 1.13, where the LifecycleService.Upgrade RPC validates etcd health server-side", @@ -111,6 +118,7 @@ func newUpgradeCmd() *cli.Command { Force: c.Bool("force"), Drain: c.Bool("drain"), DrainTimeout: c.Duration("drain-timeout"), + StabilizationDuration: c.Duration("stabilization-duration"), DeleteIfEvictionFails: c.Bool("delete-if-eviction-fails"), Stage: c.Bool("stage"), StageLabels: c.StringSlice("stage-label"), diff --git a/docs/commands/apply.md b/docs/commands/apply.md index 8520d05..0ccdedc 100644 --- a/docs/commands/apply.md +++ b/docs/commands/apply.md @@ -20,10 +20,10 @@ The `apply` command is the primary way to apply configuration changes to a runni - Dry-run apply to check for changes - If changes detected in `--dry-run` mode: print diff and **exit with code 2** - If changes detected in normal mode: - - Show diff (if `--confirm` enabled, see [global flags](../configuration.md#global-flags)) - - Ask for confirmation (if `--confirm` enabled) - - Apply configuration - - If config applied AND not `--skip-post-apply-checks`: Stabilize (wait 30s for node to be ready) + - Show diff (if `--confirm` enabled, see [global flags](../configuration.md#global-flags)) + - Ask for confirmation (if `--confirm` enabled) + - Apply configuration + - If config applied AND not `--skip-post-apply-checks`: Stabilize (wait `--stabilization-duration`, default 30s, for node to be ready) 1. **Bootstrap** (if `--auto-bootstrap` enabled): - Select first control plane node @@ -40,7 +40,8 @@ All flags can also be set via environment variables using the `TOPF_` prefix and | `--mode` | `auto` | Apply mode: `auto`, `reboot`, `no-reboot`, `staged`, `try` | | `--auto-bootstrap` | `false` | Automatically bootstrap ETCD after applying configurations | | `--skip-problematic-nodes` | `false` | Continue with healthy nodes if some fail pre-flight checks | -| `--skip-post-apply-checks` | `false` | Skip the 30-second stabilization check after applying configs | +| `--skip-post-apply-checks` | `false` | Skip the post-apply stabilization check after applying configs | +| `--stabilization-duration` | `30s` | How long a node must stay ready after applying before it is considered stable | | `--allow-not-ready` | `false` | Allow applying to nodes that are not ready (have unmet conditions) | | [`--nodes-filter`](../configuration.md#filtering-nodes) | - | Regex pattern to filter which nodes to operate on (global flag) | | [`--redact`](../configuration.md#redacting-sensitive-output) | `true` | Redact Talos secrets, certificates, SOPS-encrypted values, and vals-resolved values from output (global flag) | @@ -73,6 +74,9 @@ topf apply --skip-problematic-nodes # Apply without waiting for nodes to stabilize topf apply --skip-post-apply-checks +# Apply with a custom stabilization duration +topf apply --stabilization-duration=1m + # Apply to nodes even if they have unmet conditions topf apply --allow-not-ready ``` @@ -87,7 +91,7 @@ The apply command validates each node before attempting to apply configuration: ## Post-apply Stabilization -After applying configuration to a node, the command waits up to 30 seconds for the node to: +After applying configuration to a node, the command waits for the node to stay ready for `--stabilization-duration` (default: 30 seconds) with: - Report as ready - Have no unmet conditions diff --git a/docs/commands/upgrade.md b/docs/commands/upgrade.md index 1af9241..11dd8b9 100644 --- a/docs/commands/upgrade.md +++ b/docs/commands/upgrade.md @@ -13,6 +13,7 @@ All flags can also be set via environment variables using the `TOPF_` prefix and | `--reboot-mode` | `default` | Reboot mode during upgrade: `default` uses kexec, `powercycle` does a full reboot | | `--drain` | `true` | Cordon and drain the Kubernetes node before rebooting, then uncordon after stabilization *(modern flow only; ignored on legacy nodes, where Talos drains and uncordons server-side)* | | `--drain-timeout` | `5m` | Maximum time to wait for pod evictions (and, with `--delete-if-eviction-fails`, deletions) to complete during drain *(modern flow only)* | +| `--stabilization-duration` | `30s` | How long a node must stay ready after rebooting before it is considered stable | | `--delete-if-eviction-fails` | `false` | If graceful drain fails (e.g. a PodDisruptionBudget blocks eviction), retry by deleting pods directly (DELETE instead of EVICT, bypassing PDBs); reuses `--drain-timeout` for the delete fallback *(modern flow only)* | | `--force` | `false` | Skip etcd health checks; only applies to nodes running Talos < 1.13 (legacy `MachineService.Upgrade` RPC); has no effect on Talos >= 1.13, where the `LifecycleService.Upgrade` RPC validates etcd health server-side | | `--stage` | `false` | Install upgrade artifacts without rebooting; the node is left running and can be labeled/annotated/tainted (see `--stage-label`/`--stage-annotation`/`--stage-taint`) so an external controller or human reboots it later | @@ -78,13 +79,13 @@ All flags can also be set via environment variables using the `TOPF_` prefix and 1. **If `--stage` is set**: apply labels/annotations/taints (if any) and stop here — the node is not rebooted 1. Cordon and drain the Kubernetes node if `--drain` is enabled. **If the drain fails** (e.g. a pod cannot be evicted within `--drain-timeout`), the upgrade aborts unless `--delete-if-eviction-fails` is set: in that case, the drain retries with pod deletion (DELETE instead of EVICT, bypassing PodDisruptionBudgets, reusing `--drain-timeout`). If the forced drain also fails, the node is left cordoned with the new artifacts installed but not rebooted, and no further nodes are upgraded. In-flight upgrades on other nodes (when `--max-parallel > 1`) are allowed to complete, but no new ones are started. The node must be uncordoned and rebooted manually to recover. 1. Issue a `Reboot` with the selected reboot mode (default: kexec) -1. Wait 30 seconds for the node to stabilize +1. Wait `--stabilization-duration` (default: 30 seconds) for the node to stabilize 1. Uncordon the Kubernetes node **Legacy flow** *(Talos < 1.13)*: 1. Issue `MachineService.Upgrade`, which installs the upgrade artifacts, cordons and drains the node, and reboots — all in a single server-side sequence. `--drain` and `--drain-timeout` are ignored (Talos drains and uncordons the node itself); `--force` skips etcd health checks. `--stage` is not supported on legacy nodes. -1. Wait 30 seconds for the node to stabilize +1. Wait `--stabilization-duration` (default: 30 seconds) for the node to stabilize ## Installer Image diff --git a/internal/cmd/apply/apply.go b/internal/cmd/apply/apply.go index e5b5087..9ac664d 100644 --- a/internal/cmd/apply/apply.go +++ b/internal/cmd/apply/apply.go @@ -30,6 +30,9 @@ type Options struct { SkipPostApplyChecks bool // Allow applying to nodes that are not ready (have unmet conditions) AllowNotReady bool + // StabilizationDuration is how long a node must stay ready after + // applying before it is considered stable. + StabilizationDuration time.Duration // Apply mode passed to Talos (auto, reboot, no-reboot, staged, try) Mode machine.ApplyConfigurationRequest_Mode // MaxParallel controls how many worker nodes are applied to concurrently. @@ -195,7 +198,7 @@ func applyNode(ctx context.Context, node *topf.Node, opts Options, logger *slog. return nil } - if err = node.Stabilize(ctx, logger, time.Second*30); err != nil { + if err = node.Stabilize(ctx, logger, opts.StabilizationDuration); err != nil { return fmt.Errorf("node didn't stabilize: %w", err) } diff --git a/internal/cmd/upgrade/upgrade.go b/internal/cmd/upgrade/upgrade.go index 183b838..b9b6c83 100644 --- a/internal/cmd/upgrade/upgrade.go +++ b/internal/cmd/upgrade/upgrade.go @@ -57,6 +57,10 @@ type Options struct { // DrainTimeout is the maximum time to wait for pod evictions to complete. DrainTimeout time.Duration + // StabilizationDuration is how long a node must stay ready after + // rebooting before it is considered stable. + StabilizationDuration time.Duration + // DeleteIfEvictionFails retries the drain with direct pod deletion // (DELETE instead of EVICT, bypassing PDBs) if the graceful drain fails. DeleteIfEvictionFails bool @@ -372,7 +376,7 @@ func upgradeNodeLifecycle(ctx context.Context, t topf.Topf, node *topf.Node, opt logger.Info("reboot initiated") - if err = node.Stabilize(ctx, logger, time.Second*30); err != nil { + if err = node.Stabilize(ctx, logger, opts.StabilizationDuration); err != nil { return fmt.Errorf("node didn't stabilize: %w", err) } @@ -420,7 +424,7 @@ func upgradeNodeLegacy(ctx context.Context, _ topf.Topf, node *topf.Node, opts O logger.Info("upgrade initiated") - if err = node.Stabilize(ctx, logger, time.Second*30); err != nil { + if err = node.Stabilize(ctx, logger, opts.StabilizationDuration); err != nil { return fmt.Errorf("node didn't stabilize: %w", err) }