Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions cmd/topf/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"maps"
"slices"
"strings"
"time"

"github.com/postfinance/topf/internal/cmd/apply"
"github.com/postfinance/topf/internal/nodepool"
Expand Down Expand Up @@ -55,6 +56,12 @@ func newApplyCmd() *cli.Command {
Sources: cli.EnvVars("TOPF_ALLOW_NOT_READY"),
DefaultText: defaultTextFalse,
},
&cli.DurationFlag{
Name: "stabilization-duration",
Usage: "how long a node must stay ready after applying before it is considered stable",
Value: time.Second * 30,
Sources: cli.EnvVars("TOPF_STABILIZATION_DURATION"),
},
&cli.StringFlag{
Name: "mode",
Value: "auto",
Expand Down Expand Up @@ -83,13 +90,14 @@ func newApplyCmd() *cli.Command {
}

err = apply.Execute(ctx, t, apply.Options{
DryRun: c.Bool("dry-run"),
AutoBootstrap: c.Bool("auto-bootstrap"),
SkipProblematicNodes: c.Bool("skip-problematic-nodes"),
SkipPostApplyChecks: c.Bool("skip-post-apply-checks"),
AllowNotReady: c.Bool("allow-not-ready"),
Mode: mode,
MaxParallel: maxParallel,
DryRun: c.Bool("dry-run"),
AutoBootstrap: c.Bool("auto-bootstrap"),
SkipProblematicNodes: c.Bool("skip-problematic-nodes"),
SkipPostApplyChecks: c.Bool("skip-post-apply-checks"),
StabilizationDuration: c.Duration("stabilization-duration"),
AllowNotReady: c.Bool("allow-not-ready"),
Mode: mode,
MaxParallel: maxParallel,
})
if errors.Is(err, topf.ErrDryRunChangesDetected) {
return cli.Exit(err.Error(), 2)
Expand Down
8 changes: 8 additions & 0 deletions cmd/topf/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"maps"
"slices"
"strings"
"time"

"github.com/postfinance/topf/internal/cmd/upgrade"
"github.com/postfinance/topf/internal/nodepool"
Expand Down Expand Up @@ -63,6 +64,12 @@ func newUpgradeCmd() *cli.Command {
Sources: cli.EnvVars("TOPF_DELETE_IF_EVICTION_FAILS"),
DefaultText: defaultTextFalse,
},
&cli.DurationFlag{
Name: "stabilization-duration",
Usage: "how long a node must stay ready after rebooting before it is considered stable",
Value: time.Second * 30,
Sources: cli.EnvVars("TOPF_STABILIZATION_DURATION"),
},
&cli.BoolFlag{
Name: "force",
Usage: "skip etcd health checks during upgrade; only applies to nodes running Talos < 1.13 (legacy MachineService.Upgrade RPC); has no effect on Talos >= 1.13, where the LifecycleService.Upgrade RPC validates etcd health server-side",
Expand Down Expand Up @@ -111,6 +118,7 @@ func newUpgradeCmd() *cli.Command {
Force: c.Bool("force"),
Drain: c.Bool("drain"),
DrainTimeout: c.Duration("drain-timeout"),
StabilizationDuration: c.Duration("stabilization-duration"),
DeleteIfEvictionFails: c.Bool("delete-if-eviction-fails"),
Stage: c.Bool("stage"),
StageLabels: c.StringSlice("stage-label"),
Expand Down
16 changes: 10 additions & 6 deletions docs/commands/apply.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ The `apply` command is the primary way to apply configuration changes to a runni
- Dry-run apply to check for changes
- If changes detected in `--dry-run` mode: print diff and **exit with code 2**
- If changes detected in normal mode:
- Show diff (if `--confirm` enabled, see [global flags](../configuration.md#global-flags))
- Ask for confirmation (if `--confirm` enabled)
- Apply configuration
- If config applied AND not `--skip-post-apply-checks`: Stabilize (wait 30s for node to be ready)
- Show diff (if `--confirm` enabled, see [global flags](../configuration.md#global-flags))
- Ask for confirmation (if `--confirm` enabled)
- Apply configuration
- If config applied AND not `--skip-post-apply-checks`: Stabilize (wait `--stabilization-duration`, default 30s, for node to be ready)

1. **Bootstrap** (if `--auto-bootstrap` enabled):
- Select first control plane node
Expand All @@ -40,7 +40,8 @@ All flags can also be set via environment variables using the `TOPF_` prefix and
| `--mode` | `auto` | Apply mode: `auto`, `reboot`, `no-reboot`, `staged`, `try` |
| `--auto-bootstrap` | `false` | Automatically bootstrap ETCD after applying configurations |
| `--skip-problematic-nodes` | `false` | Continue with healthy nodes if some fail pre-flight checks |
| `--skip-post-apply-checks` | `false` | Skip the 30-second stabilization check after applying configs |
| `--skip-post-apply-checks` | `false` | Skip the post-apply stabilization check after applying configs |
| `--stabilization-duration` | `30s` | How long a node must stay ready after applying before it is considered stable |
| `--allow-not-ready` | `false` | Allow applying to nodes that are not ready (have unmet conditions) |
| [`--nodes-filter`](../configuration.md#filtering-nodes) | - | Regex pattern to filter which nodes to operate on (global flag) |
| [`--redact`](../configuration.md#redacting-sensitive-output) | `true` | Redact Talos secrets, certificates, SOPS-encrypted values, and vals-resolved values from output (global flag) |
Expand Down Expand Up @@ -73,6 +74,9 @@ topf apply --skip-problematic-nodes
# Apply without waiting for nodes to stabilize
topf apply --skip-post-apply-checks

# Apply with a custom stabilization duration
topf apply --stabilization-duration=1m

# Apply to nodes even if they have unmet conditions
topf apply --allow-not-ready
```
Expand All @@ -87,7 +91,7 @@ The apply command validates each node before attempting to apply configuration:

## Post-apply Stabilization

After applying configuration to a node, the command waits up to 30 seconds for the node to:
After applying configuration to a node, the command waits for the node to stay ready for `--stabilization-duration` (default: 30 seconds) with:

- Report as ready
- Have no unmet conditions
Expand Down
5 changes: 3 additions & 2 deletions docs/commands/upgrade.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ All flags can also be set via environment variables using the `TOPF_` prefix and
| `--reboot-mode` | `default` | Reboot mode during upgrade: `default` uses kexec, `powercycle` does a full reboot |
| `--drain` | `true` | Cordon and drain the Kubernetes node before rebooting, then uncordon after stabilization *(modern flow only; ignored on legacy nodes, where Talos drains and uncordons server-side)* |
| `--drain-timeout` | `5m` | Maximum time to wait for pod evictions (and, with `--delete-if-eviction-fails`, deletions) to complete during drain *(modern flow only)* |
| `--stabilization-duration` | `30s` | How long a node must stay ready after rebooting before it is considered stable |
| `--delete-if-eviction-fails` | `false` | If graceful drain fails (e.g. a PodDisruptionBudget blocks eviction), retry by deleting pods directly (DELETE instead of EVICT, bypassing PDBs); reuses `--drain-timeout` for the delete fallback *(modern flow only)* |
| `--force` | `false` | Skip etcd health checks; only applies to nodes running Talos < 1.13 (legacy `MachineService.Upgrade` RPC); has no effect on Talos >= 1.13, where the `LifecycleService.Upgrade` RPC validates etcd health server-side |
| `--stage` | `false` | Install upgrade artifacts without rebooting; the node is left running and can be labeled/annotated/tainted (see `--stage-label`/`--stage-annotation`/`--stage-taint`) so an external controller or human reboots it later |
Expand Down Expand Up @@ -78,13 +79,13 @@ All flags can also be set via environment variables using the `TOPF_` prefix and
1. **If `--stage` is set**: apply labels/annotations/taints (if any) and stop here — the node is not rebooted
1. Cordon and drain the Kubernetes node if `--drain` is enabled. **If the drain fails** (e.g. a pod cannot be evicted within `--drain-timeout`), the upgrade aborts unless `--delete-if-eviction-fails` is set: in that case, the drain retries with pod deletion (DELETE instead of EVICT, bypassing PodDisruptionBudgets, reusing `--drain-timeout`). If the forced drain also fails, the node is left cordoned with the new artifacts installed but not rebooted, and no further nodes are upgraded. In-flight upgrades on other nodes (when `--max-parallel > 1`) are allowed to complete, but no new ones are started. The node must be uncordoned and rebooted manually to recover.
1. Issue a `Reboot` with the selected reboot mode (default: kexec)
1. Wait 30 seconds for the node to stabilize
1. Wait `--stabilization-duration` (default: 30 seconds) for the node to stabilize
1. Uncordon the Kubernetes node

**Legacy flow** *(Talos < 1.13)*:

1. Issue `MachineService.Upgrade`, which installs the upgrade artifacts, cordons and drains the node, and reboots — all in a single server-side sequence. `--drain` and `--drain-timeout` are ignored (Talos drains and uncordons the node itself); `--force` skips etcd health checks. `--stage` is not supported on legacy nodes.
1. Wait 30 seconds for the node to stabilize
1. Wait `--stabilization-duration` (default: 30 seconds) for the node to stabilize

## Installer Image

Expand Down
5 changes: 4 additions & 1 deletion internal/cmd/apply/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ type Options struct {
SkipPostApplyChecks bool
// Allow applying to nodes that are not ready (have unmet conditions)
AllowNotReady bool
// StabilizationDuration is how long a node must stay ready after
// applying before it is considered stable.
StabilizationDuration time.Duration
// Apply mode passed to Talos (auto, reboot, no-reboot, staged, try)
Mode machine.ApplyConfigurationRequest_Mode
// MaxParallel controls how many worker nodes are applied to concurrently.
Expand Down Expand Up @@ -195,7 +198,7 @@ func applyNode(ctx context.Context, node *topf.Node, opts Options, logger *slog.
return nil
}

if err = node.Stabilize(ctx, logger, time.Second*30); err != nil {
if err = node.Stabilize(ctx, logger, opts.StabilizationDuration); err != nil {
return fmt.Errorf("node didn't stabilize: %w", err)
}

Expand Down
8 changes: 6 additions & 2 deletions internal/cmd/upgrade/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ type Options struct {
// DrainTimeout is the maximum time to wait for pod evictions to complete.
DrainTimeout time.Duration

// StabilizationDuration is how long a node must stay ready after
// rebooting before it is considered stable.
StabilizationDuration time.Duration

// DeleteIfEvictionFails retries the drain with direct pod deletion
// (DELETE instead of EVICT, bypassing PDBs) if the graceful drain fails.
DeleteIfEvictionFails bool
Expand Down Expand Up @@ -372,7 +376,7 @@ func upgradeNodeLifecycle(ctx context.Context, t topf.Topf, node *topf.Node, opt

logger.Info("reboot initiated")

if err = node.Stabilize(ctx, logger, time.Second*30); err != nil {
if err = node.Stabilize(ctx, logger, opts.StabilizationDuration); err != nil {
return fmt.Errorf("node didn't stabilize: %w", err)
}

Expand Down Expand Up @@ -420,7 +424,7 @@ func upgradeNodeLegacy(ctx context.Context, _ topf.Topf, node *topf.Node, opts O

logger.Info("upgrade initiated")

if err = node.Stabilize(ctx, logger, time.Second*30); err != nil {
if err = node.Stabilize(ctx, logger, opts.StabilizationDuration); err != nil {
return fmt.Errorf("node didn't stabilize: %w", err)
}

Expand Down