diff --git a/cmd/topf/upgrade.go b/cmd/topf/upgrade.go index 31ad2a0..3431187 100644 --- a/cmd/topf/upgrade.go +++ b/cmd/topf/upgrade.go @@ -76,6 +76,12 @@ func newUpgradeCmd() *cli.Command { Sources: cli.EnvVars("TOPF_FORCE"), DefaultText: defaultTextFalse, }, + &cli.BoolFlag{ + Name: "skip-node-prechecks", + Usage: "skip the pre-upgrade checks that require every node to be reachable and in the \"running\" stage; needed to upgrade a node that is stuck in another stage (e.g. after a bad machine image), at the cost of no longer aborting early on unhealthy nodes", + Sources: cli.EnvVars("TOPF_SKIP_NODE_PRECHECKS"), + DefaultText: defaultTextFalse, + }, &cli.BoolFlag{ Name: "stage", Usage: "install upgrade artifacts without rebooting; the node can be rebooted later to complete the upgrade (Talos >= 1.13 only)", @@ -116,6 +122,7 @@ func newUpgradeCmd() *cli.Command { DryRun: c.Bool("dry-run"), RebootMode: rebootMode, Force: c.Bool("force"), + SkipNodePreChecks: c.Bool("skip-node-prechecks"), Drain: c.Bool("drain"), DrainTimeout: c.Duration("drain-timeout"), StabilizationDuration: c.Duration("stabilization-duration"), diff --git a/docs/commands/upgrade.md b/docs/commands/upgrade.md index e77fa23..0d56b0e 100644 --- a/docs/commands/upgrade.md +++ b/docs/commands/upgrade.md @@ -16,6 +16,7 @@ All flags can also be set via environment variables using the `TOPF_` prefix and | `--stabilization-duration` | `30s` | How long a node must stay ready after rebooting before it is considered stable | | `--delete-if-eviction-fails` | `false` | If graceful drain fails (e.g. a PodDisruptionBudget blocks eviction), retry by deleting pods directly (DELETE instead of EVICT, bypassing PDBs); reuses `--drain-timeout` for the delete fallback *(modern flow only)* | | `--force` | `false` | Skip etcd health checks; only applies to nodes running Talos < 1.13 (legacy `MachineService.Upgrade` RPC); has no effect on Talos >= 1.13, where the `LifecycleService.Upgrade` RPC validates etcd health server-side | +| `--skip-node-prechecks` | `false` | Skip the pre-upgrade checks that require every node to be reachable and in the `Running` stage; needed to upgrade a node that is stuck in another stage (e.g. after a bad machine image) | | `--stage` | `false` | Install upgrade artifacts without rebooting; the node is left running and can be labeled/annotated/tainted (see `--stage-label`/`--stage-annotation`/`--stage-taint`) so an external controller or human reboots it later | | `--stage-label` | - | Kubernetes node label to apply after staging (`key=value`); can be repeated; requires `--stage` | | `--stage-annotation` | - | Kubernetes node annotation to apply after staging (`key=value`); can be repeated; requires `--stage` | @@ -46,6 +47,17 @@ All flags can also be set via environment variables using the `TOPF_` prefix and > direct deletion for the fallback attempt, which is what lets it bypass > PDBs. > +> **When to use `--skip-node-prechecks`.** By default the upgrade aborts if +> any node is unreachable or not in the `Running` stage, so a broken cluster +> is not made worse. That check also blocks the recovery case: a node that +> fails to boot after an upgrade (e.g. a bad machine image) never reaches +> `Running`, so no upgrade — not even one back to a known-good installer — +> can be issued for it. `--skip-node-prechecks` bypasses the check and goes +> straight to the plan phase. Unhealthy nodes are no longer caught up front; +> they fail individually once their own upgrade is attempted. Combine it with +> [`--nodes-filter`](../configuration.md#filtering-nodes) to target only the +> node that needs recovering. +> > **Staging upgrades with `--stage`** *(Talos >= 1.13 only)*. Sometimes you > want to install new Talos artifacts on nodes without immediately rebooting > them — e.g. to spread reboots over a maintenance window or let an external @@ -66,7 +78,7 @@ All flags can also be set via environment variables using the `TOPF_` prefix and ## Behavior -1. **Pre-flight checks**: Ensures all nodes are in the `Running` stage +1. **Pre-flight checks**: Ensures all nodes are reachable and in the `Running` stage (skipped with `--skip-node-prechecks`) 1. **Version comparison**: Extracts schematic and version from the installer image and only upgrades nodes where either differs from the current state 1. **Per-node confirmation**: Before each upgrade (unless `--confirm=false`, see [global flags](../configuration.md#global-flags)) 1. **API selection**: Per node, if the running Talos version is >= 1.13.0, the modern flow (a) is used; otherwise the legacy flow (b) is used @@ -174,6 +186,9 @@ topf upgrade --delete-if-eviction-fails # Upgrade with a longer drain timeout (shared by graceful and fallback) topf upgrade --delete-if-eviction-fails --drain-timeout=10m +# Upgrade a node stuck in a non-running stage (e.g. after a bad machine image) +topf upgrade --skip-node-prechecks --nodes-filter '^node1$' + # Stage an upgrade without rebooting (reboot manually later to complete it) topf upgrade --stage diff --git a/internal/cmd/upgrade/upgrade.go b/internal/cmd/upgrade/upgrade.go index d521007..dba34eb 100644 --- a/internal/cmd/upgrade/upgrade.go +++ b/internal/cmd/upgrade/upgrade.go @@ -50,6 +50,11 @@ type Options struct { // server-side and has no force knob. Force bool + // SkipNodePreChecks skips the pre-upgrade checks that require every + // node to be reachable and in the "running" stage. Needed to upgrade a + // node that is stuck in another stage, e.g. after a bad machine image. + SkipNodePreChecks bool + // Drain controls whether the Kubernetes node is cordoned and drained // before the reboot and uncordoned after the node becomes Ready again. Drain bool @@ -104,7 +109,7 @@ func Execute(ctx context.Context, t topf.Topf, opts Options) error { return err } - if err := preChecks(logger, nodes); err != nil { + if err := preChecks(logger, nodes, opts); err != nil { return err } @@ -213,8 +218,15 @@ func validateOptions(opts *Options) error { } // preChecks verifies that every node is reachable and running before any -// upgrade is attempted, reporting all problems at once. -func preChecks(logger *slog.Logger, nodes []*topf.Node) error { +// upgrade is attempted, reporting all problems at once. It is a no-op when +// opts.SkipNodePreChecks is set. +func preChecks(logger *slog.Logger, nodes []*topf.Node, opts Options) error { + if opts.SkipNodePreChecks { + logger.Warn("skipping node pre-checks: unreachable or non-running nodes will only fail once their upgrade is attempted") + + return nil + } + abort := false for _, node := range nodes {