feat: allow skipping node prechecks on upgrade

cmd/topf/upgrade.go

@@ -76,6 +76,12 @@ func newUpgradeCmd() *cli.Command {
 				Sources:     cli.EnvVars("TOPF_FORCE"),
 				DefaultText: defaultTextFalse,
 			},
+			&cli.BoolFlag{
+				Name:        "skip-node-prechecks",
+				Usage:       "skip the pre-upgrade checks that require every node to be reachable and in the \"running\" stage; needed to upgrade a node that is stuck in another stage (e.g. after a bad machine image), at the cost of no longer aborting early on unhealthy nodes",
+				Sources:     cli.EnvVars("TOPF_SKIP_NODE_PRECHECKS"),
+				DefaultText: defaultTextFalse,
+			},
 			&cli.BoolFlag{
 				Name:        "stage",
 				Usage:       "install upgrade artifacts without rebooting; the node can be rebooted later to complete the upgrade (Talos >= 1.13 only)",
@@ -116,6 +122,7 @@ func newUpgradeCmd() *cli.Command {
 				DryRun:                c.Bool("dry-run"),
 				RebootMode:            rebootMode,
 				Force:                 c.Bool("force"),
+				SkipNodePreChecks:     c.Bool("skip-node-prechecks"),
 				Drain:                 c.Bool("drain"),
 				DrainTimeout:          c.Duration("drain-timeout"),
 				StabilizationDuration: c.Duration("stabilization-duration"),

docs/commands/upgrade.md

@@ -16,6 +16,7 @@ All flags can also be set via environment variables using the `TOPF_` prefix and
 | `--stabilization-duration` | `30s` | How long a node must stay ready after rebooting before it is considered stable |
 | `--delete-if-eviction-fails` | `false` | If graceful drain fails (e.g. a PodDisruptionBudget blocks eviction), retry by deleting pods directly (DELETE instead of EVICT, bypassing PDBs); reuses `--drain-timeout` for the delete fallback *(modern flow only)* |
 | `--force` | `false` | Skip etcd health checks; only applies to nodes running Talos < 1.13 (legacy `MachineService.Upgrade` RPC); has no effect on Talos >= 1.13, where the `LifecycleService.Upgrade` RPC validates etcd health server-side |
+| `--skip-node-prechecks` | `false` | Skip the pre-upgrade checks that require every node to be reachable and in the `Running` stage; needed to upgrade a node that is stuck in another stage (e.g. after a bad machine image) |
 | `--stage` | `false` | Install upgrade artifacts without rebooting; the node is left running and can be labeled/annotated/tainted (see `--stage-label`/`--stage-annotation`/`--stage-taint`) so an external controller or human reboots it later |
 | `--stage-label` | - | Kubernetes node label to apply after staging (`key=value`); can be repeated; requires `--stage` |
 | `--stage-annotation` | - | Kubernetes node annotation to apply after staging (`key=value`); can be repeated; requires `--stage` |
@@ -46,6 +47,17 @@ All flags can also be set via environment variables using the `TOPF_` prefix and
 > direct deletion for the fallback attempt, which is what lets it bypass
 > PDBs.
 >
+> **When to use `--skip-node-prechecks`.** By default the upgrade aborts if
+> any node is unreachable or not in the `Running` stage, so a broken cluster
+> is not made worse. That check also blocks the recovery case: a node that
+> fails to boot after an upgrade (e.g. a bad machine image) never reaches
+> `Running`, so no upgrade — not even one back to a known-good installer —
+> can be issued for it. `--skip-node-prechecks` bypasses the check and goes
+> straight to the plan phase. Unhealthy nodes are no longer caught up front;
+> they fail individually once their own upgrade is attempted. Combine it with
+> [`--nodes-filter`](../configuration.md#filtering-nodes) to target only the
+> node that needs recovering.
+>
 > **Staging upgrades with `--stage`** *(Talos >= 1.13 only)*. Sometimes you
 > want to install new Talos artifacts on nodes without immediately rebooting
 > them — e.g. to spread reboots over a maintenance window or let an external
@@ -66,7 +78,7 @@ All flags can also be set via environment variables using the `TOPF_` prefix and
 
 ## Behavior
 
-1. **Pre-flight checks**: Ensures all nodes are in the `Running` stage
+1. **Pre-flight checks**: Ensures all nodes are reachable and in the `Running` stage (skipped with `--skip-node-prechecks`)
 1. **Version comparison**: Extracts schematic and version from the installer image and only upgrades nodes where either differs from the current state
 1. **Per-node confirmation**: Before each upgrade (unless `--confirm=false`, see [global flags](../configuration.md#global-flags))
 1. **API selection**: Per node, if the running Talos version is >= 1.13.0, the modern flow (a) is used; otherwise the legacy flow (b) is used
@@ -174,6 +186,9 @@ topf upgrade --delete-if-eviction-fails
 # Upgrade with a longer drain timeout (shared by graceful and fallback)
 topf upgrade --delete-if-eviction-fails --drain-timeout=10m
 
+# Upgrade a node stuck in a non-running stage (e.g. after a bad machine image)
+topf upgrade --skip-node-prechecks --nodes-filter '^node1$'
+
 # Stage an upgrade without rebooting (reboot manually later to complete it)
 topf upgrade --stage
 

internal/cmd/upgrade/upgrade.go

@@ -50,6 +50,11 @@ type Options struct {
 	// server-side and has no force knob.
 	Force bool
 
+	// SkipNodePreChecks skips the pre-upgrade checks that require every
+	// node to be reachable and in the "running" stage. Needed to upgrade a
+	// node that is stuck in another stage, e.g. after a bad machine image.
+	SkipNodePreChecks bool
+
 	// Drain controls whether the Kubernetes node is cordoned and drained
 	// before the reboot and uncordoned after the node becomes Ready again.
 	Drain bool
@@ -104,7 +109,7 @@ func Execute(ctx context.Context, t topf.Topf, opts Options) error {
 		return err
 	}
 
-	if err := preChecks(logger, nodes); err != nil {
+	if err := preChecks(logger, nodes, opts); err != nil {
 		return err
 	}
 
@@ -213,8 +218,15 @@ func validateOptions(opts *Options) error {
 }
 
 // preChecks verifies that every node is reachable and running before any
-// upgrade is attempted, reporting all problems at once.
-func preChecks(logger *slog.Logger, nodes []*topf.Node) error {
+// upgrade is attempted, reporting all problems at once. It is a no-op when
+// opts.SkipNodePreChecks is set.
+func preChecks(logger *slog.Logger, nodes []*topf.Node, opts Options) error {
+	if opts.SkipNodePreChecks {
+		logger.Warn("skipping node pre-checks: unreachable or non-running nodes will only fail once their upgrade is attempted")
+
+		return nil
+	}
+
 	abort := false
 
 	for _, node := range nodes {