@@ -622,6 +622,96 @@ public class AgentProperties{
622622 public static final Property <Boolean > REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT
623623 = new Property <>("reboot.host.and.alert.management.on.heartbeat.timeout" , false );
624624
625+ /**
626+ * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh, kvmspheartbeat.sh
627+ * and kvmsmpheartbeat.sh) when a heartbeat write fails persistently. Only consulted when
628+ * {@link #REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT} is true: that boolean remains the
629+ * on/off switch for fencing, this setting only chooses what the fence does. Allowed values:
630+ * <ul>
631+ * <li>{@code hard-reboot} (default): immediate sysrq-trigger reboot. Required default for setups
632+ * where a stale NFSv3 mount can prevent a graceful shutdown from completing.</li>
633+ * <li>{@code graceful-reboot}: {@code systemctl reboot} instead of sysrq; allows VMs
634+ * to stop cleanly. Use only if a stale storage mount cannot block shutdown.</li>
635+ * <li>{@code restart-agent}: stop every VM on the host, then restart cloudstack-agent; the host stays up. If it
636+ * already fired within {@link #KVM_HEARTBEAT_FENCE_RESTART_AGENT_COOLDOWN}, hard-reboot instead.</li>
637+ * <li>{@code log-only}: stop every VM on the host, then only write a message to the host's syslog (no
638+ * reboot, no agent restart, no management server alert); the operator must investigate.</li>
639+ * </ul>
640+ * Any other value falls back to {@code hard-reboot}. {@code restart-agent} and {@code graceful-reboot} need
641+ * systemctl; if it is missing or the systemctl call fails, the fence hard-reboots.<br>
642+ * {@code restart-agent} and {@code log-only} stop the VMs because, once the heartbeat is stale, the management
643+ * server may declare the host Down and HA may start the same VMs on another host, where both copies would write
644+ * to the same disks. If any VM cannot be stopped the fence falls back to {@code hard-reboot}. See
645+ * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING}.<br>
646+ * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated
647+ * local storage, where transient I/O contention can cause a heartbeat write to time out
648+ * without the host actually being unhealthy.<br>
649+ * Read by the heartbeat shell scripts directly from agent.properties.<br>
650+ * Data type: String.<br>
651+ * Default value: {@code hard-reboot}
652+ */
653+ public static final Property <String > KVM_HEARTBEAT_FENCE_ACTION
654+ = new Property <>("kvm.heartbeat.fence.action" , "hard-reboot" );
655+
656+ /**
657+ * When true, the {@code restart-agent} and {@code log-only} fence actions leave the host's VMs running instead of
658+ * stopping them first. Only safe when no VM on the host can be started on another host by HA (VM HA off for all
659+ * of them and host HA not enabled); otherwise a host the management server declares Down can end up with the
660+ * same VM running twice, writing to the same disks. No effect on {@code hard-reboot} and {@code graceful-reboot}.
661+ * This is the master opt-out: when true nothing is stopped and {@link #KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS}
662+ * is ignored.<br>
663+ * Read by the heartbeat shell scripts directly from agent.properties.<br>
664+ * Data type: Boolean.<br>
665+ * Default value: <code>false</code>
666+ */
667+ public static final Property <Boolean > KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING
668+ = new Property <>("kvm.heartbeat.fence.keep.vms.running" , false );
669+
670+ /**
671+ * Whether the {@code restart-agent} and {@code log-only} fence actions also stop system VMs (virtual routers,
672+ * console proxy, secondary storage and internal LB VMs, recognised by their {@code r-}, {@code v-}, {@code s-} and
673+ * {@code b-} instance names). When true every VM on the host is stopped. When false only user VMs are stopped;
674+ * system VMs keep running and do not trigger the {@code hard-reboot} fallback. Risk of false: if the management
675+ * server declares the host Down it restarts the system VMs on another host, so a virtual router could run twice
676+ * with the same IP and MAC addresses. Ignored when {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true. No
677+ * effect on {@code hard-reboot} and {@code graceful-reboot}.
678+ * Read by the heartbeat shell scripts directly from agent.properties.<br>
679+ * Data type: Boolean.<br>
680+ * Default value: <code>true</code>
681+ */
682+ public static final Property <Boolean > KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS
683+ = new Property <>("kvm.heartbeat.fence.include.system.vms" , true );
684+
685+ /**
686+ * Seconds the {@code restart-agent} and {@code log-only} fence actions may spend stopping VMs before the host is
687+ * hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that keeps running even if the
688+ * agent kills the heartbeat script when {@link #HEARTBEAT_UPDATE_TIMEOUT} runs out (stopping a VM can block for a
689+ * long time on hung storage). If the VMs are not all confirmed stopped within this budget, the deadman
690+ * hard-reboots the host. Each virsh call is also limited to 10 seconds, or this value if lower, when coreutils
691+ * {@code timeout} is installed. Keep it below {@link #HEARTBEAT_UPDATE_TIMEOUT}. A value that is not a positive
692+ * integer falls back to the default. No effect on {@code hard-reboot} and {@code graceful-reboot}, or when
693+ * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true.<br>
694+ * Read by the heartbeat shell scripts directly from agent.properties.<br>
695+ * Data type: Integer.<br>
696+ * Default value: <code>30</code>
697+ */
698+ public static final Property <Integer > KVM_HEARTBEAT_FENCE_STOP_VMS_TIMEOUT
699+ = new Property <>("kvm.heartbeat.fence.stop.vms.timeout" , 30 );
700+
701+ /**
702+ * Loop guard for the {@code restart-agent} fence action, in seconds. While storage keeps failing, the restarted
703+ * agent runs the heartbeat again, it fails again and {@code restart-agent} would restart the agent again, over and
704+ * over. Each {@code restart-agent} fence records its time under /run (kept across agent restarts, cleared by a
705+ * reboot); if {@code restart-agent} already fired less than this many seconds ago, the fence hard-reboots the host
706+ * instead. 0 turns the guard off. A value that is not a whole number of seconds falls back to the default. No
707+ * effect on {@code hard-reboot}, {@code graceful-reboot} and {@code log-only}.<br>
708+ * Read by the heartbeat shell scripts directly from agent.properties.<br>
709+ * Data type: Integer.<br>
710+ * Default value: <code>600</code>
711+ */
712+ public static final Property <Integer > KVM_HEARTBEAT_FENCE_RESTART_AGENT_COOLDOWN
713+ = new Property <>("kvm.heartbeat.fence.restart.agent.cooldown" , 600 );
714+
625715 /**
626716 * Enables manually setting CPU's topology on KVM's VM. <br>
627717 * Data type: Boolean.<br>
0 commit comments