From fb48eaa53e8a4b1dd116394e40bae1671a182b5a Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Tue, 17 Mar 2020 12:13:52 +0100 Subject: [PATCH 01/15] Feature: scheduler: add priority-fencing-delay cluster option This feature addresses the relevant topics and implements the ideas brought up from: https://github.com/ClusterLabs/fence-agents/pull/308 This commit adds priority-fencing-delay option (just the option, not the feature itself). Enforce specified delay for the fencings that are targeting the lost nodes with the highest total resource priority in case we don't have the majority of the nodes in our cluster partition, so that the more significant nodes potentially win any fencing match, which is especially meaningful under split-brain of 2-node cluster. A promoted resource instance takes the base priority + 1 on calculation if the base priority is not 0. If all the nodes have equal priority, then any pcmk_delay_base/max configured for the corresponding fencing resources will be applied. Otherwise as long as it's set, even if to 0, it takes precedence over any configured pcmk_delay_base/max. By default, priority fencing delay is disabled. --- include/crm/msg_xml.h | 1 + lib/pengine/common.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h index de99959e462..7fa3c39f30f 100644 --- a/include/crm/msg_xml.h +++ b/include/crm/msg_xml.h @@ -378,6 +378,7 @@ # define XML_CONFIG_ATTR_FORCE_QUIT "shutdown-escalation" # define XML_CONFIG_ATTR_RECHECK "cluster-recheck-interval" # define XML_CONFIG_ATTR_FENCE_REACTION "fence-reaction" +# define XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY "priority-fencing-delay" # define XML_ALERT_ATTR_PATH "path" # define XML_ALERT_ATTR_TIMEOUT "timeout" diff --git a/lib/pengine/common.c b/lib/pengine/common.c index e82434a37ee..08de8f765cd 100644 --- a/lib/pengine/common.c +++ b/lib/pengine/common.c @@ -121,6 +121,23 @@ pe_cluster_option pe_opts[] = { "Allow performing fencing operations in parallel", NULL }, { "startup-fencing", "startup_fencing", "boolean", NULL, "true", &check_boolean, "STONITH unseen nodes", "Advanced Use Only! Not using the default is very unsafe!" }, + { + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL, + NULL, &check_timer, + "Enforced fencing delay targeting the lost nodes with the highest total resource priority", + "Enforce specified delay for the fencings that are targeting the lost " + "nodes with the highest total resource priority in case we don't " + "have the majority of the nodes in our cluster partition, so that " + "the more significant nodes potentially win any fencing match, " + "which is especially meaningful under split-brain of 2-node " + "cluster. A promoted resource instance takes the base priority + 1 " + "on calculation if the base priority is not 0. If all the nodes " + "have equal priority, then any pcmk_delay_base/max configured for " + "the corresponding fencing resources will be applied. Otherwise as " + "long as it's set, even if to 0, it takes precedence over any " + "configured pcmk_delay_base/max. By default, priority fencing " + "delay is disabled." + }, /* Timeouts etc */ { "cluster-delay", "transition_idle_timeout", "time", NULL, "60s", &check_time, From 05c68007500e36d8af563eb73bed247f883dd999 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Tue, 17 Mar 2020 14:33:35 +0100 Subject: [PATCH 02/15] Feature: scheduler: implement priority-fencing-delay --- include/crm/pengine/internal.h | 4 +- include/crm/pengine/status.h | 2 + lib/pengine/native.c | 47 ++++++++++++++++ lib/pengine/unpack.c | 43 +++++++++------ lib/pengine/utils.c | 98 ++++++++++++++++++++++++++++++++-- pengine/allocate.c | 14 ++--- pengine/native.c | 8 +-- 7 files changed, 184 insertions(+), 32 deletions(-) diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h index 023ab92b7bc..7c3e2c09171 100644 --- a/include/crm/pengine/internal.h +++ b/include/crm/pengine/internal.h @@ -329,7 +329,7 @@ typedef struct op_digest_cache_s { op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, pe_working_set_t * data_set); -action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set); +pe_action_t *pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, bool priority_delay, pe_working_set_t * data_set); void trigger_unfencing( resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set); @@ -346,7 +346,7 @@ gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options, void * print_data, gboolean print_all); -void pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason); +void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay); node_t *pe_create_node(const char *id, const char *uname, const char *type, const char *score, pe_working_set_t * data_set); diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h index 8be84d57b75..2af38596d0a 100644 --- a/include/crm/pengine/status.h +++ b/include/crm/pengine/status.h @@ -153,6 +153,7 @@ typedef struct pe_working_set_s { GList *param_check; // History entries that need to be checked GList *stop_needed; // Containers that need stop actions int ninstances; // Total number of resource instances + int priority_fencing_delay; // Enforced priority fencing delay } pe_working_set_t; enum pe_check_parameters { @@ -202,6 +203,7 @@ struct node_shared_s { gboolean remote_was_fenced; gboolean remote_maintenance; /* what the remote-rsc is thinking */ gboolean unpacked; + int priority; // calculated based on the priority of resources running on the node }; struct node_s { diff --git a/lib/pengine/native.c b/lib/pengine/native.c index 3c9f8f59242..16bbcdc2f5d 100644 --- a/lib/pengine/native.c +++ b/lib/pengine/native.c @@ -33,6 +33,51 @@ is_multiply_active(pe_resource_t *rsc) return count > 1; } +static void +native_priority_to_node(pe_resource_t * rsc, pe_node_t * node) +{ + int priority = 0; + + if (rsc->priority == 0) { + return; + } + + if (rsc->role == RSC_ROLE_MASTER) { + // Promoted instance takes base priority + 1 + priority = rsc->priority + 1; + + } else { + priority = rsc->priority; + } + + node->details->priority += priority; + pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s)", + node->details->uname, node->details->priority, + rsc->role == RSC_ROLE_MASTER ? "promoted " : "", + rsc->id, rsc->priority, + rsc->role == RSC_ROLE_MASTER ? " + 1" : ""); + + /* Priority of a resource running on a guest node is added to the cluster + * node as well. */ + if (node->details->remote_rsc + && node->details->remote_rsc->container) { + GListPtr gIter = node->details->remote_rsc->container->running_on; + + for (; gIter != NULL; gIter = gIter->next) { + pe_node_t *a_node = gIter->data; + + a_node->details->priority += priority; + pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s) " + "from guest node '%s'", + a_node->details->uname, a_node->details->priority, + rsc->role == RSC_ROLE_MASTER ? "promoted " : "", + rsc->id, rsc->priority, + rsc->role == RSC_ROLE_MASTER ? " + 1" : "", + node->details->uname); + } + } +} + void native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set) { @@ -54,6 +99,8 @@ native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set) rsc->running_on = g_list_append(rsc->running_on, node); if (rsc->variant == pe_native) { node->details->running_rsc = g_list_append(node->details->running_rsc, rsc); + + native_priority_to_node(rsc, node); } if (rsc->variant == pe_native && node->details->maintenance) { diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c index 24e56f51205..1cd0faecaa6 100644 --- a/lib/pengine/unpack.c +++ b/lib/pengine/unpack.c @@ -73,9 +73,11 @@ is_dangling_container_remote_node(node_t *node) * \param[in,out] data_set Current working set of cluster * \param[in,out] node Node to fence * \param[in] reason Text description of why fencing is needed + * \param[in] priority_delay Whether to consider `priority-fencing-delay` */ void -pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) +pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, + const char *reason, bool priority_delay) { CRM_CHECK(node, return); @@ -125,7 +127,8 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) reason); } node->details->unclean = TRUE; - pe_fence_op(node, NULL, TRUE, reason, data_set); + // No need to apply `priority-fencing-delay` for remote nodes + pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set); } else if (node->details->unclean) { crm_trace("Cluster node %s %s because %s", @@ -139,7 +142,7 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) pe_can_fence(data_set, node)? "will be fenced" : "is unclean", reason); node->details->unclean = TRUE; - pe_fence_op(node, NULL, TRUE, reason, data_set); + pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set); } } @@ -223,6 +226,15 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) crm_debug("Concurrent fencing is %s", is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled"); + // Default value -1 means `priority-fencing-delay` is disabled + data_set->priority_fencing_delay = -1; + value = pe_pref(data_set->config_hash, + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); + if (value) { + data_set->priority_fencing_delay = crm_get_msec(value) / 1000; + crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay); + } + set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything); crm_debug("Stop all active resources: %s", is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false"); @@ -1278,7 +1290,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) /* Everything else should flow from this automatically * At least until the PE becomes able to migrate off healthy resources */ - pe_fence_node(data_set, this_node, "cluster does not have quorum"); + pe_fence_node(data_set, this_node, "cluster does not have quorum", FALSE); } } } @@ -1350,7 +1362,7 @@ determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_s } else { /* mark it unclean */ - pe_fence_node(data_set, this_node, "peer is unexpectedly down"); + pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE); crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s", crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state)); } @@ -1407,10 +1419,10 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat online = crm_is_true(is_peer); } else if (in_cluster == NULL) { - pe_fence_node(data_set, this_node, "peer has not been seen by the cluster"); + pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE); } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) { - pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria"); + pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria", FALSE); } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) { @@ -1429,14 +1441,15 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat online = FALSE; } else if (crm_is_true(in_cluster) == FALSE) { - pe_fence_node(data_set, this_node, "peer is no longer part of the cluster"); + // Consider `priority-fencing-delay` for lost nodes + pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE); } else if (crm_is_true(is_peer) == FALSE) { - pe_fence_node(data_set, this_node, "peer process is no longer available"); + pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE); /* Everything is running at this point, now check join state */ } else if (do_terminate) { - pe_fence_node(data_set, this_node, "termination was requested"); + pe_fence_node(data_set, this_node, "termination was requested", FALSE); } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) { crm_info("Node %s is active", this_node->details->uname); @@ -1448,7 +1461,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat this_node->details->pending = TRUE; } else { - pe_fence_node(data_set, this_node, "peer was in an unknown state"); + pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE); crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d", this_node->details->uname, crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown); @@ -2012,7 +2025,7 @@ process_rsc_state(resource_t * rsc, node_t * node, if (reason == NULL) { reason = crm_strdup_printf("%s is thought to be active there", rsc->id); } - pe_fence_node(data_set, node, reason); + pe_fence_node(data_set, node, reason, FALSE); } free(reason); } @@ -2034,7 +2047,7 @@ process_rsc_state(resource_t * rsc, node_t * node, * but also mark the node as unclean */ reason = crm_strdup_printf("%s failed there", rsc->id); - pe_fence_node(data_set, node, reason); + pe_fence_node(data_set, node, reason, FALSE); free(reason); break; @@ -2101,7 +2114,7 @@ process_rsc_state(resource_t * rsc, node_t * node, /* connection resource to baremetal resource failed in a way that * should result in fencing the remote-node. */ pe_fence_node(data_set, tmpnode, - "remote connection is unrecoverable"); + "remote connection is unrecoverable", FALSE); } } @@ -3155,7 +3168,7 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod && remote_node && remote_node->details->unclean) { - action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, data_set); + pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, FALSE, data_set); crm_notice("Waiting for %s to complete before clearing %s failure for remote node %s", fence?fence->uuid:"nil", task, rsc->id); order_actions(fence, clear_op, pe_order_implies_then); diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c index 558075a1452..10c7b0b67ec 100644 --- a/lib/pengine/utils.c +++ b/lib/pengine/utils.c @@ -622,7 +622,7 @@ custom_action(resource_t * rsc, char *key, const char *task, if (is_set(action->rsc->flags, pe_rsc_managed) && save_action && a_task == stop_rsc && action->node->details->unclean == FALSE) { - pe_fence_node(data_set, action->node, "resource actions are unrunnable"); + pe_fence_node(data_set, action->node, "resource actions are unrunnable", FALSE); } } else if (action->node->details->pending) { @@ -2342,9 +2342,76 @@ find_unfencing_devices(GListPtr candidates, GListPtr matches) return matches; } +static int +node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set) +{ + int member_count = 0; + int online_count = 0; + int top_priority = 0; + int lowest_priority = 0; + GListPtr gIter = NULL; -action_t * -pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set) + // `priority-fencing-delay` is disabled + if (data_set->priority_fencing_delay < 0) { + return -1; + } + + /* No need to delay fencing if the fencing target is not a normal cluster + * member, for example if it's a remote node or a guest node. */ + if (node->details->type != node_member) { + return 0; + } + + // No need to delay fencing if the fencing target is in our partition + if (node->details->online) { + return 0; + } + + for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + pe_node_t *n = gIter->data; + + if (n->details->type != node_member) { + continue; + } + + member_count ++; + + if (n->details->online) { + online_count++; + } + + if (member_count == 1 + || n->details->priority > top_priority) { + top_priority = n->details->priority; + } + + if (member_count == 1 + || n->details->priority < lowest_priority) { + lowest_priority = n->details->priority; + } + } + + // No need to delay if we have more than half of the cluster members + if (online_count > member_count / 2) { + return 0; + } + + /* All the nodes have equal priority. + * Any configured corresponding `pcmk_delay_base/max` will be applied. */ + if (lowest_priority == top_priority) { + return -1; + } + + if (node->details->priority < top_priority) { + return 0; + } + + return data_set->priority_fencing_delay; +} + +pe_action_t * +pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, + bool priority_delay, pe_working_set_t * data_set) { char *op_key = NULL; action_t *stonith_op = NULL; @@ -2415,6 +2482,29 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe free(op_key); } + if (data_set->priority_fencing_delay >= 0 + + /* It's a suitable case where `priority-fencing-delay` applies. + * At least add `priority-fencing-delay` field as an indicator. */ + && (priority_delay + + /* Re-calculate priority delay for the suitable case when + * pe_fence_op() is called again by stage6() after node priority has + * been actually calculated with native_add_running() */ + || g_hash_table_lookup(stonith_op->meta, + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) { + + /* Add `priority-fencing-delay` to the fencing op even if it's 0 for + * the targeting node. So that it takes precedence over any possible + * `pcmk_delay_base/max`. + */ + char *delay_s = crm_itoa(node_priority_fencing_delay(node, data_set)); + + g_hash_table_insert(stonith_op->meta, + strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY), + delay_s); + } + if(optional == FALSE && pe_can_fence(data_set, node)) { pe_action_required(stonith_op, NULL, reason); } else if(reason && stonith_op->reason == NULL) { @@ -2440,7 +2530,7 @@ trigger_unfencing( && node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { - action_t *unfence = pe_fence_op(node, "on", FALSE, reason, data_set); + pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set); if(dependency) { order_actions(unfence, dependency, pe_order_optional); diff --git a/pengine/allocate.c b/pengine/allocate.c index afe0f6ae55b..2d2ddcc7427 100644 --- a/pengine/allocate.c +++ b/pengine/allocate.c @@ -958,7 +958,7 @@ probe_resources(pe_working_set_t * data_set) if (is_baremetal_remote_node(node) && node->details->remote_rsc && (get_remote_node_state(node) == remote_state_failed)) { - pe_fence_node(data_set, node, "the connection is unrecoverable"); + pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE); } continue; @@ -1494,7 +1494,7 @@ fence_guest(pe_node_t *node, pe_working_set_t *data_set) /* Create a fence pseudo-event, so we have an event to order actions * against, and crmd can always detect it. */ - stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set); + stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, data_set); update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__); @@ -1503,7 +1503,7 @@ fence_guest(pe_node_t *node, pe_working_set_t *data_set) * (even though start might be closer to what is done for a real reboot). */ if(stop && is_set(stop->flags, pe_action_pseudo)) { - pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set); + pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, data_set); crm_info("Implying guest node %s is down (action %d) after %s fencing", node->details->uname, stonith_op->id, stop->node->details->uname); order_actions(parent_stonith_op, stonith_op, @@ -1596,7 +1596,7 @@ stage6(pe_working_set_t * data_set) if (node->details->unclean && need_stonith && pe_can_fence(data_set, node)) { - stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set); + stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set); pe_warn("Scheduling Node %s for STONITH", node->details->uname); stonith_constraints(node, stonith_op, data_set); @@ -1895,7 +1895,7 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) CRM_ASSERT(container); if(is_set(container->flags, pe_rsc_failed)) { - pe_fence_node(data_set, action->node, "container failed"); + pe_fence_node(data_set, action->node, "container failed", FALSE); } crm_trace("Order %s action %s relative to %s%s for %s%s", @@ -2102,7 +2102,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) * to the remote connection, since the stop will become implied * by the fencing. */ - pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); + pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE); } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) { /* State must be remote_state_unknown or remote_state_stopped. @@ -2150,7 +2150,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) * Since we have no way to find out, it is * necessary to fence the node. */ - pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable"); + pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE); } if(cluster_node && state == remote_state_stopped) { diff --git a/pengine/native.c b/pengine/native.c index fd8929d951a..82d1bafcd22 100644 --- a/pengine/native.c +++ b/pengine/native.c @@ -1501,7 +1501,7 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) for (GList *item = allowed_nodes; item; item = item->next) { pe_node_t *node = item->data; - pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); + pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set); crm_debug("Ordering any stops of %s before %s, and any starts after", rsc->id, unfence->uuid); @@ -1970,7 +1970,7 @@ rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; - pe_fence_node(data_set, node, "deadman ticket was lost"); + pe_fence_node(data_set, node, "deadman ticket was lost", FALSE); } break; @@ -2685,7 +2685,7 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d } if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { - action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); + pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set); order_actions(stop, unfence, pe_order_implies_first); if (!node_has_been_unfenced(current)) { @@ -2715,7 +2715,7 @@ order_after_unfencing(resource_t *rsc, pe_node_t *node, action_t *action, * the node being unfenced, and all its resources being stopped, * whenever a new resource is added -- which would be highly suboptimal. */ - action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); + pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set); order_actions(unfence, action, order); From fcd987d270dbece738e0a1fb7bbeb91b6a6d9488 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Tue, 17 Mar 2020 15:29:06 +0100 Subject: [PATCH 03/15] Test: scheduler: add regression test for priority-fencing-delay This is based on the existing test whitebox-imply-stop-on-fence. --- pengine/regression.sh | 1 + pengine/test10/priority-fencing-delay.dot | 109 ++++ pengine/test10/priority-fencing-delay.exp | 570 ++++++++++++++++++ pengine/test10/priority-fencing-delay.scores | 301 +++++++++ pengine/test10/priority-fencing-delay.summary | 102 ++++ pengine/test10/priority-fencing-delay.xml | 351 +++++++++++ 6 files changed, 1434 insertions(+) create mode 100644 pengine/test10/priority-fencing-delay.dot create mode 100644 pengine/test10/priority-fencing-delay.exp create mode 100644 pengine/test10/priority-fencing-delay.scores create mode 100644 pengine/test10/priority-fencing-delay.summary create mode 100644 pengine/test10/priority-fencing-delay.xml diff --git a/pengine/regression.sh b/pengine/regression.sh index cb8b034ed05..166ed21c735 100755 --- a/pengine/regression.sh +++ b/pengine/regression.sh @@ -573,6 +573,7 @@ do_test probe-timeout "cl#5099 - Default probe timeout" do_test order-first-probes "cl#5301 - respect order constraints when relevant resources are being probed" do_test concurrent-fencing "Allow performing fencing operations in parallel" +do_test priority-fencing-delay "Delay fencing targeting the more significant node" echo "" do_test systemhealth1 "System Health () #1" diff --git a/pengine/test10/priority-fencing-delay.dot b/pengine/test10/priority-fencing-delay.dot new file mode 100644 index 00000000000..62ba699eb6d --- /dev/null +++ b/pengine/test10/priority-fencing-delay.dot @@ -0,0 +1,109 @@ + digraph "g" { +"R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-01_kiff-01_start_0 kiff-02" -> "R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style = bold] +"R-lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"R-lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-01_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] +"R-lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-02_kiff-01_start_0 kiff-02" -> "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style = bold] +"R-lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] +"R-lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd-clone_stop_0" -> "clvmd_stop_0 kiff-01" [ style = bold] +"clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +"clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] +"clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"clvmd_monitor_0 lxc-01_kiff-02" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"clvmd_monitor_0 lxc-02_kiff-02" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"clvmd_stop_0 kiff-01" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd_stop_0 kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] +"clvmd_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] +"dlm-clone_stop_0" -> "dlm_stop_0 kiff-01" [ style = bold] +"dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +"dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"dlm_monitor_0 lxc-01_kiff-02" -> "dlm-clone_stopped_0" [ style = bold] +"dlm_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"dlm_monitor_0 lxc-02_kiff-02" -> "dlm-clone_stopped_0" [ style = bold] +"dlm_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"dlm_stop_0 kiff-01" -> "dlm-clone_stopped_0" [ style = bold] +"dlm_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"fence-kiff-02_monitor_60000 kiff-02" [ style=bold color="green" fontcolor="black"] +"fence-kiff-02_start_0 kiff-02" -> "fence-kiff-02_monitor_60000 kiff-02" [ style = bold] +"fence-kiff-02_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"fence-kiff-02_stop_0 kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"fence-kiff-02_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"lxc-01_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_monitor_30000 kiff-02" [ style = bold] +"lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] +"lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] +"lxc-01_kiff-01_stop_0 kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"lxc-02_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_monitor_30000 kiff-02" [ style = bold] +"lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] +"lxc-02_kiff-01_stop_0 kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"shared0-clone_stop_0" -> "shared0-clone_stopped_0" [ style = bold] +"shared0-clone_stop_0" -> "shared0_stop_0 kiff-01" [ style = bold] +"shared0-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +"shared0-clone_stopped_0" -> "clvmd-clone_stop_0" [ style = bold] +"shared0-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"shared0_monitor_0 lxc-01_kiff-02" -> "shared0-clone_stopped_0" [ style = bold] +"shared0_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"shared0_monitor_0 lxc-02_kiff-02" -> "shared0-clone_stopped_0" [ style = bold] +"shared0_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"shared0_stop_0 kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] +"shared0_stop_0 kiff-01" -> "shared0-clone_stopped_0" [ style = bold] +"shared0_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"stonith 'reboot' kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "clvmd-clone_stop_0" [ style = bold] +"stonith 'reboot' kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "dlm-clone_stop_0" [ style = bold] +"stonith 'reboot' kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold] +"stonith 'reboot' kiff-01" -> "shared0_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" [ style=bold color="green" fontcolor="black"] +"stonith 'reboot' lxc-01_kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] +"stonith 'reboot' lxc-02_kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" [ style=bold color="green" fontcolor="orange"] +"vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"vm-fs_monitor_20000 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] +"vm-fs_start_0 lxc-01_kiff-01" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] +"vm-fs_start_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] +"vm-fs_stop_0 lxc-01_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_stop_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] +} diff --git a/pengine/test10/priority-fencing-delay.exp b/pengine/test10/priority-fencing-delay.exp new file mode 100644 index 00000000000..c6315a1280a --- /dev/null +++ b/pengine/test10/priority-fencing-delay.exp @@ -0,0 +1,570 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pengine/test10/priority-fencing-delay.scores b/pengine/test10/priority-fencing-delay.scores new file mode 100644 index 00000000000..b96175efbf1 --- /dev/null +++ b/pengine/test10/priority-fencing-delay.scores @@ -0,0 +1,301 @@ +Allocation scores: +pcmk__clone_allocate: clvmd-clone allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd-clone allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:0 allocation score on kiff-01: 1 +pcmk__clone_allocate: clvmd:0 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:1 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:1 allocation score on kiff-02: 1 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:2 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:2 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:3 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:3 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:4 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:4 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:5 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:5 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: dlm-clone allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm-clone allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm-clone allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm-clone allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm-clone allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm-clone allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on kiff-01: 1 +pcmk__clone_allocate: dlm:0 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:1 allocation score on kiff-02: 1 +pcmk__clone_allocate: dlm:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:2 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:3 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:4 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:5 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: shared0-clone allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0-clone allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:0 allocation score on kiff-01: 1 +pcmk__clone_allocate: shared0:0 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:1 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:1 allocation score on kiff-02: 1 +pcmk__clone_allocate: shared0:1 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:1 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:1 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:1 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:2 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:2 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:3 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:3 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:4 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:4 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:5 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:5 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-02_kiff-02: 0 +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on kiff-02: 100 +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on kiff-02: 100 +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on kiff-02: 1 +pcmk__native_allocate: clvmd:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on kiff-02: 1 +pcmk__native_allocate: dlm:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on kiff-01: 0 +pcmk__native_allocate: fence-kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on kiff-01: 0 +pcmk__native_allocate: fence-kiff-02 allocation score on kiff-02: 0 +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on kiff-02: 1 +pcmk__native_allocate: shared0:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: vm-fs allocation score on kiff-01: 0 +pcmk__native_allocate: vm-fs allocation score on kiff-02: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-01_kiff-01: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-01_kiff-02: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-02_kiff-01: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-02_kiff-02: 0 diff --git a/pengine/test10/priority-fencing-delay.summary b/pengine/test10/priority-fencing-delay.summary new file mode 100644 index 00000000000..83f6d8d6c03 --- /dev/null +++ b/pengine/test10/priority-fencing-delay.summary @@ -0,0 +1,102 @@ + +Current cluster status: +Node kiff-01 (1): UNCLEAN (offline) +Online: [ kiff-02 ] +Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + + vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 + fence-kiff-02 (stonith:fence_ipmilan): Started kiff-01 (UNCLEAN) + Clone Set: dlm-clone [dlm] + dlm (ocf::pacemaker:controld): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: clvmd-clone [clvmd] + clvmd (ocf::heartbeat:clvm): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: shared0-clone [shared0] + shared0 (ocf::heartbeat:Filesystem): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): FAILED kiff-01 (UNCLEAN) + R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-01 (UNCLEAN) + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + +Transition Summary: + * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) 'guest is unclean' + * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' + * Fence (reboot) kiff-01 'peer is no longer part of the cluster' + * Recover vm-fs ( lxc-01_kiff-01 ) + * Move fence-kiff-02 ( kiff-01 -> kiff-02 ) + * Stop dlm:0 ( kiff-01 ) due to node availability + * Stop clvmd:0 ( kiff-01 ) due to node availability + * Stop shared0:0 ( kiff-01 ) due to node availability + * Recover R-lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) + * Move R-lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) + * Move lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) + * Move lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) + +Executing cluster transition: + * Resource action: vm-fs monitor on lxc-02_kiff-02 + * Resource action: vm-fs monitor on lxc-01_kiff-02 + * Pseudo action: fence-kiff-02_stop_0 + * Resource action: dlm monitor on lxc-02_kiff-02 + * Resource action: dlm monitor on lxc-01_kiff-02 + * Resource action: clvmd monitor on lxc-02_kiff-02 + * Resource action: clvmd monitor on lxc-01_kiff-02 + * Resource action: shared0 monitor on lxc-02_kiff-02 + * Resource action: shared0 monitor on lxc-01_kiff-02 + * Pseudo action: lxc-01_kiff-01_stop_0 + * Pseudo action: lxc-02_kiff-01_stop_0 + * Fencing kiff-01 (reboot) + * Pseudo action: R-lxc-01_kiff-01_stop_0 + * Pseudo action: R-lxc-02_kiff-01_stop_0 + * Pseudo action: stonith-lxc-02_kiff-01-reboot on lxc-02_kiff-01 + * Pseudo action: stonith-lxc-01_kiff-01-reboot on lxc-01_kiff-01 + * Pseudo action: vm-fs_stop_0 + * Resource action: fence-kiff-02 start on kiff-02 + * Pseudo action: shared0-clone_stop_0 + * Resource action: R-lxc-01_kiff-01 start on kiff-02 + * Resource action: R-lxc-02_kiff-01 start on kiff-02 + * Resource action: lxc-01_kiff-01 start on kiff-02 + * Resource action: lxc-02_kiff-01 start on kiff-02 + * Resource action: vm-fs start on lxc-01_kiff-01 + * Resource action: fence-kiff-02 monitor=60000 on kiff-02 + * Pseudo action: shared0_stop_0 + * Pseudo action: shared0-clone_stopped_0 + * Resource action: R-lxc-01_kiff-01 monitor=10000 on kiff-02 + * Resource action: R-lxc-02_kiff-01 monitor=10000 on kiff-02 + * Resource action: lxc-01_kiff-01 monitor=30000 on kiff-02 + * Resource action: lxc-02_kiff-01 monitor=30000 on kiff-02 + * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 + * Pseudo action: clvmd-clone_stop_0 + * Pseudo action: clvmd_stop_0 + * Pseudo action: clvmd-clone_stopped_0 + * Pseudo action: dlm-clone_stop_0 + * Pseudo action: dlm_stop_0 + * Pseudo action: dlm-clone_stopped_0 + +Revised cluster status: +Online: [ kiff-02 ] +OFFLINE: [ kiff-01 ] +Containers: [ lxc-01_kiff-01:R-lxc-01_kiff-01 lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-01:R-lxc-02_kiff-01 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + + vm-fs (ocf::heartbeat:Filesystem): Started lxc-01_kiff-01 + R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 + fence-kiff-02 (stonith:fence_ipmilan): Started kiff-02 + Clone Set: dlm-clone [dlm] + Started: [ kiff-02 ] + Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: clvmd-clone [clvmd] + Started: [ kiff-02 ] + Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: shared0-clone [shared0] + Started: [ kiff-02 ] + Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 + R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + diff --git a/pengine/test10/priority-fencing-delay.xml b/pengine/test10/priority-fencing-delay.xml new file mode 100644 index 00000000000..04d1d3e2904 --- /dev/null +++ b/pengine/test10/priority-fencing-delay.xml @@ -0,0 +1,351 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 47cfa3890dcdf56dac14e5505da55124bafa8757 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 18 Mar 2020 15:33:23 +0100 Subject: [PATCH 04/15] Feature: libstonithd: introduce fence_with_delay() operation A parameter value -1 disables enforced fencing delay. Operation fence() is now a wrapper for fence_with_delay(). --- include/crm/fencing/internal.h | 1 + include/crm/stonith-ng.h | 19 +++++++++++++++++++ lib/fencing/st_client.c | 25 +++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h index 24df230132c..264e0d1872f 100644 --- a/include/crm/fencing/internal.h +++ b/include/crm/fencing/internal.h @@ -57,6 +57,7 @@ xmlNode *create_device_registration_xml(const char *id, /*! Timeout period per a device execution */ # define F_STONITH_TIMEOUT "st_timeout" # define F_STONITH_TOLERANCE "st_tolerance" +# define F_STONITH_DELAY "st_delay" /*! Action specific timeout period returned in query of fencing devices. */ # define F_STONITH_ACTION_TIMEOUT "st_action_timeout" /*! Host in query result is not allowed to run this action */ diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h index 23f879b8053..02b751d6676 100644 --- a/include/crm/stonith-ng.h +++ b/include/crm/stonith-ng.h @@ -404,6 +404,25 @@ typedef struct stonith_api_operations_s stonith_key_value_t *params, int timeout, char **output, char **error_output); + /*! + * \brief Issue a fencing action against a node with enforced fencing delay. + * + * \note Possible actions are, 'on', 'off', and 'reboot'. + * + * \param st, stonith connection + * \param options, call options + * \param node, The target node to fence + * \param action, The fencing action to take + * \param timeout, The default per device timeout to use with each device + * capable of fencing the target. + * \param delay, Any enforced fencing delay. -1 to disable + * + * \retval 0 success + * \retval negative error code on failure. + */ + int (*fence_with_delay)(stonith_t *st, int options, const char *node, const char *action, + int timeout, int tolerance, int delay); + } stonith_api_operations_t; struct stonith_s diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 4a1310f835c..e451582040e 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1178,8 +1178,8 @@ stonith_api_status(stonith_t * stonith, int call_options, const char *id, const } static int -stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, - int timeout, int tolerance) +stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node, + const char *action, int timeout, int tolerance, int delay) { int rc = 0; xmlNode *data = NULL; @@ -1190,12 +1190,24 @@ stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance); + if (delay >= 0) { + crm_xml_add_int(data, F_STONITH_DELAY, delay); + } + rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); return rc; } +static int +stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, + int timeout, int tolerance) +{ + return stonith_api_fence_with_delay(stonith, call_options, node, action, + timeout, tolerance, -1); +} + static int stonith_api_confirm(stonith_t * stonith, int call_options, const char *target) { @@ -1924,6 +1936,14 @@ stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNod crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout); crm_trace("Sending %s message to STONITH service, Timeout: %ds", op, timeout); + if (data) { + const char *delay_s = crm_element_value(data, F_STONITH_DELAY); + + if (delay_s) { + crm_xml_add(op_msg, F_STONITH_DELAY, delay_s); + } + } + rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, 1000 * (timeout + 60), &op_reply); free_xml(op_msg); @@ -2203,6 +2223,7 @@ stonith_api_new(void) new_stonith->cmds->monitor = stonith_api_monitor; new_stonith->cmds->status = stonith_api_status; new_stonith->cmds->fence = stonith_api_fence; + new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay; new_stonith->cmds->confirm = stonith_api_confirm; new_stonith->cmds->history = stonith_api_history; From b42f6e38379b88d89b80dfc97746ebc2283ac359 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 18 Mar 2020 15:40:37 +0100 Subject: [PATCH 05/15] Feature: controller: request fencing with any enforced priority fencing delay --- crmd/te_actions.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crmd/te_actions.c b/crmd/te_actions.c index 19bb1999017..8fcfcd925c0 100644 --- a/crmd/te_actions.c +++ b/crmd/te_actions.c @@ -164,6 +164,7 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) const char *uuid = NULL; const char *target = NULL; const char *type = NULL; + const char *priority_delay = NULL; gboolean invalid_action = FALSE; enum stonith_call_options options = st_opt_none; @@ -182,9 +183,11 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) return FALSE; } + priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); + crm_notice("Requesting fencing (%s) of node %s " - CRM_XS " action=%s timeout=%d", - type, target, id, transition_graph->stonith_timeout); + CRM_XS " action=%s timeout=%u priority_delay=%s", + type, target, id, transition_graph->stonith_timeout, priority_delay); /* Passing NULL means block until we can connect... */ te_connect_stonith(NULL); @@ -193,8 +196,9 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) options |= st_opt_allow_suicide; } - rc = stonith_api->cmds->fence(stonith_api, options, target, type, - transition_graph->stonith_timeout / 1000, 0); + rc = stonith_api->cmds->fence_with_delay(stonith_api, options, target, type, + (int) (transition_graph->stonith_timeout / 1000), + 0, crm_atoi(priority_delay, "-1")); stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000, st_opt_timeout_updates, From 6c3039aa71d2789b67719395e3223d56c09dd5bf Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 18 Mar 2020 15:44:24 +0100 Subject: [PATCH 06/15] Feature: stonith_admin: add --delay option to support enforced fencing delay It can be specified with --fence, --reboot or --unfence commands. The default value -1 disables enforced fencing delay. --- fencing/admin.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/fencing/admin.c b/fencing/admin.c index 5fedb7b5a62..1c183de1c9d 100644 --- a/fencing/admin.c +++ b/fencing/admin.c @@ -176,6 +176,10 @@ static struct crm_option long_options[] = { "Operation timeout in seconds (default 120;\n" "\t\t\tused with most commands)." }, + { "delay", required_argument, NULL, 'y', + "Enforced fencing delay in seconds (default -1 (disabled);\n" + "\t\t\twith --fence, --reboot, --unfence)." + }, { "as-node-id", no_argument, NULL, 'n', "(Advanced) The supplied node is the corosync node ID\n" "\t\t\t(with --last)." @@ -201,6 +205,7 @@ struct { char *name; int timeout; int tolerance; + int delay; int rc; } async_fence_data; @@ -265,11 +270,13 @@ async_fence_helper(gpointer user_data) st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, notify_callback); - call_id = st->cmds->fence(st, - st_opt_allow_suicide, - async_fence_data.target, - async_fence_data.action, - async_fence_data.timeout, async_fence_data.tolerance); + call_id = st->cmds->fence_with_delay(st, + st_opt_allow_suicide, + async_fence_data.target, + async_fence_data.action, + async_fence_data.timeout, + async_fence_data.tolerance, + async_fence_data.delay); if (call_id < 0) { g_main_loop_quit(mainloop); @@ -285,7 +292,8 @@ async_fence_helper(gpointer user_data) } static int -mainloop_fencing(stonith_t * st, const char *target, const char *action, int timeout, int tolerance) +mainloop_fencing(stonith_t * st, const char *target, const char *action, + int timeout, int tolerance, int delay) { crm_trigger_t *trig; @@ -294,6 +302,7 @@ mainloop_fencing(stonith_t * st, const char *target, const char *action, int tim async_fence_data.action = action; async_fence_data.timeout = timeout; async_fence_data.tolerance = tolerance; + async_fence_data.delay = delay; async_fence_data.rc = -1; trig = mainloop_add_trigger(G_PRIORITY_HIGH, async_fence_helper, NULL); @@ -492,6 +501,7 @@ main(int argc, char **argv) int verbose = 0; int argerr = 0; int timeout = 120; + int delay = -1; int option_index = 0; int fence_level = 0; int no_connect = 0; @@ -574,6 +584,9 @@ main(int argc, char **argv) case 't': timeout = crm_atoi(optarg, NULL); break; + case 'y': + delay = crm_atoi(optarg, NULL); + break; case 'B': case 'F': case 'U': @@ -760,13 +773,13 @@ main(int argc, char **argv) rc = st->cmds->confirm(st, st_opts, target); break; case 'B': - rc = mainloop_fencing(st, target, "reboot", timeout, tolerance); + rc = mainloop_fencing(st, target, "reboot", timeout, tolerance, delay); break; case 'F': - rc = mainloop_fencing(st, target, "off", timeout, tolerance); + rc = mainloop_fencing(st, target, "off", timeout, tolerance, delay); break; case 'U': - rc = mainloop_fencing(st, target, "on", timeout, tolerance); + rc = mainloop_fencing(st, target, "on", timeout, tolerance, delay); break; case 'h': show_last_fenced(as_nodeid, target); From 5c7136e938acdc4bbd6655f137b454afab0c6cdd Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 18 Mar 2020 15:48:32 +0100 Subject: [PATCH 07/15] Feature: fencer: handle any enforced fencing delay Enforced fencing delay takes precedence over any pcmk_delay_base/max configured for the corresponding fencing resources. Enforced fencing delay is applied only for the first device in the first fencing topology level. Consistently use g_timeout_add_seconds() for pcmk_delay_base/max as well. --- fencing/commands.c | 71 ++++++++++++++++++++++++++-------------------- fencing/internal.h | 4 +++ fencing/remote.c | 19 +++++++++++++ 3 files changed, 64 insertions(+), 30 deletions(-) diff --git a/fencing/commands.c b/fencing/commands.c index 66c805415aa..e71cb5cde68 100644 --- a/fencing/commands.c +++ b/fencing/commands.c @@ -80,7 +80,7 @@ typedef struct async_command_s { int default_timeout; /* seconds */ int timeout; /* seconds */ - int start_delay; /* milliseconds */ + int start_delay; /* seconds */ int delay_id; char *op; @@ -123,7 +123,7 @@ static int get_action_delay_max(stonith_device_t * device, const char * action) { const char *value = NULL; - int delay_max_ms = 0; + int delay_max = 0; if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) { return 0; @@ -131,17 +131,17 @@ get_action_delay_max(stonith_device_t * device, const char * action) value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_MAX); if (value) { - delay_max_ms = crm_get_msec(value); + delay_max = crm_get_msec(value) / 1000; } - return delay_max_ms; + return delay_max; } static int get_action_delay_base(stonith_device_t * device, const char * action) { const char *value = NULL; - int delay_base_ms = 0; + int delay_base = 0; if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) { return 0; @@ -149,10 +149,10 @@ get_action_delay_base(stonith_device_t * device, const char * action) value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_BASE); if (value) { - delay_base_ms = crm_get_msec(value); + delay_base = crm_get_msec(value) / 1000; } - return delay_base_ms; + return delay_base; } /*! @@ -243,6 +243,9 @@ create_async_command(xmlNode * msg) crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options)); crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout)); cmd->timeout = cmd->default_timeout; + // Default value -1 means no enforced fencing delay + cmd->start_delay = -1; + crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay)); cmd->origin = crm_element_value_copy(msg, F_ORIG); cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID); @@ -349,7 +352,7 @@ stonith_device_execute(stonith_device_t * device) if (pending_op && pending_op->delay_id) { crm_trace - ("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %dms", + ("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %ds", pending_op->action, pending_op->victim ? " targeting " : "", pending_op->victim ? pending_op->victim : "", device->id, pending_op->start_delay); @@ -464,6 +467,7 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) { int delay_max = 0; int delay_base = 0; + bool delay_enforced = (cmd->start_delay >= 0); CRM_CHECK(cmd != NULL, return); CRM_CHECK(device != NULL, return); @@ -496,30 +500,37 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) device->pending_ops = g_list_append(device->pending_ops, cmd); mainloop_set_trigger(device->work); - delay_max = get_action_delay_max(device, cmd->action); - delay_base = get_action_delay_base(device, cmd->action); - if (delay_max == 0) { - delay_max = delay_base; - } - if (delay_max < delay_base) { - crm_warn("Base-delay (%dms) is larger than max-delay (%dms) " - "for %s on %s - limiting to max-delay", - delay_base, delay_max, cmd->action, device->id); - delay_base = delay_max; + // No enforced fencing delay + if (delay_enforced == FALSE) { + delay_max = get_action_delay_max(device, cmd->action); + delay_base = get_action_delay_base(device, cmd->action); + if (delay_max == 0) { + delay_max = delay_base; + } + if (delay_max < delay_base) { + crm_warn("Base-delay (%ds) is larger than max-delay (%ds) " + "for %s on %s - limiting to max-delay", + delay_base, delay_max, cmd->action, device->id); + delay_base = delay_max; + } + if (delay_max > 0) { + // coverity[dont_call] We're not using rand() for security + cmd->start_delay = + ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) + + delay_base; + } } - if (delay_max > 0) { - // coverity[dont_call] We're not using rand() for security - cmd->start_delay = - ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) - + delay_base; - crm_notice("Delaying '%s' action%s%s on %s for %dms (timeout=%ds, base=%dms, " - "max=%dms)", - cmd->action, - cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "", - device->id, cmd->start_delay, cmd->timeout, - delay_base, delay_max); + + if (cmd->start_delay > 0) { + crm_notice("Delaying '%s' action%s%s on %s for %s%ds (timeout=%ds, base=%ds, " + "max=%ds)", + cmd->action, + cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "", + device->id, delay_enforced ? "enforced " : "", + cmd->start_delay, cmd->timeout, + delay_base, delay_max); cmd->delay_id = - g_timeout_add(cmd->start_delay, start_delay_helper, cmd); + g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd); } } diff --git a/fencing/internal.h b/fencing/internal.h index a51b0e62802..03ecb15f680 100644 --- a/fencing/internal.h +++ b/fencing/internal.h @@ -98,6 +98,10 @@ typedef struct remote_fencing_op_s { * values associated with the devices this fencing operation may call */ gint total_timeout; + /*! Enforced fencing delay. + * Default value -1 means no enforced fencing delay. */ + int delay; + /*! Delegate is the node being asked to perform a fencing action * on behalf of the node that owns the remote operation. Some operations * will involve multiple delegates. This value represents the final delegate diff --git a/fencing/remote.c b/fencing/remote.c index 17df700c105..615fec4f194 100644 --- a/fencing/remote.c +++ b/fencing/remote.c @@ -835,6 +835,11 @@ stonith_topology_next(remote_fencing_op_t * op) op->client_name, op->originator, op->id); set_op_device_list(op, tp->levels[op->level]); + // The enforced delay has been applied for the first fencing level + if (op->level > 1 && op->delay > 0) { + op->delay = 0; + } + if (g_list_next(op->devices_list) && safe_str_eq(op->action, "reboot")) { /* A reboot has been requested for a topology level with multiple * devices. Instead of rebooting the devices sequentially, we will @@ -993,6 +998,10 @@ create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer) crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout)); + // Default value -1 means no enforced fencing delay + op->delay = -1; + crm_element_value_int(request, F_STONITH_DELAY, &(op->delay)); + if (peer && dev) { op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID); } else { @@ -1440,6 +1449,12 @@ advance_op_topology(remote_fencing_op_t *op, const char *device, xmlNode *msg, /* Necessary devices remain, so execute the next one */ crm_trace("Next targeting %s on behalf of %s@%s (rc was %d)", op->target, op->originator, op->client_name, rc); + + // The enforced delay has been applied for the first device + if (op->delay > 0) { + op->delay = 0; + } + call_remote_stonith(op, NULL); } else { /* We're done with all devices and phases, so finalize operation */ @@ -1495,6 +1510,10 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer) crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options); + if (op->delay >= 0) { + crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay); + } + if (device) { timeout_one = TIMEOUT_MULTIPLY_FACTOR * get_device_timeout(op, peer, device); From 432ee3a6da93e32577a2e259a18eaa88a60894e1 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 18 Mar 2020 15:55:18 +0100 Subject: [PATCH 08/15] Test: fencer: add cpg_topology_delay test to verify enforced fencing delay with fencing topology --- fencing/regression.py.in | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/fencing/regression.py.in b/fencing/regression.py.in index 28a4e2fca1c..66fa05fec36 100644 --- a/fencing/regression.py.in +++ b/fencing/regression.py.in @@ -952,6 +952,35 @@ class Tests(object): test.add_stonith_neg_log_pattern("does not advertise support for 'reboot', performing 'off'") test.add_stonith_log_pattern("with device 'true1' returned: 0 (OK)") + # make sure enforced fencing delay is applied only for the first device in the first level + for test_type in test_types: + if test_type["use_cpg"] == 0: + continue + + test = self.new_test("%s_topology_delay" % test_type["prefix"], + "Verify enforced fencing delay is applied only for the first device in the first level.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "-R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "-R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "-R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "-R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + + test.add_cmd("stonith_admin", "-r node3 -i 1 -v true1") + test.add_cmd("stonith_admin", "-r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "-r node3 -i 2 -v true2") + test.add_cmd("stonith_admin", "-r node3 -i 2 -v true3") + + test.add_cmd("stonith_admin", "-F node3 --delay 1") + + test.add_stonith_log_pattern("Delaying 'off' action targeting node3 on true1 for enforced 1s") + test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on false1") + test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on true2") + test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on true3") + def build_nodeid_tests(self): """ Register tests that use a corosync node id """ From d7bab0edf02306b227e7d6b5bbac43f8b0a7883c Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 18 Mar 2020 16:30:35 +0100 Subject: [PATCH 09/15] Doc: Pacemaker Explained: document priority-fencing-delay cluster option --- doc/Pacemaker_Explained/en-US/Ch-Options.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt index 7981cd110c5..db532bf09e6 100644 --- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt +++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt @@ -241,6 +241,22 @@ indexterm:[Cluster,Option,concurrent-fencing] Is the cluster allowed to initiate multiple fence actions concurrently? '(since 1.1.15)' +| priority-fencing-delay | | +indexterm:[priority-fencing-delay,Cluster Option] +indexterm:[Cluster,Option,priority-fencing-delay] +Enforce specified delay for the fencings that are targeting the lost +nodes with the highest total resource priority in case we don't +have the majority of the nodes in our cluster partition, so that +the more significant nodes potentially win any fencing match, +which is especially meaningful under split-brain of 2-node +cluster. A promoted resource instance takes the base priority + 1 +on calculation if the base priority is not 0. If all the nodes +have equal priority, then any pcmk_delay_base/max configured for +the corresponding fencing resources will be applied. Otherwise as +long as it's set, even if to 0, it takes precedence over any +configured pcmk_delay_base/max. By default, priority fencing +delay is disabled. + | cluster-delay | 60s | indexterm:[cluster-delay,Cluster Option] indexterm:[Cluster,Option,cluster-delay] From 59d220b1a699ceac02a9f801d8677fe45da06bca Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 1 Apr 2020 12:23:29 +0200 Subject: [PATCH 10/15] Feature: scheduler: priority-fencing-delay defaults to 0 meaning disabled This commit also documents the upcoming new behavior as discussed from: https://github.com/ClusterLabs/pacemaker/pull/2012 Any static/random delays that are introduced by `pcmk_delay_base/max` configured for the corresponding fencing resources will be added to this delay. This delay should be significantly greater than, safely twice, the maximum `pcmk_delay_base/max`. By default, priority fencing delay is disabled. --- doc/Pacemaker_Explained/en-US/Ch-Options.txt | 16 ++++++++-------- include/crm/pengine/status.h | 2 +- lib/pengine/common.c | 18 +++++++++--------- lib/pengine/unpack.c | 2 -- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt index db532bf09e6..348986ce61e 100644 --- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt +++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt @@ -241,21 +241,21 @@ indexterm:[Cluster,Option,concurrent-fencing] Is the cluster allowed to initiate multiple fence actions concurrently? '(since 1.1.15)' -| priority-fencing-delay | | +| priority-fencing-delay | 0 | indexterm:[priority-fencing-delay,Cluster Option] indexterm:[Cluster,Option,priority-fencing-delay] -Enforce specified delay for the fencings that are targeting the lost +Apply specified delay for the fencings that are targeting the lost nodes with the highest total resource priority in case we don't have the majority of the nodes in our cluster partition, so that the more significant nodes potentially win any fencing match, which is especially meaningful under split-brain of 2-node cluster. A promoted resource instance takes the base priority + 1 -on calculation if the base priority is not 0. If all the nodes -have equal priority, then any pcmk_delay_base/max configured for -the corresponding fencing resources will be applied. Otherwise as -long as it's set, even if to 0, it takes precedence over any -configured pcmk_delay_base/max. By default, priority fencing -delay is disabled. +on calculation if the base priority is not 0. Any static/random +delays that are introduced by `pcmk_delay_base/max` configured +for the corresponding fencing resources will be added to this +delay. This delay should be significantly greater than, safely +twice, the maximum `pcmk_delay_base/max`. By default, priority +fencing delay is disabled. '(since 1.1.23)' | cluster-delay | 60s | indexterm:[cluster-delay,Cluster Option] diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h index 2af38596d0a..b1f858eecb9 100644 --- a/include/crm/pengine/status.h +++ b/include/crm/pengine/status.h @@ -153,7 +153,7 @@ typedef struct pe_working_set_s { GList *param_check; // History entries that need to be checked GList *stop_needed; // Containers that need stop actions int ninstances; // Total number of resource instances - int priority_fencing_delay; // Enforced priority fencing delay + int priority_fencing_delay; // Priority fencing delay } pe_working_set_t; enum pe_check_parameters { diff --git a/lib/pengine/common.c b/lib/pengine/common.c index 08de8f765cd..c15532672fc 100644 --- a/lib/pengine/common.c +++ b/lib/pengine/common.c @@ -123,20 +123,20 @@ pe_cluster_option pe_opts[] = { "STONITH unseen nodes", "Advanced Use Only! Not using the default is very unsafe!" }, { XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL, - NULL, &check_timer, - "Enforced fencing delay targeting the lost nodes with the highest total resource priority", - "Enforce specified delay for the fencings that are targeting the lost " + "0", &check_timer, + "Apply fencing delay targeting the lost nodes with the highest total resource priority", + "Apply specified delay for the fencings that are targeting the lost " "nodes with the highest total resource priority in case we don't " "have the majority of the nodes in our cluster partition, so that " "the more significant nodes potentially win any fencing match, " "which is especially meaningful under split-brain of 2-node " "cluster. A promoted resource instance takes the base priority + 1 " - "on calculation if the base priority is not 0. If all the nodes " - "have equal priority, then any pcmk_delay_base/max configured for " - "the corresponding fencing resources will be applied. Otherwise as " - "long as it's set, even if to 0, it takes precedence over any " - "configured pcmk_delay_base/max. By default, priority fencing " - "delay is disabled." + "on calculation if the base priority is not 0. Any static/random " + "delays that are introduced by `pcmk_delay_base/max` configured " + "for the corresponding fencing resources will be added to this " + "delay. This delay should be significantly greater than, safely " + "twice, the maximum `pcmk_delay_base/max`. By default, priority " + "fencing delay is disabled." }, /* Timeouts etc */ diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c index 1cd0faecaa6..66833128996 100644 --- a/lib/pengine/unpack.c +++ b/lib/pengine/unpack.c @@ -226,8 +226,6 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) crm_debug("Concurrent fencing is %s", is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled"); - // Default value -1 means `priority-fencing-delay` is disabled - data_set->priority_fencing_delay = -1; value = pe_pref(data_set->config_hash, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); if (value) { From 69cc3f6dda12f6af7fa667d706dd80e90ff09ddf Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 1 Apr 2020 12:55:51 +0200 Subject: [PATCH 11/15] Feature: scheduler: do not differentiate the case where all the nodes have equal priority In any cases, priority-fencing-delay won't take precedence over any configured pcmk_delay_base/max. --- lib/pengine/utils.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c index 10c7b0b67ec..1fe4b19380a 100644 --- a/lib/pengine/utils.c +++ b/lib/pengine/utils.c @@ -2352,17 +2352,17 @@ node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set) GListPtr gIter = NULL; // `priority-fencing-delay` is disabled - if (data_set->priority_fencing_delay < 0) { - return -1; + if (data_set->priority_fencing_delay <= 0) { + return 0; } - /* No need to delay fencing if the fencing target is not a normal cluster + /* No need to request a delay if the fencing target is not a normal cluster * member, for example if it's a remote node or a guest node. */ if (node->details->type != node_member) { return 0; } - // No need to delay fencing if the fencing target is in our partition + // No need to request a delay if the fencing target is in our partition if (node->details->online) { return 0; } @@ -2399,7 +2399,7 @@ node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set) /* All the nodes have equal priority. * Any configured corresponding `pcmk_delay_base/max` will be applied. */ if (lowest_priority == top_priority) { - return -1; + return 0; } if (node->details->priority < top_priority) { @@ -2482,7 +2482,7 @@ pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, free(op_key); } - if (data_set->priority_fencing_delay >= 0 + if (data_set->priority_fencing_delay > 0 /* It's a suitable case where `priority-fencing-delay` applies. * At least add `priority-fencing-delay` field as an indicator. */ From 98a5ae14b8193fdc0c23e85078ebd46487d54d1a Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 1 Apr 2020 13:12:29 +0200 Subject: [PATCH 12/15] Feature: fencer: any delays from pcmk_delay_base/max are added to requested fencing delay Requested fencing delay doesn't take precedence over any configured pcmk_delay_base/max. A delay value -1 now means disable also any static/random fencing delays from pcmk_delay_base/max. It's not used by any consumers for now. --- fencing/commands.c | 54 ++++++++++++++++++++-------------------- fencing/internal.h | 4 +-- fencing/remote.c | 13 +++------- include/crm/stonith-ng.h | 5 ++-- lib/fencing/st_client.c | 7 ++---- 5 files changed, 38 insertions(+), 45 deletions(-) diff --git a/fencing/commands.c b/fencing/commands.c index e71cb5cde68..7a9c1f5c8cd 100644 --- a/fencing/commands.c +++ b/fencing/commands.c @@ -243,8 +243,7 @@ create_async_command(xmlNode * msg) crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options)); crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout)); cmd->timeout = cmd->default_timeout; - // Default value -1 means no enforced fencing delay - cmd->start_delay = -1; + // Value -1 means disable any static/random fencing delays crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay)); cmd->origin = crm_element_value_copy(msg, F_ORIG); @@ -467,7 +466,7 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) { int delay_max = 0; int delay_base = 0; - bool delay_enforced = (cmd->start_delay >= 0); + int requested_delay = cmd->start_delay; CRM_CHECK(cmd != NULL, return); CRM_CHECK(device != NULL, return); @@ -500,35 +499,36 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) device->pending_ops = g_list_append(device->pending_ops, cmd); mainloop_set_trigger(device->work); - // No enforced fencing delay - if (delay_enforced == FALSE) { - delay_max = get_action_delay_max(device, cmd->action); - delay_base = get_action_delay_base(device, cmd->action); - if (delay_max == 0) { - delay_max = delay_base; - } - if (delay_max < delay_base) { - crm_warn("Base-delay (%ds) is larger than max-delay (%ds) " - "for %s on %s - limiting to max-delay", - delay_base, delay_max, cmd->action, device->id); - delay_base = delay_max; - } - if (delay_max > 0) { - // coverity[dont_call] We're not using rand() for security - cmd->start_delay = - ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) - + delay_base; - } + // Value -1 means disable any static/random fencing delays + if (requested_delay < 0) { + return; + } + + delay_max = get_action_delay_max(device, cmd->action); + delay_base = get_action_delay_base(device, cmd->action); + if (delay_max == 0) { + delay_max = delay_base; + } + if (delay_max < delay_base) { + crm_warn("Base-delay (%ds) is larger than max-delay (%ds) " + "for %s on %s - limiting to max-delay", + delay_base, delay_max, cmd->action, device->id); + delay_base = delay_max; + } + if (delay_max > 0) { + // coverity[dont_call] We're not using rand() for security + cmd->start_delay += + ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) + + delay_base; } if (cmd->start_delay > 0) { - crm_notice("Delaying '%s' action%s%s on %s for %s%ds (timeout=%ds, base=%ds, " - "max=%ds)", + crm_notice("Delaying '%s' action%s%s on %s for %ds (timeout=%ds, " + "requested_delay=%ds, base=%ds, max=%ds)", cmd->action, cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "", - device->id, delay_enforced ? "enforced " : "", - cmd->start_delay, cmd->timeout, - delay_base, delay_max); + device->id, cmd->start_delay, cmd->timeout, + requested_delay, delay_base, delay_max); cmd->delay_id = g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd); } diff --git a/fencing/internal.h b/fencing/internal.h index 03ecb15f680..16b0c33dcbc 100644 --- a/fencing/internal.h +++ b/fencing/internal.h @@ -98,8 +98,8 @@ typedef struct remote_fencing_op_s { * values associated with the devices this fencing operation may call */ gint total_timeout; - /*! Enforced fencing delay. - * Default value -1 means no enforced fencing delay. */ + /*! Requested fencing delay. + * Value -1 means disable any static/random fencing delays. */ int delay; /*! Delegate is the node being asked to perform a fencing action diff --git a/fencing/remote.c b/fencing/remote.c index 615fec4f194..9658b65d4fd 100644 --- a/fencing/remote.c +++ b/fencing/remote.c @@ -835,7 +835,7 @@ stonith_topology_next(remote_fencing_op_t * op) op->client_name, op->originator, op->id); set_op_device_list(op, tp->levels[op->level]); - // The enforced delay has been applied for the first fencing level + // The requested delay has been applied for the first fencing level if (op->level > 1 && op->delay > 0) { op->delay = 0; } @@ -997,9 +997,7 @@ create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer) op = calloc(1, sizeof(remote_fencing_op_t)); crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout)); - - // Default value -1 means no enforced fencing delay - op->delay = -1; + // Value -1 means disable any static/random fencing delays crm_element_value_int(request, F_STONITH_DELAY, &(op->delay)); if (peer && dev) { @@ -1450,7 +1448,7 @@ advance_op_topology(remote_fencing_op_t *op, const char *device, xmlNode *msg, crm_trace("Next targeting %s on behalf of %s@%s (rc was %d)", op->target, op->originator, op->client_name, rc); - // The enforced delay has been applied for the first device + // The requested delay has been applied for the first device if (op->delay > 0) { op->delay = 0; } @@ -1509,10 +1507,7 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer) crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name); crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options); - - if (op->delay >= 0) { - crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay); - } + crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay); if (device) { timeout_one = TIMEOUT_MULTIPLY_FACTOR * diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h index 02b751d6676..0699091a314 100644 --- a/include/crm/stonith-ng.h +++ b/include/crm/stonith-ng.h @@ -405,7 +405,7 @@ typedef struct stonith_api_operations_s char **error_output); /*! - * \brief Issue a fencing action against a node with enforced fencing delay. + * \brief Issue a fencing action against a node with requested fencing delay. * * \note Possible actions are, 'on', 'off', and 'reboot'. * @@ -415,7 +415,8 @@ typedef struct stonith_api_operations_s * \param action, The fencing action to take * \param timeout, The default per device timeout to use with each device * capable of fencing the target. - * \param delay, Any enforced fencing delay. -1 to disable + * \param delay, Apply a fencing delay. Value -1 means disable also any + * static/random fencing delays from pcmk_delay_base/max * * \retval 0 success * \retval negative error code on failure. diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index e451582040e..3a182428569 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1189,10 +1189,7 @@ stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char * crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance); - - if (delay >= 0) { - crm_xml_add_int(data, F_STONITH_DELAY, delay); - } + crm_xml_add_int(data, F_STONITH_DELAY, delay); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); @@ -1205,7 +1202,7 @@ stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const int timeout, int tolerance) { return stonith_api_fence_with_delay(stonith, call_options, node, action, - timeout, tolerance, -1); + timeout, tolerance, 0); } static int From e6beace0e31e4a0188922e3dc156a72abf8f04fa Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 1 Apr 2020 15:04:34 +0200 Subject: [PATCH 13/15] Feature: stonith_admin: --delay option defaults to 0 This commit also documents the current behavior in the help: - Any static/random delays from pcmk_delay_base/max will be added to requested fencing delay. - A delay value -1 now means disable also any static/random fencing delays from pcmk_delay_base/max. --- fencing/admin.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fencing/admin.c b/fencing/admin.c index 1c183de1c9d..445b7419fca 100644 --- a/fencing/admin.c +++ b/fencing/admin.c @@ -177,8 +177,10 @@ static struct crm_option long_options[] = { "\t\t\tused with most commands)." }, { "delay", required_argument, NULL, 'y', - "Enforced fencing delay in seconds (default -1 (disabled);\n" - "\t\t\twith --fence, --reboot, --unfence)." + "Apply a fencing delay in seconds. Any static/random delays from\n" + "\t\t\tpcmk_delay_base/max will be added, otherwise all\n" + "\t\t\tdisabled with the value -1\n" + "\t\t\t(default 0; with --fence, --reboot, --unfence)." }, { "as-node-id", no_argument, NULL, 'n', "(Advanced) The supplied node is the corosync node ID\n" @@ -501,7 +503,7 @@ main(int argc, char **argv) int verbose = 0; int argerr = 0; int timeout = 120; - int delay = -1; + int delay = 0; int option_index = 0; int fence_level = 0; int no_connect = 0; From ec67feb4b69f28cb63fc08fb1999f748b5e545fa Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 1 Apr 2020 15:52:50 +0200 Subject: [PATCH 14/15] Feature: controller: requested priority fencing delay defaults to 0 --- crmd/te_actions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crmd/te_actions.c b/crmd/te_actions.c index 8fcfcd925c0..45bbf31b9b0 100644 --- a/crmd/te_actions.c +++ b/crmd/te_actions.c @@ -198,7 +198,7 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) rc = stonith_api->cmds->fence_with_delay(stonith_api, options, target, type, (int) (transition_graph->stonith_timeout / 1000), - 0, crm_atoi(priority_delay, "-1")); + 0, crm_atoi(priority_delay, "0")); stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000, st_opt_timeout_updates, From 73676581344218f17e0b3c47751a351fa28e7887 Mon Sep 17 00:00:00 2001 From: "Gao,Yan" Date: Wed, 1 Apr 2020 19:17:48 +0200 Subject: [PATCH 15/15] Test: fencer: update cpg_topology_delay test to also verify pcmk_delay_base is added This commit also updates log patterns for the log changes. --- fencing/regression.py.in | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/fencing/regression.py.in b/fencing/regression.py.in index 66fa05fec36..21538b6e44e 100644 --- a/fencing/regression.py.in +++ b/fencing/regression.py.in @@ -952,22 +952,23 @@ class Tests(object): test.add_stonith_neg_log_pattern("does not advertise support for 'reboot', performing 'off'") test.add_stonith_log_pattern("with device 'true1' returned: 0 (OK)") - # make sure enforced fencing delay is applied only for the first device in the first level + # make sure requested fencing delay is applied only for the first device in the first level + # make sure static delay from pcmk_delay_base is added for test_type in test_types: if test_type["use_cpg"] == 0: continue test = self.new_test("%s_topology_delay" % test_type["prefix"], - "Verify enforced fencing delay is applied only for the first device in the first level.", + "Verify requested fencing delay is applied only for the first device in the first level and pcmk_delay_base is added.", test_type["use_cpg"]) test.add_cmd("stonith_admin", - "-R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + "-R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"") test.add_cmd("stonith_admin", - "-R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + "-R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"") test.add_cmd("stonith_admin", - "-R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + "-R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") test.add_cmd("stonith_admin", - "-R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + "-R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") test.add_cmd("stonith_admin", "-r node3 -i 1 -v true1") test.add_cmd("stonith_admin", "-r node3 -i 1 -v false1") @@ -976,8 +977,8 @@ class Tests(object): test.add_cmd("stonith_admin", "-F node3 --delay 1") - test.add_stonith_log_pattern("Delaying 'off' action targeting node3 on true1 for enforced 1s") - test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on false1") + test.add_stonith_log_pattern("Delaying 'off' action targeting node3 on true1 for 2s (timeout=120s, requested_delay=1s, base=1s, max=1s)") + test.add_stonith_log_pattern("Delaying 'off' action targeting node3 on false1 for 1s (timeout=120s, requested_delay=0s, base=1s, max=1s)") test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on true2") test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on true3")