diff --git a/crmd/te_actions.c b/crmd/te_actions.c index 19bb1999017..45bbf31b9b0 100644 --- a/crmd/te_actions.c +++ b/crmd/te_actions.c @@ -164,6 +164,7 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) const char *uuid = NULL; const char *target = NULL; const char *type = NULL; + const char *priority_delay = NULL; gboolean invalid_action = FALSE; enum stonith_call_options options = st_opt_none; @@ -182,9 +183,11 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) return FALSE; } + priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); + crm_notice("Requesting fencing (%s) of node %s " - CRM_XS " action=%s timeout=%d", - type, target, id, transition_graph->stonith_timeout); + CRM_XS " action=%s timeout=%u priority_delay=%s", + type, target, id, transition_graph->stonith_timeout, priority_delay); /* Passing NULL means block until we can connect... */ te_connect_stonith(NULL); @@ -193,8 +196,9 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action) options |= st_opt_allow_suicide; } - rc = stonith_api->cmds->fence(stonith_api, options, target, type, - transition_graph->stonith_timeout / 1000, 0); + rc = stonith_api->cmds->fence_with_delay(stonith_api, options, target, type, + (int) (transition_graph->stonith_timeout / 1000), + 0, crm_atoi(priority_delay, "0")); stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000, st_opt_timeout_updates, diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt index 7981cd110c5..348986ce61e 100644 --- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt +++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt @@ -241,6 +241,22 @@ indexterm:[Cluster,Option,concurrent-fencing] Is the cluster allowed to initiate multiple fence actions concurrently? '(since 1.1.15)' +| priority-fencing-delay | 0 | +indexterm:[priority-fencing-delay,Cluster Option] +indexterm:[Cluster,Option,priority-fencing-delay] +Apply specified delay for the fencings that are targeting the lost +nodes with the highest total resource priority in case we don't +have the majority of the nodes in our cluster partition, so that +the more significant nodes potentially win any fencing match, +which is especially meaningful under split-brain of 2-node +cluster. A promoted resource instance takes the base priority + 1 +on calculation if the base priority is not 0. Any static/random +delays that are introduced by `pcmk_delay_base/max` configured +for the corresponding fencing resources will be added to this +delay. This delay should be significantly greater than, safely +twice, the maximum `pcmk_delay_base/max`. By default, priority +fencing delay is disabled. '(since 1.1.23)' + | cluster-delay | 60s | indexterm:[cluster-delay,Cluster Option] indexterm:[Cluster,Option,cluster-delay] diff --git a/fencing/admin.c b/fencing/admin.c index 5fedb7b5a62..445b7419fca 100644 --- a/fencing/admin.c +++ b/fencing/admin.c @@ -176,6 +176,12 @@ static struct crm_option long_options[] = { "Operation timeout in seconds (default 120;\n" "\t\t\tused with most commands)." }, + { "delay", required_argument, NULL, 'y', + "Apply a fencing delay in seconds. Any static/random delays from\n" + "\t\t\tpcmk_delay_base/max will be added, otherwise all\n" + "\t\t\tdisabled with the value -1\n" + "\t\t\t(default 0; with --fence, --reboot, --unfence)." + }, { "as-node-id", no_argument, NULL, 'n', "(Advanced) The supplied node is the corosync node ID\n" "\t\t\t(with --last)." @@ -201,6 +207,7 @@ struct { char *name; int timeout; int tolerance; + int delay; int rc; } async_fence_data; @@ -265,11 +272,13 @@ async_fence_helper(gpointer user_data) st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, notify_callback); - call_id = st->cmds->fence(st, - st_opt_allow_suicide, - async_fence_data.target, - async_fence_data.action, - async_fence_data.timeout, async_fence_data.tolerance); + call_id = st->cmds->fence_with_delay(st, + st_opt_allow_suicide, + async_fence_data.target, + async_fence_data.action, + async_fence_data.timeout, + async_fence_data.tolerance, + async_fence_data.delay); if (call_id < 0) { g_main_loop_quit(mainloop); @@ -285,7 +294,8 @@ async_fence_helper(gpointer user_data) } static int -mainloop_fencing(stonith_t * st, const char *target, const char *action, int timeout, int tolerance) +mainloop_fencing(stonith_t * st, const char *target, const char *action, + int timeout, int tolerance, int delay) { crm_trigger_t *trig; @@ -294,6 +304,7 @@ mainloop_fencing(stonith_t * st, const char *target, const char *action, int tim async_fence_data.action = action; async_fence_data.timeout = timeout; async_fence_data.tolerance = tolerance; + async_fence_data.delay = delay; async_fence_data.rc = -1; trig = mainloop_add_trigger(G_PRIORITY_HIGH, async_fence_helper, NULL); @@ -492,6 +503,7 @@ main(int argc, char **argv) int verbose = 0; int argerr = 0; int timeout = 120; + int delay = 0; int option_index = 0; int fence_level = 0; int no_connect = 0; @@ -574,6 +586,9 @@ main(int argc, char **argv) case 't': timeout = crm_atoi(optarg, NULL); break; + case 'y': + delay = crm_atoi(optarg, NULL); + break; case 'B': case 'F': case 'U': @@ -760,13 +775,13 @@ main(int argc, char **argv) rc = st->cmds->confirm(st, st_opts, target); break; case 'B': - rc = mainloop_fencing(st, target, "reboot", timeout, tolerance); + rc = mainloop_fencing(st, target, "reboot", timeout, tolerance, delay); break; case 'F': - rc = mainloop_fencing(st, target, "off", timeout, tolerance); + rc = mainloop_fencing(st, target, "off", timeout, tolerance, delay); break; case 'U': - rc = mainloop_fencing(st, target, "on", timeout, tolerance); + rc = mainloop_fencing(st, target, "on", timeout, tolerance, delay); break; case 'h': show_last_fenced(as_nodeid, target); diff --git a/fencing/commands.c b/fencing/commands.c index 66c805415aa..7a9c1f5c8cd 100644 --- a/fencing/commands.c +++ b/fencing/commands.c @@ -80,7 +80,7 @@ typedef struct async_command_s { int default_timeout; /* seconds */ int timeout; /* seconds */ - int start_delay; /* milliseconds */ + int start_delay; /* seconds */ int delay_id; char *op; @@ -123,7 +123,7 @@ static int get_action_delay_max(stonith_device_t * device, const char * action) { const char *value = NULL; - int delay_max_ms = 0; + int delay_max = 0; if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) { return 0; @@ -131,17 +131,17 @@ get_action_delay_max(stonith_device_t * device, const char * action) value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_MAX); if (value) { - delay_max_ms = crm_get_msec(value); + delay_max = crm_get_msec(value) / 1000; } - return delay_max_ms; + return delay_max; } static int get_action_delay_base(stonith_device_t * device, const char * action) { const char *value = NULL; - int delay_base_ms = 0; + int delay_base = 0; if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) { return 0; @@ -149,10 +149,10 @@ get_action_delay_base(stonith_device_t * device, const char * action) value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_BASE); if (value) { - delay_base_ms = crm_get_msec(value); + delay_base = crm_get_msec(value) / 1000; } - return delay_base_ms; + return delay_base; } /*! @@ -243,6 +243,8 @@ create_async_command(xmlNode * msg) crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options)); crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout)); cmd->timeout = cmd->default_timeout; + // Value -1 means disable any static/random fencing delays + crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay)); cmd->origin = crm_element_value_copy(msg, F_ORIG); cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID); @@ -349,7 +351,7 @@ stonith_device_execute(stonith_device_t * device) if (pending_op && pending_op->delay_id) { crm_trace - ("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %dms", + ("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %ds", pending_op->action, pending_op->victim ? " targeting " : "", pending_op->victim ? pending_op->victim : "", device->id, pending_op->start_delay); @@ -464,6 +466,7 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) { int delay_max = 0; int delay_base = 0; + int requested_delay = cmd->start_delay; CRM_CHECK(cmd != NULL, return); CRM_CHECK(device != NULL, return); @@ -496,30 +499,38 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) device->pending_ops = g_list_append(device->pending_ops, cmd); mainloop_set_trigger(device->work); + // Value -1 means disable any static/random fencing delays + if (requested_delay < 0) { + return; + } + delay_max = get_action_delay_max(device, cmd->action); delay_base = get_action_delay_base(device, cmd->action); if (delay_max == 0) { delay_max = delay_base; } if (delay_max < delay_base) { - crm_warn("Base-delay (%dms) is larger than max-delay (%dms) " + crm_warn("Base-delay (%ds) is larger than max-delay (%ds) " "for %s on %s - limiting to max-delay", delay_base, delay_max, cmd->action, device->id); delay_base = delay_max; } if (delay_max > 0) { // coverity[dont_call] We're not using rand() for security - cmd->start_delay = + cmd->start_delay += ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) + delay_base; - crm_notice("Delaying '%s' action%s%s on %s for %dms (timeout=%ds, base=%dms, " - "max=%dms)", - cmd->action, - cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "", - device->id, cmd->start_delay, cmd->timeout, - delay_base, delay_max); + } + + if (cmd->start_delay > 0) { + crm_notice("Delaying '%s' action%s%s on %s for %ds (timeout=%ds, " + "requested_delay=%ds, base=%ds, max=%ds)", + cmd->action, + cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "", + device->id, cmd->start_delay, cmd->timeout, + requested_delay, delay_base, delay_max); cmd->delay_id = - g_timeout_add(cmd->start_delay, start_delay_helper, cmd); + g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd); } } diff --git a/fencing/internal.h b/fencing/internal.h index a51b0e62802..16b0c33dcbc 100644 --- a/fencing/internal.h +++ b/fencing/internal.h @@ -98,6 +98,10 @@ typedef struct remote_fencing_op_s { * values associated with the devices this fencing operation may call */ gint total_timeout; + /*! Requested fencing delay. + * Value -1 means disable any static/random fencing delays. */ + int delay; + /*! Delegate is the node being asked to perform a fencing action * on behalf of the node that owns the remote operation. Some operations * will involve multiple delegates. This value represents the final delegate diff --git a/fencing/regression.py.in b/fencing/regression.py.in index 28a4e2fca1c..21538b6e44e 100644 --- a/fencing/regression.py.in +++ b/fencing/regression.py.in @@ -952,6 +952,36 @@ class Tests(object): test.add_stonith_neg_log_pattern("does not advertise support for 'reboot', performing 'off'") test.add_stonith_log_pattern("with device 'true1' returned: 0 (OK)") + # make sure requested fencing delay is applied only for the first device in the first level + # make sure static delay from pcmk_delay_base is added + for test_type in test_types: + if test_type["use_cpg"] == 0: + continue + + test = self.new_test("%s_topology_delay" % test_type["prefix"], + "Verify requested fencing delay is applied only for the first device in the first level and pcmk_delay_base is added.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "-R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"") + test.add_cmd("stonith_admin", + "-R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"") + test.add_cmd("stonith_admin", + "-R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "-R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + + test.add_cmd("stonith_admin", "-r node3 -i 1 -v true1") + test.add_cmd("stonith_admin", "-r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "-r node3 -i 2 -v true2") + test.add_cmd("stonith_admin", "-r node3 -i 2 -v true3") + + test.add_cmd("stonith_admin", "-F node3 --delay 1") + + test.add_stonith_log_pattern("Delaying 'off' action targeting node3 on true1 for 2s (timeout=120s, requested_delay=1s, base=1s, max=1s)") + test.add_stonith_log_pattern("Delaying 'off' action targeting node3 on false1 for 1s (timeout=120s, requested_delay=0s, base=1s, max=1s)") + test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on true2") + test.add_stonith_neg_log_pattern("Delaying 'off' action targeting node3 on true3") + def build_nodeid_tests(self): """ Register tests that use a corosync node id """ diff --git a/fencing/remote.c b/fencing/remote.c index 17df700c105..9658b65d4fd 100644 --- a/fencing/remote.c +++ b/fencing/remote.c @@ -835,6 +835,11 @@ stonith_topology_next(remote_fencing_op_t * op) op->client_name, op->originator, op->id); set_op_device_list(op, tp->levels[op->level]); + // The requested delay has been applied for the first fencing level + if (op->level > 1 && op->delay > 0) { + op->delay = 0; + } + if (g_list_next(op->devices_list) && safe_str_eq(op->action, "reboot")) { /* A reboot has been requested for a topology level with multiple * devices. Instead of rebooting the devices sequentially, we will @@ -992,6 +997,8 @@ create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer) op = calloc(1, sizeof(remote_fencing_op_t)); crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout)); + // Value -1 means disable any static/random fencing delays + crm_element_value_int(request, F_STONITH_DELAY, &(op->delay)); if (peer && dev) { op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID); @@ -1440,6 +1447,12 @@ advance_op_topology(remote_fencing_op_t *op, const char *device, xmlNode *msg, /* Necessary devices remain, so execute the next one */ crm_trace("Next targeting %s on behalf of %s@%s (rc was %d)", op->target, op->originator, op->client_name, rc); + + // The requested delay has been applied for the first device + if (op->delay > 0) { + op->delay = 0; + } + call_remote_stonith(op, NULL); } else { /* We're done with all devices and phases, so finalize operation */ @@ -1494,6 +1507,7 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer) crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name); crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options); + crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay); if (device) { timeout_one = TIMEOUT_MULTIPLY_FACTOR * diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h index 24df230132c..264e0d1872f 100644 --- a/include/crm/fencing/internal.h +++ b/include/crm/fencing/internal.h @@ -57,6 +57,7 @@ xmlNode *create_device_registration_xml(const char *id, /*! Timeout period per a device execution */ # define F_STONITH_TIMEOUT "st_timeout" # define F_STONITH_TOLERANCE "st_tolerance" +# define F_STONITH_DELAY "st_delay" /*! Action specific timeout period returned in query of fencing devices. */ # define F_STONITH_ACTION_TIMEOUT "st_action_timeout" /*! Host in query result is not allowed to run this action */ diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h index de99959e462..7fa3c39f30f 100644 --- a/include/crm/msg_xml.h +++ b/include/crm/msg_xml.h @@ -378,6 +378,7 @@ # define XML_CONFIG_ATTR_FORCE_QUIT "shutdown-escalation" # define XML_CONFIG_ATTR_RECHECK "cluster-recheck-interval" # define XML_CONFIG_ATTR_FENCE_REACTION "fence-reaction" +# define XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY "priority-fencing-delay" # define XML_ALERT_ATTR_PATH "path" # define XML_ALERT_ATTR_TIMEOUT "timeout" diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h index 023ab92b7bc..7c3e2c09171 100644 --- a/include/crm/pengine/internal.h +++ b/include/crm/pengine/internal.h @@ -329,7 +329,7 @@ typedef struct op_digest_cache_s { op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, pe_working_set_t * data_set); -action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set); +pe_action_t *pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, bool priority_delay, pe_working_set_t * data_set); void trigger_unfencing( resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set); @@ -346,7 +346,7 @@ gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options, void * print_data, gboolean print_all); -void pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason); +void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay); node_t *pe_create_node(const char *id, const char *uname, const char *type, const char *score, pe_working_set_t * data_set); diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h index 8be84d57b75..b1f858eecb9 100644 --- a/include/crm/pengine/status.h +++ b/include/crm/pengine/status.h @@ -153,6 +153,7 @@ typedef struct pe_working_set_s { GList *param_check; // History entries that need to be checked GList *stop_needed; // Containers that need stop actions int ninstances; // Total number of resource instances + int priority_fencing_delay; // Priority fencing delay } pe_working_set_t; enum pe_check_parameters { @@ -202,6 +203,7 @@ struct node_shared_s { gboolean remote_was_fenced; gboolean remote_maintenance; /* what the remote-rsc is thinking */ gboolean unpacked; + int priority; // calculated based on the priority of resources running on the node }; struct node_s { diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h index 23f879b8053..0699091a314 100644 --- a/include/crm/stonith-ng.h +++ b/include/crm/stonith-ng.h @@ -404,6 +404,26 @@ typedef struct stonith_api_operations_s stonith_key_value_t *params, int timeout, char **output, char **error_output); + /*! + * \brief Issue a fencing action against a node with requested fencing delay. + * + * \note Possible actions are, 'on', 'off', and 'reboot'. + * + * \param st, stonith connection + * \param options, call options + * \param node, The target node to fence + * \param action, The fencing action to take + * \param timeout, The default per device timeout to use with each device + * capable of fencing the target. + * \param delay, Apply a fencing delay. Value -1 means disable also any + * static/random fencing delays from pcmk_delay_base/max + * + * \retval 0 success + * \retval negative error code on failure. + */ + int (*fence_with_delay)(stonith_t *st, int options, const char *node, const char *action, + int timeout, int tolerance, int delay); + } stonith_api_operations_t; struct stonith_s diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 4a1310f835c..3a182428569 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1178,8 +1178,8 @@ stonith_api_status(stonith_t * stonith, int call_options, const char *id, const } static int -stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, - int timeout, int tolerance) +stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node, + const char *action, int timeout, int tolerance, int delay) { int rc = 0; xmlNode *data = NULL; @@ -1189,6 +1189,7 @@ stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance); + crm_xml_add_int(data, F_STONITH_DELAY, delay); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); @@ -1196,6 +1197,14 @@ stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const return rc; } +static int +stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, + int timeout, int tolerance) +{ + return stonith_api_fence_with_delay(stonith, call_options, node, action, + timeout, tolerance, 0); +} + static int stonith_api_confirm(stonith_t * stonith, int call_options, const char *target) { @@ -1924,6 +1933,14 @@ stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNod crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout); crm_trace("Sending %s message to STONITH service, Timeout: %ds", op, timeout); + if (data) { + const char *delay_s = crm_element_value(data, F_STONITH_DELAY); + + if (delay_s) { + crm_xml_add(op_msg, F_STONITH_DELAY, delay_s); + } + } + rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, 1000 * (timeout + 60), &op_reply); free_xml(op_msg); @@ -2203,6 +2220,7 @@ stonith_api_new(void) new_stonith->cmds->monitor = stonith_api_monitor; new_stonith->cmds->status = stonith_api_status; new_stonith->cmds->fence = stonith_api_fence; + new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay; new_stonith->cmds->confirm = stonith_api_confirm; new_stonith->cmds->history = stonith_api_history; diff --git a/lib/pengine/common.c b/lib/pengine/common.c index e82434a37ee..c15532672fc 100644 --- a/lib/pengine/common.c +++ b/lib/pengine/common.c @@ -121,6 +121,23 @@ pe_cluster_option pe_opts[] = { "Allow performing fencing operations in parallel", NULL }, { "startup-fencing", "startup_fencing", "boolean", NULL, "true", &check_boolean, "STONITH unseen nodes", "Advanced Use Only! Not using the default is very unsafe!" }, + { + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL, + "0", &check_timer, + "Apply fencing delay targeting the lost nodes with the highest total resource priority", + "Apply specified delay for the fencings that are targeting the lost " + "nodes with the highest total resource priority in case we don't " + "have the majority of the nodes in our cluster partition, so that " + "the more significant nodes potentially win any fencing match, " + "which is especially meaningful under split-brain of 2-node " + "cluster. A promoted resource instance takes the base priority + 1 " + "on calculation if the base priority is not 0. Any static/random " + "delays that are introduced by `pcmk_delay_base/max` configured " + "for the corresponding fencing resources will be added to this " + "delay. This delay should be significantly greater than, safely " + "twice, the maximum `pcmk_delay_base/max`. By default, priority " + "fencing delay is disabled." + }, /* Timeouts etc */ { "cluster-delay", "transition_idle_timeout", "time", NULL, "60s", &check_time, diff --git a/lib/pengine/native.c b/lib/pengine/native.c index 3c9f8f59242..16bbcdc2f5d 100644 --- a/lib/pengine/native.c +++ b/lib/pengine/native.c @@ -33,6 +33,51 @@ is_multiply_active(pe_resource_t *rsc) return count > 1; } +static void +native_priority_to_node(pe_resource_t * rsc, pe_node_t * node) +{ + int priority = 0; + + if (rsc->priority == 0) { + return; + } + + if (rsc->role == RSC_ROLE_MASTER) { + // Promoted instance takes base priority + 1 + priority = rsc->priority + 1; + + } else { + priority = rsc->priority; + } + + node->details->priority += priority; + pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s)", + node->details->uname, node->details->priority, + rsc->role == RSC_ROLE_MASTER ? "promoted " : "", + rsc->id, rsc->priority, + rsc->role == RSC_ROLE_MASTER ? " + 1" : ""); + + /* Priority of a resource running on a guest node is added to the cluster + * node as well. */ + if (node->details->remote_rsc + && node->details->remote_rsc->container) { + GListPtr gIter = node->details->remote_rsc->container->running_on; + + for (; gIter != NULL; gIter = gIter->next) { + pe_node_t *a_node = gIter->data; + + a_node->details->priority += priority; + pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s) " + "from guest node '%s'", + a_node->details->uname, a_node->details->priority, + rsc->role == RSC_ROLE_MASTER ? "promoted " : "", + rsc->id, rsc->priority, + rsc->role == RSC_ROLE_MASTER ? " + 1" : "", + node->details->uname); + } + } +} + void native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set) { @@ -54,6 +99,8 @@ native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set) rsc->running_on = g_list_append(rsc->running_on, node); if (rsc->variant == pe_native) { node->details->running_rsc = g_list_append(node->details->running_rsc, rsc); + + native_priority_to_node(rsc, node); } if (rsc->variant == pe_native && node->details->maintenance) { diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c index 24e56f51205..66833128996 100644 --- a/lib/pengine/unpack.c +++ b/lib/pengine/unpack.c @@ -73,9 +73,11 @@ is_dangling_container_remote_node(node_t *node) * \param[in,out] data_set Current working set of cluster * \param[in,out] node Node to fence * \param[in] reason Text description of why fencing is needed + * \param[in] priority_delay Whether to consider `priority-fencing-delay` */ void -pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) +pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, + const char *reason, bool priority_delay) { CRM_CHECK(node, return); @@ -125,7 +127,8 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) reason); } node->details->unclean = TRUE; - pe_fence_op(node, NULL, TRUE, reason, data_set); + // No need to apply `priority-fencing-delay` for remote nodes + pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set); } else if (node->details->unclean) { crm_trace("Cluster node %s %s because %s", @@ -139,7 +142,7 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) pe_can_fence(data_set, node)? "will be fenced" : "is unclean", reason); node->details->unclean = TRUE; - pe_fence_op(node, NULL, TRUE, reason, data_set); + pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set); } } @@ -223,6 +226,13 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) crm_debug("Concurrent fencing is %s", is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled"); + value = pe_pref(data_set->config_hash, + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY); + if (value) { + data_set->priority_fencing_delay = crm_get_msec(value) / 1000; + crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay); + } + set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything); crm_debug("Stop all active resources: %s", is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false"); @@ -1278,7 +1288,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) /* Everything else should flow from this automatically * At least until the PE becomes able to migrate off healthy resources */ - pe_fence_node(data_set, this_node, "cluster does not have quorum"); + pe_fence_node(data_set, this_node, "cluster does not have quorum", FALSE); } } } @@ -1350,7 +1360,7 @@ determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_s } else { /* mark it unclean */ - pe_fence_node(data_set, this_node, "peer is unexpectedly down"); + pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE); crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s", crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state)); } @@ -1407,10 +1417,10 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat online = crm_is_true(is_peer); } else if (in_cluster == NULL) { - pe_fence_node(data_set, this_node, "peer has not been seen by the cluster"); + pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE); } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) { - pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria"); + pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria", FALSE); } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) { @@ -1429,14 +1439,15 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat online = FALSE; } else if (crm_is_true(in_cluster) == FALSE) { - pe_fence_node(data_set, this_node, "peer is no longer part of the cluster"); + // Consider `priority-fencing-delay` for lost nodes + pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE); } else if (crm_is_true(is_peer) == FALSE) { - pe_fence_node(data_set, this_node, "peer process is no longer available"); + pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE); /* Everything is running at this point, now check join state */ } else if (do_terminate) { - pe_fence_node(data_set, this_node, "termination was requested"); + pe_fence_node(data_set, this_node, "termination was requested", FALSE); } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) { crm_info("Node %s is active", this_node->details->uname); @@ -1448,7 +1459,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat this_node->details->pending = TRUE; } else { - pe_fence_node(data_set, this_node, "peer was in an unknown state"); + pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE); crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d", this_node->details->uname, crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown); @@ -2012,7 +2023,7 @@ process_rsc_state(resource_t * rsc, node_t * node, if (reason == NULL) { reason = crm_strdup_printf("%s is thought to be active there", rsc->id); } - pe_fence_node(data_set, node, reason); + pe_fence_node(data_set, node, reason, FALSE); } free(reason); } @@ -2034,7 +2045,7 @@ process_rsc_state(resource_t * rsc, node_t * node, * but also mark the node as unclean */ reason = crm_strdup_printf("%s failed there", rsc->id); - pe_fence_node(data_set, node, reason); + pe_fence_node(data_set, node, reason, FALSE); free(reason); break; @@ -2101,7 +2112,7 @@ process_rsc_state(resource_t * rsc, node_t * node, /* connection resource to baremetal resource failed in a way that * should result in fencing the remote-node. */ pe_fence_node(data_set, tmpnode, - "remote connection is unrecoverable"); + "remote connection is unrecoverable", FALSE); } } @@ -3155,7 +3166,7 @@ static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNod && remote_node && remote_node->details->unclean) { - action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, data_set); + pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL, FALSE, data_set); crm_notice("Waiting for %s to complete before clearing %s failure for remote node %s", fence?fence->uuid:"nil", task, rsc->id); order_actions(fence, clear_op, pe_order_implies_then); diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c index 558075a1452..1fe4b19380a 100644 --- a/lib/pengine/utils.c +++ b/lib/pengine/utils.c @@ -622,7 +622,7 @@ custom_action(resource_t * rsc, char *key, const char *task, if (is_set(action->rsc->flags, pe_rsc_managed) && save_action && a_task == stop_rsc && action->node->details->unclean == FALSE) { - pe_fence_node(data_set, action->node, "resource actions are unrunnable"); + pe_fence_node(data_set, action->node, "resource actions are unrunnable", FALSE); } } else if (action->node->details->pending) { @@ -2342,9 +2342,76 @@ find_unfencing_devices(GListPtr candidates, GListPtr matches) return matches; } +static int +node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set) +{ + int member_count = 0; + int online_count = 0; + int top_priority = 0; + int lowest_priority = 0; + GListPtr gIter = NULL; -action_t * -pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set) + // `priority-fencing-delay` is disabled + if (data_set->priority_fencing_delay <= 0) { + return 0; + } + + /* No need to request a delay if the fencing target is not a normal cluster + * member, for example if it's a remote node or a guest node. */ + if (node->details->type != node_member) { + return 0; + } + + // No need to request a delay if the fencing target is in our partition + if (node->details->online) { + return 0; + } + + for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + pe_node_t *n = gIter->data; + + if (n->details->type != node_member) { + continue; + } + + member_count ++; + + if (n->details->online) { + online_count++; + } + + if (member_count == 1 + || n->details->priority > top_priority) { + top_priority = n->details->priority; + } + + if (member_count == 1 + || n->details->priority < lowest_priority) { + lowest_priority = n->details->priority; + } + } + + // No need to delay if we have more than half of the cluster members + if (online_count > member_count / 2) { + return 0; + } + + /* All the nodes have equal priority. + * Any configured corresponding `pcmk_delay_base/max` will be applied. */ + if (lowest_priority == top_priority) { + return 0; + } + + if (node->details->priority < top_priority) { + return 0; + } + + return data_set->priority_fencing_delay; +} + +pe_action_t * +pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, + bool priority_delay, pe_working_set_t * data_set) { char *op_key = NULL; action_t *stonith_op = NULL; @@ -2415,6 +2482,29 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe free(op_key); } + if (data_set->priority_fencing_delay > 0 + + /* It's a suitable case where `priority-fencing-delay` applies. + * At least add `priority-fencing-delay` field as an indicator. */ + && (priority_delay + + /* Re-calculate priority delay for the suitable case when + * pe_fence_op() is called again by stage6() after node priority has + * been actually calculated with native_add_running() */ + || g_hash_table_lookup(stonith_op->meta, + XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) { + + /* Add `priority-fencing-delay` to the fencing op even if it's 0 for + * the targeting node. So that it takes precedence over any possible + * `pcmk_delay_base/max`. + */ + char *delay_s = crm_itoa(node_priority_fencing_delay(node, data_set)); + + g_hash_table_insert(stonith_op->meta, + strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY), + delay_s); + } + if(optional == FALSE && pe_can_fence(data_set, node)) { pe_action_required(stonith_op, NULL, reason); } else if(reason && stonith_op->reason == NULL) { @@ -2440,7 +2530,7 @@ trigger_unfencing( && node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { - action_t *unfence = pe_fence_op(node, "on", FALSE, reason, data_set); + pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set); if(dependency) { order_actions(unfence, dependency, pe_order_optional); diff --git a/pengine/allocate.c b/pengine/allocate.c index afe0f6ae55b..2d2ddcc7427 100644 --- a/pengine/allocate.c +++ b/pengine/allocate.c @@ -958,7 +958,7 @@ probe_resources(pe_working_set_t * data_set) if (is_baremetal_remote_node(node) && node->details->remote_rsc && (get_remote_node_state(node) == remote_state_failed)) { - pe_fence_node(data_set, node, "the connection is unrecoverable"); + pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE); } continue; @@ -1494,7 +1494,7 @@ fence_guest(pe_node_t *node, pe_working_set_t *data_set) /* Create a fence pseudo-event, so we have an event to order actions * against, and crmd can always detect it. */ - stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set); + stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, data_set); update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__); @@ -1503,7 +1503,7 @@ fence_guest(pe_node_t *node, pe_working_set_t *data_set) * (even though start might be closer to what is done for a real reboot). */ if(stop && is_set(stop->flags, pe_action_pseudo)) { - pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set); + pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, data_set); crm_info("Implying guest node %s is down (action %d) after %s fencing", node->details->uname, stonith_op->id, stop->node->details->uname); order_actions(parent_stonith_op, stonith_op, @@ -1596,7 +1596,7 @@ stage6(pe_working_set_t * data_set) if (node->details->unclean && need_stonith && pe_can_fence(data_set, node)) { - stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set); + stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set); pe_warn("Scheduling Node %s for STONITH", node->details->uname); stonith_constraints(node, stonith_op, data_set); @@ -1895,7 +1895,7 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) CRM_ASSERT(container); if(is_set(container->flags, pe_rsc_failed)) { - pe_fence_node(data_set, action->node, "container failed"); + pe_fence_node(data_set, action->node, "container failed", FALSE); } crm_trace("Order %s action %s relative to %s%s for %s%s", @@ -2102,7 +2102,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) * to the remote connection, since the stop will become implied * by the fencing. */ - pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); + pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE); } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) { /* State must be remote_state_unknown or remote_state_stopped. @@ -2150,7 +2150,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) * Since we have no way to find out, it is * necessary to fence the node. */ - pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable"); + pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE); } if(cluster_node && state == remote_state_stopped) { diff --git a/pengine/native.c b/pengine/native.c index fd8929d951a..82d1bafcd22 100644 --- a/pengine/native.c +++ b/pengine/native.c @@ -1501,7 +1501,7 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) for (GList *item = allowed_nodes; item; item = item->next) { pe_node_t *node = item->data; - pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); + pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set); crm_debug("Ordering any stops of %s before %s, and any starts after", rsc->id, unfence->uuid); @@ -1970,7 +1970,7 @@ rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; - pe_fence_node(data_set, node, "deadman ticket was lost"); + pe_fence_node(data_set, node, "deadman ticket was lost", FALSE); } break; @@ -2685,7 +2685,7 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d } if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { - action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); + pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set); order_actions(stop, unfence, pe_order_implies_first); if (!node_has_been_unfenced(current)) { @@ -2715,7 +2715,7 @@ order_after_unfencing(resource_t *rsc, pe_node_t *node, action_t *action, * the node being unfenced, and all its resources being stopped, * whenever a new resource is added -- which would be highly suboptimal. */ - action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); + pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set); order_actions(unfence, action, order); diff --git a/pengine/regression.sh b/pengine/regression.sh index cb8b034ed05..166ed21c735 100755 --- a/pengine/regression.sh +++ b/pengine/regression.sh @@ -573,6 +573,7 @@ do_test probe-timeout "cl#5099 - Default probe timeout" do_test order-first-probes "cl#5301 - respect order constraints when relevant resources are being probed" do_test concurrent-fencing "Allow performing fencing operations in parallel" +do_test priority-fencing-delay "Delay fencing targeting the more significant node" echo "" do_test systemhealth1 "System Health () #1" diff --git a/pengine/test10/priority-fencing-delay.dot b/pengine/test10/priority-fencing-delay.dot new file mode 100644 index 00000000000..62ba699eb6d --- /dev/null +++ b/pengine/test10/priority-fencing-delay.dot @@ -0,0 +1,109 @@ + digraph "g" { +"R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-01_kiff-01_start_0 kiff-02" -> "R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style = bold] +"R-lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"R-lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-01_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] +"R-lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-02_kiff-01_start_0 kiff-02" -> "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style = bold] +"R-lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] +"R-lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd-clone_stop_0" -> "clvmd_stop_0 kiff-01" [ style = bold] +"clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +"clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] +"clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"clvmd_monitor_0 lxc-01_kiff-02" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"clvmd_monitor_0 lxc-02_kiff-02" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"clvmd_stop_0 kiff-01" -> "clvmd-clone_stopped_0" [ style = bold] +"clvmd_stop_0 kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] +"clvmd_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] +"dlm-clone_stop_0" -> "dlm_stop_0 kiff-01" [ style = bold] +"dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +"dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"dlm_monitor_0 lxc-01_kiff-02" -> "dlm-clone_stopped_0" [ style = bold] +"dlm_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"dlm_monitor_0 lxc-02_kiff-02" -> "dlm-clone_stopped_0" [ style = bold] +"dlm_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"dlm_stop_0 kiff-01" -> "dlm-clone_stopped_0" [ style = bold] +"dlm_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"fence-kiff-02_monitor_60000 kiff-02" [ style=bold color="green" fontcolor="black"] +"fence-kiff-02_start_0 kiff-02" -> "fence-kiff-02_monitor_60000 kiff-02" [ style = bold] +"fence-kiff-02_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"fence-kiff-02_stop_0 kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"fence-kiff-02_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"lxc-01_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_monitor_30000 kiff-02" [ style = bold] +"lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] +"lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] +"lxc-01_kiff-01_stop_0 kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"lxc-02_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_monitor_30000 kiff-02" [ style = bold] +"lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] +"lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] +"lxc-02_kiff-01_stop_0 kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"shared0-clone_stop_0" -> "shared0-clone_stopped_0" [ style = bold] +"shared0-clone_stop_0" -> "shared0_stop_0 kiff-01" [ style = bold] +"shared0-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +"shared0-clone_stopped_0" -> "clvmd-clone_stop_0" [ style = bold] +"shared0-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"shared0_monitor_0 lxc-01_kiff-02" -> "shared0-clone_stopped_0" [ style = bold] +"shared0_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"shared0_monitor_0 lxc-02_kiff-02" -> "shared0-clone_stopped_0" [ style = bold] +"shared0_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"shared0_stop_0 kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] +"shared0_stop_0 kiff-01" -> "shared0-clone_stopped_0" [ style = bold] +"shared0_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +"stonith 'reboot' kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "clvmd-clone_stop_0" [ style = bold] +"stonith 'reboot' kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "dlm-clone_stop_0" [ style = bold] +"stonith 'reboot' kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold] +"stonith 'reboot' kiff-01" -> "shared0_stop_0 kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' kiff-01" [ style=bold color="green" fontcolor="black"] +"stonith 'reboot' lxc-01_kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] +"stonith 'reboot' lxc-02_kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"stonith 'reboot' lxc-02_kiff-01" [ style=bold color="green" fontcolor="orange"] +"vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] +"vm-fs_monitor_20000 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] +"vm-fs_start_0 lxc-01_kiff-01" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] +"vm-fs_start_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] +"vm-fs_stop_0 lxc-01_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_stop_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] +} diff --git a/pengine/test10/priority-fencing-delay.exp b/pengine/test10/priority-fencing-delay.exp new file mode 100644 index 00000000000..c6315a1280a --- /dev/null +++ b/pengine/test10/priority-fencing-delay.exp @@ -0,0 +1,570 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pengine/test10/priority-fencing-delay.scores b/pengine/test10/priority-fencing-delay.scores new file mode 100644 index 00000000000..b96175efbf1 --- /dev/null +++ b/pengine/test10/priority-fencing-delay.scores @@ -0,0 +1,301 @@ +Allocation scores: +pcmk__clone_allocate: clvmd-clone allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd-clone allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd-clone allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:0 allocation score on kiff-01: 1 +pcmk__clone_allocate: clvmd:0 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:0 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:1 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:1 allocation score on kiff-02: 1 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:1 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:2 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:2 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:2 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:3 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:3 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:3 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:4 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:4 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:4 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: clvmd:5 allocation score on kiff-01: 0 +pcmk__clone_allocate: clvmd:5 allocation score on kiff-02: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: clvmd:5 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: dlm-clone allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm-clone allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm-clone allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm-clone allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm-clone allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm-clone allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on kiff-01: 1 +pcmk__clone_allocate: dlm:0 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:1 allocation score on kiff-02: 1 +pcmk__clone_allocate: dlm:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:2 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:3 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:4 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on kiff-01: 0 +pcmk__clone_allocate: dlm:5 allocation score on kiff-02: 0 +pcmk__clone_allocate: dlm:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__clone_allocate: dlm:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__clone_allocate: shared0-clone allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0-clone allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0-clone allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:0 allocation score on kiff-01: 1 +pcmk__clone_allocate: shared0:0 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:0 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:1 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:1 allocation score on kiff-02: 1 +pcmk__clone_allocate: shared0:1 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:1 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:1 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:1 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:2 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:2 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:2 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:3 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:3 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:3 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:4 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:4 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:4 allocation score on lxc-02_kiff-02: 0 +pcmk__clone_allocate: shared0:5 allocation score on kiff-01: 0 +pcmk__clone_allocate: shared0:5 allocation score on kiff-02: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-01_kiff-01: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-01_kiff-02: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-02_kiff-01: 0 +pcmk__clone_allocate: shared0:5 allocation score on lxc-02_kiff-02: 0 +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on kiff-02: 100 +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-01_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on kiff-02: 100 +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: R-lxc-02_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on kiff-02: 1 +pcmk__native_allocate: clvmd:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: clvmd:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on kiff-02: 1 +pcmk__native_allocate: dlm:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: dlm:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on kiff-01: 0 +pcmk__native_allocate: fence-kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on kiff-01: 0 +pcmk__native_allocate: fence-kiff-02 allocation score on kiff-02: 0 +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: fence-kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-01_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-01 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on kiff-02: 0 +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: lxc-02_kiff-02 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:0 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on kiff-02: 1 +pcmk__native_allocate: shared0:1 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:1 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:2 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:3 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:4 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on kiff-01: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on kiff-02: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-01_kiff-01: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-01_kiff-02: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-02_kiff-01: -INFINITY +pcmk__native_allocate: shared0:5 allocation score on lxc-02_kiff-02: -INFINITY +pcmk__native_allocate: vm-fs allocation score on kiff-01: 0 +pcmk__native_allocate: vm-fs allocation score on kiff-02: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-01_kiff-01: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-01_kiff-02: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-02_kiff-01: 0 +pcmk__native_allocate: vm-fs allocation score on lxc-02_kiff-02: 0 diff --git a/pengine/test10/priority-fencing-delay.summary b/pengine/test10/priority-fencing-delay.summary new file mode 100644 index 00000000000..83f6d8d6c03 --- /dev/null +++ b/pengine/test10/priority-fencing-delay.summary @@ -0,0 +1,102 @@ + +Current cluster status: +Node kiff-01 (1): UNCLEAN (offline) +Online: [ kiff-02 ] +Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + + vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 + fence-kiff-02 (stonith:fence_ipmilan): Started kiff-01 (UNCLEAN) + Clone Set: dlm-clone [dlm] + dlm (ocf::pacemaker:controld): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: clvmd-clone [clvmd] + clvmd (ocf::heartbeat:clvm): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: shared0-clone [shared0] + shared0 (ocf::heartbeat:Filesystem): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): FAILED kiff-01 (UNCLEAN) + R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-01 (UNCLEAN) + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + +Transition Summary: + * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) 'guest is unclean' + * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' + * Fence (reboot) kiff-01 'peer is no longer part of the cluster' + * Recover vm-fs ( lxc-01_kiff-01 ) + * Move fence-kiff-02 ( kiff-01 -> kiff-02 ) + * Stop dlm:0 ( kiff-01 ) due to node availability + * Stop clvmd:0 ( kiff-01 ) due to node availability + * Stop shared0:0 ( kiff-01 ) due to node availability + * Recover R-lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) + * Move R-lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) + * Move lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) + * Move lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) + +Executing cluster transition: + * Resource action: vm-fs monitor on lxc-02_kiff-02 + * Resource action: vm-fs monitor on lxc-01_kiff-02 + * Pseudo action: fence-kiff-02_stop_0 + * Resource action: dlm monitor on lxc-02_kiff-02 + * Resource action: dlm monitor on lxc-01_kiff-02 + * Resource action: clvmd monitor on lxc-02_kiff-02 + * Resource action: clvmd monitor on lxc-01_kiff-02 + * Resource action: shared0 monitor on lxc-02_kiff-02 + * Resource action: shared0 monitor on lxc-01_kiff-02 + * Pseudo action: lxc-01_kiff-01_stop_0 + * Pseudo action: lxc-02_kiff-01_stop_0 + * Fencing kiff-01 (reboot) + * Pseudo action: R-lxc-01_kiff-01_stop_0 + * Pseudo action: R-lxc-02_kiff-01_stop_0 + * Pseudo action: stonith-lxc-02_kiff-01-reboot on lxc-02_kiff-01 + * Pseudo action: stonith-lxc-01_kiff-01-reboot on lxc-01_kiff-01 + * Pseudo action: vm-fs_stop_0 + * Resource action: fence-kiff-02 start on kiff-02 + * Pseudo action: shared0-clone_stop_0 + * Resource action: R-lxc-01_kiff-01 start on kiff-02 + * Resource action: R-lxc-02_kiff-01 start on kiff-02 + * Resource action: lxc-01_kiff-01 start on kiff-02 + * Resource action: lxc-02_kiff-01 start on kiff-02 + * Resource action: vm-fs start on lxc-01_kiff-01 + * Resource action: fence-kiff-02 monitor=60000 on kiff-02 + * Pseudo action: shared0_stop_0 + * Pseudo action: shared0-clone_stopped_0 + * Resource action: R-lxc-01_kiff-01 monitor=10000 on kiff-02 + * Resource action: R-lxc-02_kiff-01 monitor=10000 on kiff-02 + * Resource action: lxc-01_kiff-01 monitor=30000 on kiff-02 + * Resource action: lxc-02_kiff-01 monitor=30000 on kiff-02 + * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 + * Pseudo action: clvmd-clone_stop_0 + * Pseudo action: clvmd_stop_0 + * Pseudo action: clvmd-clone_stopped_0 + * Pseudo action: dlm-clone_stop_0 + * Pseudo action: dlm_stop_0 + * Pseudo action: dlm-clone_stopped_0 + +Revised cluster status: +Online: [ kiff-02 ] +OFFLINE: [ kiff-01 ] +Containers: [ lxc-01_kiff-01:R-lxc-01_kiff-01 lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-01:R-lxc-02_kiff-01 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + + vm-fs (ocf::heartbeat:Filesystem): Started lxc-01_kiff-01 + R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 + fence-kiff-02 (stonith:fence_ipmilan): Started kiff-02 + Clone Set: dlm-clone [dlm] + Started: [ kiff-02 ] + Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: clvmd-clone [clvmd] + Started: [ kiff-02 ] + Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + Clone Set: shared0-clone [shared0] + Started: [ kiff-02 ] + Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] + R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 + R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + diff --git a/pengine/test10/priority-fencing-delay.xml b/pengine/test10/priority-fencing-delay.xml new file mode 100644 index 00000000000..04d1d3e2904 --- /dev/null +++ b/pengine/test10/priority-fencing-delay.xml @@ -0,0 +1,351 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +