Make all retries and delays configurable (#344)

Closes #186

In some tasks we have to specify number of retries and delays between to wait for some operation to finish.
Seems that this constants should be moved to defaults and also these constants should be overridable by extra-vars.

Also in some tasks timeouts are used instead retries and delays.

Now all retries and delays are configurable and some timeouts deprecated.

Author: hackallcode

CHANGELOG.md

@@ -26,6 +26,15 @@ README.md to use the newest tag with new release
 - `cartridge_not_save_cookie_in_app_config` variable that allows to disable persisting cluster
   cookie in the application configuration file
 - `patch_instance_in_runtime` step to update instance parameters in runtime
+- Variables `bootstrap_vshard_retries`, `bootstrap_vshard_delay`, `connect_to_membership_retries`,
+  `connect_to_membership_delay` to change hardcoded values
+
+### Changed
+
+- Timeout `instance_start_timeout` (to check that all instances become started)
+  deprecated and replaced with `instance_start_retries` and `instance_start_delay`
+- Timeout `instance_discover_buckets_timeout` (to check that instances discover buckets)
+  deprecated and replaced with `instance_discover_buckets_retries` and `instance_discover_buckets_delay`
 
 ## [1.9.0] - 2021-04-30
 

defaults/main.yml

@@ -103,9 +103,15 @@ restarted: null
 expelled: false
 stateboard: false
 
+instance_start_timeout: null  # DEPRECATED
+instance_start_retries: 10
+instance_start_delay: 5
+
 cartridge_wait_buckets_discovery: true
-instance_start_timeout: 60
-instance_discover_buckets_timeout: 60
+
+instance_discover_buckets_timeout: null  # DEPRECATED
+instance_discover_buckets_retries: 20
+instance_discover_buckets_delay: 5
 
 allowed_members_states: ['RolesConfigured']
 wait_members_alive_retries: 10
@@ -142,6 +148,12 @@ cartridge_failover: null  # DEPRECATED
 cartridge_failover_params: null
 cartridge_failover_promote_params: null
 
+connect_to_membership_retries: 3
+connect_to_membership_delay: 5
+
+bootstrap_vshard_retries: 3
+bootstrap_vshard_delay: 5
+
 allow_warning_issues: false
 show_issues: true
 wait_cluster_has_no_issues_retries: 10

doc/scenario.md

@@ -464,7 +464,9 @@ Input facts (set by config):
 
 - `expelled` - indicates if instance must be expelled from topology;
 - `stateboard` - indicates that the instance is a stateboard;
-- `instance_start_timeout` - time in seconds to wait for instance to be started;
+- `instance_start_retries` - retries to check that all instances become started;
+- `instance_start_delay` - delay before retry to check that all instances become started;
+- [DEPRECATED] `instance_start_timeout` - time in seconds to wait for instance to be started;
 - `cartridge_wait_buckets_discovery` - indicates if routers should wait for buckets discovery after vshard bootstrap.
 
 ### connect_to_membership
@@ -480,7 +482,9 @@ Input facts (set by config):
 - `expelled` - indicates if instance must be expelled from topology;
 - `stateboard` - indicates that the instance is a stateboard;
 - `cartridge_app_name` - application name;
-- `config` - instance configuration ([more details here](/doc/instances.md)).
+- `config` - instance configuration ([more details here](/doc/instances.md));
+- `connect_to_membership_retries` - retries to connect to membership;
+- `connect_to_membership_delay` - delay before retry to connect to membership.
 
 Output facts:
 
@@ -633,7 +637,11 @@ Input facts (set by config):
 - `expelled` - indicates if instance must be expelled from topology;
 - `stateboard` - indicates that the instance is a stateboard;
 - `cartridge_bootstrap_vshard` - indicates if vshard should be bootstrapped;
-- `instance_discover_buckets_timeout` - time in seconds to wait for instance to discover buckets;
+- `bootstrap_vshard_retries` - retries to bootstrap vshard;
+- `bootstrap_vshard_delay` - delay before retry to bootstrap vshard;
+- `instance_discover_buckets_retries` - retries to check that instances discover buckets;
+- `instance_discover_buckets_delay` - delay before retry to check that instances discover buckets;
+- [DEPRECATED] `instance_discover_buckets_timeout` - time in seconds to wait for instance to discover buckets;
 - `cartridge_wait_buckets_discovery` - indicates if routers should wait for buckets discovery after vshard bootstrap;
 
 ### configure_failover
@@ -648,8 +656,8 @@ Input facts (set by role):
 
 Input facts (set by config):
 
-- [DEPRECATED] `cartridge_failover` - indicates if eventual failover should be enabled or disabled;
-- `cartridge_failover_params` - failover parameters.
+- `cartridge_failover_params` - failover parameters;
+- [DEPRECATED] `cartridge_failover` - indicates if eventual failover should be enabled or disabled.
 
 ### wait_members_alive
 

doc/variables.md

@@ -91,16 +91,19 @@ For more details see [scenario documentation](/doc/scenario.md).
 * `expelled` (`boolean`, default: `false`): a boolean flag that indicates if instance must be expelled from topology;
 * `stateboard` (`boolean`, default: `false`): a boolean flag that indicates
    that the instance is a [stateboard](/doc/stateboard.md);
-* `instance_start_timeout` (`number`, default: 60): time in seconds to wait for instance to be started;
+* `instance_start_retries` (`number`, default: 10) retries to check that all instances become started;
+* `instance_start_delay` (`number`, default: 5): delay before retry to check that all instances become started;
+* [DEPRECATED] `instance_start_timeout` (`number`, default: 60): time in seconds to wait for instance to be started;
 * `cartridge_wait_buckets_discovery` (`boolean`, default: `true`): a boolean
   flag that indicates if routers should wait for buckets discovery after vshard bootstrap;
-* `instance_discover_buckets_timeout` (`number`, default: 60): time in seconds
+* `instance_discover_buckets_retries` (`number`, default: 20) retries to check that instances discover buckets;
+* `instance_discover_buckets_delay` (`number`, default: 5): delay before retry to check that instances discover buckets;
+* [DEPRECATED] `instance_discover_buckets_timeout` (`number`, default: 60): time in seconds
   to wait for instance to discover buckets;
 * `allowed_members_states` (`list-of-strings`, default: `[RolesConfigured]`):
   list of allowed instance states;
-* `wait_members_alive_retries` (`number`, default: 60) retries to check that all instances become alive;
-  all instances become alive;
-* `wait_members_alive_delay` (`number`, default: 5): delay to retry instances status check;
+* `wait_members_alive_retries` (`number`, default: 10) retries to check that all instances become alive;
+* `wait_members_alive_delay` (`number`, default: 5): delay before retry to check that all instances become alive;
 
 ## Replicasets configuration
 
@@ -125,8 +128,12 @@ For more details see [scenario documentation](/doc/scenario.md).
 
 ## Cluster configuration
 
+- `connect_to_membership_retries` (`number`, default: 3): retries to connect to membership;
+- `connect_to_membership_delay` (`number`, default: 5): delay before retry to connect to membership;
 * `cartridge_bootstrap_vshard` (`boolean`, default: `false`): a boolean
   flag that indicates if vshard should be bootstrapped;
+* `bootstrap_vshard_retries` (`number`, default: 3): retries to bootstrap vshard;
+* `bootstrap_vshard_delay` (`number`, default: 5): delay before retry to bootstrap vshard;
 * `cartridge_app_config` (`dict`): application config sections to patch;
 * `cartridge_auth`: (`dict`): [authorization configuration](/doc/auth.md);
 * `cartridge_failover_params` (`dict`): [failover](/doc/failover.md) parameters;

library/cartridge_get_cached_facts.py

@@ -17,6 +17,10 @@
         'cartridge_app_user',
         'cartridge_auth',
         'cartridge_bootstrap_vshard',
+        'bootstrap_vshard_retries',
+        'bootstrap_vshard_delay',
+        'connect_to_membership_retries',
+        'connect_to_membership_delay',
         'cartridge_runtime_params',
         'cartridge_cluster_cookie',
         'cartridge_not_save_cookie_in_app_config',
@@ -58,7 +62,11 @@
         'expelled',
         'failover_priority',
         'instance_discover_buckets_timeout',
+        'instance_discover_buckets_retries',
+        'instance_discover_buckets_delay',
         'instance_start_timeout',
+        'instance_start_retries',
+        'instance_start_delay',
         'replicaset_alias',
         'restarted',
         'roles',

library/cartridge_validate_config.py

@@ -17,6 +17,10 @@
     'cartridge_not_save_cookie_in_app_config',
     'cartridge_auth',
     'cartridge_bootstrap_vshard',
+    'bootstrap_vshard_retries',
+    'bootstrap_vshard_delay',
+    'connect_to_membership_retries',
+    'connect_to_membership_delay',
     'cartridge_failover',
     'cartridge_failover_params',
     'cartridge_app_config',
@@ -96,6 +100,10 @@
     'cartridge_runtime_params': dict,
     'cartridge_defaults': dict,
     'cartridge_bootstrap_vshard': bool,
+    'bootstrap_vshard_retries': int,
+    'bootstrap_vshard_delay': int,
+    'connect_to_membership_retries': int,
+    'connect_to_membership_delay': int,
     'cartridge_wait_buckets_discovery': bool,
     'cartridge_failover': bool,
     'cartridge_app_config': dict,
@@ -109,7 +117,11 @@
     'stateboard': bool,
     'cartridge_multiversion': bool,
     'instance_start_timeout': int,
+    'instance_start_retries': int,
+    'instance_start_delay': int,
     'instance_discover_buckets_timeout': int,
+    'instance_discover_buckets_retries': int,
+    'instance_discover_buckets_delay': int,
     'twophase_netbox_call_timeout': int,
     'twophase_upload_config_timeout': int,
     'twophase_apply_config_timeout': int,
@@ -543,6 +555,16 @@ def validate_config(params):
                 "Variable 'edit_topology_timeout' is deprecated since 1.9.0 and will be removed in 2.0.0. "
                 "Use 'edit_topology_healthy_timeout' instead."
             )
+        if instance_vars.get('instance_start_timeout') is not None:
+            warnings.append(
+                "Variable 'instance_start_timeout' is deprecated since 1.10.0 and will be removed in 2.0.0. "
+                "Use 'instance_start_retries' and 'instance_start_delay' instead."
+            )
+        if instance_vars.get('instance_discover_buckets_timeout') is not None:
+            warnings.append(
+                "Variable 'instance_discover_buckets_timeout' is deprecated since 1.10.0 and will be removed in 2.0.0. "
+                "Use 'instance_discover_buckets_retries' and 'instance_discover_buckets_delay' instead."
+            )
 
         # Validate types
         errmsg = validate_types(instance_vars)

molecule/default/hosts.yml

@@ -89,8 +89,8 @@ all:
             advertise_uri: 'vm2:3201'
             http_port: 8201
             memtx_memory: '{{ common_memtx_memory }}'
-          instance_start_timeout: 120
-          instance_discover_buckets_timeout: 120
+          instance_start_retries: 25
+          instance_discover_buckets_retries: 25
 
         core-2:
           config:

molecule/tasks_from/hosts.yml

@@ -64,8 +64,8 @@ all:
             advertise_uri: 'vm1:3303'
             http_port: 8103
             memtx_memory: 268436000
-          instance_start_timeout: 120
-          instance_discover_buckets_timeout: 120
+          instance_start_retries: 25
+          instance_discover_buckets_retries: 25
 
         core-2:
           config:

tasks/set_instance_facts.yml

@@ -64,9 +64,15 @@
       expelled: '{{ expelled }}'
       stateboard: '{{ stateboard }}'
 
-      cartridge_wait_buckets_discovery: '{{ cartridge_wait_buckets_discovery }}'
       instance_start_timeout: '{{ instance_start_timeout }}'
+      instance_start_retries: '{{ instance_start_retries }}'
+      instance_start_delay: '{{ instance_start_delay }}'
+
+      cartridge_wait_buckets_discovery: '{{ cartridge_wait_buckets_discovery }}'
+
       instance_discover_buckets_timeout: '{{ instance_discover_buckets_timeout }}'
+      instance_discover_buckets_retries: '{{ instance_discover_buckets_retries }}'
+      instance_discover_buckets_delay: '{{ instance_discover_buckets_delay }}'
 
       allowed_members_states: '{{ allowed_members_states }}'
       wait_members_alive_retries: '{{ wait_members_alive_retries }}'
@@ -102,6 +108,12 @@
       cartridge_failover_params: '{{ cartridge_failover_params }}'
       cartridge_failover_promote_params: '{{ cartridge_failover_promote_params }}'
 
+      connect_to_membership_retries: '{{ connect_to_membership_retries }}'
+      connect_to_membership_delay: '{{ connect_to_membership_delay }}'
+
+      bootstrap_vshard_retries: '{{ bootstrap_vshard_retries }}'
+      bootstrap_vshard_delay: '{{ bootstrap_vshard_delay }}'
+
       allow_warning_issues: '{{ allow_warning_issues }}'
       show_issues: '{{ show_issues }}'
       wait_cluster_has_no_issues_retries: '{{ wait_cluster_has_no_issues_retries }}'

tasks/steps/bootstrap_vshard.yml

@@ -18,8 +18,8 @@
         apply_config_timeout: '{{ twophase_apply_config_timeout }}'
       register: bootstrap_vshard
       until: not bootstrap_vshard.failed
-      retries: 3
-      delay: 5
+      retries: '{{ bootstrap_vshard_retries }}'
+      delay: '{{ bootstrap_vshard_delay }}'
       run_once: true
       delegate_to: '{{ control_instance.name }}'
 
@@ -30,6 +30,10 @@
         check_buckets_are_discovered: true
       register: check_instance
       until: not check_instance.failed
-      retries: '{{ instance_discover_buckets_timeout // 5 }}'
-      delay: 5
+      retries: '{{
+          instance_discover_buckets_timeout // instance_discover_buckets_delay
+          if instance_discover_buckets_timeout is not none
+          else instance_discover_buckets_retries
+        }}'
+      delay: '{{ instance_discover_buckets_delay }}'
       when: cartridge_wait_buckets_discovery