diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java b/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java index 956e8aecabca..7b59783b19c1 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/TaskExecutor.java @@ -1152,6 +1152,11 @@ public synchronized void doHeartbeat() { */ public void addSubTaskGroup(SubTaskGroup subTaskGroup) { log.info("Adding SubTaskGroup #{}: {}", subTaskGroups.size(), subTaskGroup.name); + if (subTaskGroup.getSubTaskCount() == 0) { + // Allowing to add this just messes up the positions. + log.info("Ignoring subtask SubTaskGroup {} as it is empty", subTaskGroup.name); + return; + } subTaskGroup.setRunnableTaskContext(this, subTaskPosition); subTaskGroups.add(subTaskGroup); subTaskPosition++; diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterCreate.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterCreate.java index cbe6fb473c9c..ae0a65f65530 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterCreate.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterCreate.java @@ -32,10 +32,10 @@ protected AddOnClusterCreate(BaseTaskDependencies baseTaskDependencies) { @Override public void run() { log.info("Started {} task for uuid={}", getName(), taskParams().getUniverseUUID()); - + Universe universe = null; try { Cluster cluster = taskParams().getAddOnClusters().get(0); - Universe universe = + universe = lockAndFreezeUniverseForUpdate( taskParams().expectedUniverseVersion, u -> { @@ -97,16 +97,25 @@ public void run() { log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); throw t; } finally { - // Mark the update of the universe as done. This will allow future edits/updates to the - // universe to happen. - Universe universe = unlockUniverseForUpdate(); - if (universe.getConfig().getOrDefault(Universe.USE_CUSTOM_IMAGE, "false").equals("true")) { - universe.updateConfig( - ImmutableMap.of( - Universe.USE_CUSTOM_IMAGE, - Boolean.toString( - universe.getUniverseDetails().nodeDetailsSet.stream() - .allMatch(n -> n.ybPrebuiltAmi)))); + if (universe != null) { + // Universe is locked by this task. + try { + // Fetch the latest universe. + universe = Universe.getOrBadRequest(universe.getUniverseUUID()); + if (universe + .getConfig() + .getOrDefault(Universe.USE_CUSTOM_IMAGE, "false") + .equals("true")) { + universe.updateConfig( + ImmutableMap.of( + Universe.USE_CUSTOM_IMAGE, + Boolean.toString( + universe.getUniverseDetails().nodeDetailsSet.stream() + .allMatch(n -> n.ybPrebuiltAmi)))); + } + } finally { + unlockUniverseForUpdate(); + } } } log.info("Finished {} task.", getName()); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterDelete.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterDelete.java index 21a470e63ebb..5b4ad43c2b7d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterDelete.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/AddOnClusterDelete.java @@ -66,7 +66,9 @@ public void run() { if (params().isForceDelete) { universe = forceLockUniverseForUpdate(-1 /* expectedUniverseVersion */); } else { - universe = lockUniverseForUpdate(params().expectedUniverseVersion); + universe = + lockAndFreezeUniverseForUpdate( + params().expectedUniverseVersion, null /* Txn callback */); } Cluster clusterToDelete = universe.getCluster(params().clusterUUID); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/BackupUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/BackupUniverse.java index 446026226301..521fdb798cf6 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/BackupUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/BackupUniverse.java @@ -107,7 +107,7 @@ public void run() { checkUniverseVersion(); // Update the universe DB with the update to be performed and set the 'updateInProgress' flag // to prevent other updates from happening. - universe = lockUniverseForUpdate(-1); + universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); try { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java index 92468ec7c434..707b6aa70b05 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java @@ -105,7 +105,9 @@ public void run() { } } - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); kubernetesStatus.startYBUniverseEventStatus( universe, taskParams().getKubernetesResourceDetails(), diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateTableSpacesInUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateTableSpacesInUniverse.java index fd049ee457cb..5ef0812a91c8 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateTableSpacesInUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateTableSpacesInUniverse.java @@ -40,7 +40,9 @@ public void run() { checkUniverseVersion(); // Update the DB to prevent other changes from happening. - universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); createTableSpacesTask().setSubTaskGroupType(SubTaskGroupType.CreatingTablespaces); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfig.java index 5a7f6693a621..131ded675a3c 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfig.java @@ -69,10 +69,12 @@ public void run() { Universe targetUniverse = Universe.getOrBadRequest(xClusterConfig.getTargetUniverseUUID()); try { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); try { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), targetUniverse.getVersion(), null /* Txn callback */); createCheckXUniverseAutoFlag(sourceUniverse, targetUniverse) .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.PreflightChecks); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteDrConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteDrConfig.java index 143d747a3fc6..b94e8ec5f239 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteDrConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteDrConfig.java @@ -61,12 +61,16 @@ public void run() { try { if (sourceUniverse != null) { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); } try { if (targetUniverse != null) { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), + targetUniverse.getVersion(), + null /* Txn callback */); } for (XClusterConfig xcc : drConfig.getXClusterConfigs()) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfig.java index da0486a353a0..62e8a9e2da26 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfig.java @@ -45,12 +45,16 @@ public void run() { try { if (sourceUniverse != null) { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); } try { if (targetUniverse != null) { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), + targetUniverse.getVersion(), + null /* Txn callback */); } createDeleteXClusterConfigSubtasks(xClusterConfig, sourceUniverse, targetUniverse); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverse.java index b02fc7c7dd30..709e502502f7 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverse.java @@ -85,7 +85,7 @@ public void run() { if (params().isForceDelete) { universe = forceLockUniverseForUpdate(-1); } else { - universe = lockUniverseForUpdate(-1 /* expectedUniverseVersion */); + universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); } kubernetesStatus.startYBUniverseEventStatus( universe, diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyUniverse.java index 82b2c55be490..5861f501aa02 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyUniverse.java @@ -99,7 +99,7 @@ public void run() { if (params().isForceDelete) { universe = forceLockUniverseForUpdate(-1); } else { - universe = lockUniverseForUpdate(-1); + universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); } // Delete xCluster configs involving this universe and put the locked universes to diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditDrConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditDrConfig.java index 752a8135b1fc..ef1f13079bdf 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditDrConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditDrConfig.java @@ -55,14 +55,18 @@ public void run() { Universe.getOrBadRequest(newXClusterConfig.getTargetUniverseUUID()); try { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); try { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), targetUniverse.getVersion(), null /* Txn callback */); try { // Lock the new target universe. - lockUniverseForUpdate( - newTargetUniverse.getUniverseUUID(), newTargetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + newTargetUniverse.getUniverseUUID(), + newTargetUniverse.getVersion(), + null /* Txn callback */); addSubtasksToUseNewXClusterConfig( currentXClusterConfig, diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java index e46a07600504..725ea4a3c571 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverse.java @@ -78,7 +78,9 @@ public void run() { verifyParams(UniverseOpType.EDIT); // TODO: Would it make sense to have a precheck k8s task that does // some precheck operations to verify kubeconfig, svcaccount, connectivity to universe here ? - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); addBasicPrecheckTasks(); kubernetesStatus.startYBUniverseEventStatus( diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditUniverse.java index f581f64a3c40..4ef00430d523 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditUniverse.java @@ -126,24 +126,31 @@ public void run() { // Run all the tasks. getRunnableTask().runSubTasks(); } catch (Throwable t) { - log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); errorString = t.getMessage(); + log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); throw t; } finally { releaseReservedNodes(); - // Mark the update of the universe as done. This will allow future edits/updates to the - // universe to happen. - universe = unlockUniverseForUpdate(errorString); - - if (universe != null - && universe.getConfig().getOrDefault(Universe.USE_CUSTOM_IMAGE, "false").equals("true")) { - universe.updateConfig( - ImmutableMap.of( - Universe.USE_CUSTOM_IMAGE, - Boolean.toString( - universe.getUniverseDetails().nodeDetailsSet.stream() - .allMatch(n -> n.ybPrebuiltAmi)))); - universe.save(); + if (universe != null) { + // Universe is locked by this task. + try { + // Fetch the latest universe. + universe = Universe.getOrBadRequest(universe.getUniverseUUID()); + if (universe + .getConfig() + .getOrDefault(Universe.USE_CUSTOM_IMAGE, "false") + .equals("true")) { + universe.updateConfig( + ImmutableMap.of( + Universe.USE_CUSTOM_IMAGE, + Boolean.toString( + universe.getUniverseDetails().nodeDetailsSet.stream() + .allMatch(n -> n.ybPrebuiltAmi)))); + universe.save(); + } + } finally { + unlockUniverseForUpdate(errorString); + } } } log.info("Finished {} task.", getName()); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java index 9fa453996215..490be365246e 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java @@ -46,10 +46,12 @@ public void run() { XClusterConfigEditFormData editFormData = taskParams().getEditFormData(); // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); try { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), targetUniverse.getVersion(), null /* Txn callback */); try { // Check Auto flags on source and target universes while resuming xCluster. diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/FailoverDrConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/FailoverDrConfig.java index dca89699e998..ab556de145ab 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/FailoverDrConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/FailoverDrConfig.java @@ -62,10 +62,12 @@ public void run() { Universe.getOrBadRequest(currentXClusterConfig.getTargetUniverseUUID()); try { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); try { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), targetUniverse.getVersion(), null /* Txn callback */); createSetDrStatesTask( currentXClusterConfig, diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/LdapUnivSync.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/LdapUnivSync.java index 494f64bda4ed..231c24b5049d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/LdapUnivSync.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/LdapUnivSync.java @@ -65,7 +65,7 @@ public void run() { taskParams().ldapGroups = ldapGroups; try { - lockUniverseForUpdate(taskParams().getUniverseUUID(), -1); + lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); // queryLdap createQueryLdapServerTask(taskParams()).setSubTaskGroupType(SubTaskGroupType.QueryLdapServer); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/MultiTableBackup.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/MultiTableBackup.java index b2e66c1d478f..fa9a37d6179a 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/MultiTableBackup.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/MultiTableBackup.java @@ -95,7 +95,7 @@ public void run() { // Update the universe DB with the update to be performed and set the 'updateInProgress' flag // to prevent other updates from happening. - universe = lockUniverseForUpdate(-1); + universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); try { String masterAddresses = universe.getMasterAddresses(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/PauseUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/PauseUniverse.java index fc361a47b4ba..b70a587cb38f 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/PauseUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/PauseUniverse.java @@ -49,7 +49,7 @@ public void run() { try { // Update the universe DB with the update to be performed and set the // 'updateInProgress' flag to prevent other updates from happening. - Universe universe = lockUniverseForUpdate(-1 /* expectedUniverseVersion */); + Universe universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); if (universe.getUniverseDetails().universePaused) { String msg = "Unable to pause universe \"" + universe.getName() + "\" as it is already paused."; diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreate.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreate.java index b0b31f73f4c4..c5de786678f9 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreate.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreate.java @@ -44,11 +44,11 @@ protected void createPrecheckTasks(Universe universe) { @Override public void run() { log.info("Started {} task for uuid={}", getName(), taskParams().getUniverseUUID()); - + Universe universe = null; try { // Set the 'updateInProgress' flag to prevent other updates from happening. Cluster cluster = taskParams().getReadOnlyClusters().get(0); - Universe universe = + universe = lockAndFreezeUniverseForUpdate( taskParams().expectedUniverseVersion, u -> { @@ -157,17 +157,26 @@ public void run() { log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); throw t; } finally { - // Mark the update of the universe as done. This will allow future edits/updates to the - // universe to happen. - Universe universe = unlockUniverseForUpdate(); - if (universe.getConfig().getOrDefault(Universe.USE_CUSTOM_IMAGE, "false").equals("true")) { - universe.updateConfig( - ImmutableMap.of( - Universe.USE_CUSTOM_IMAGE, - Boolean.toString( - universe.getUniverseDetails().nodeDetailsSet.stream() - .allMatch(n -> n.ybPrebuiltAmi)))); - universe.save(); + if (universe != null) { + // Universe is locked by this task. + try { + // Fetch the latest universe. + universe = Universe.getOrBadRequest(universe.getUniverseUUID()); + if (universe + .getConfig() + .getOrDefault(Universe.USE_CUSTOM_IMAGE, "false") + .equals("true")) { + universe.updateConfig( + ImmutableMap.of( + Universe.USE_CUSTOM_IMAGE, + Boolean.toString( + universe.getUniverseDetails().nodeDetailsSet.stream() + .allMatch(n -> n.ybPrebuiltAmi)))); + universe.save(); + } + } finally { + unlockUniverseForUpdate(); + } } } log.info("Finished {} task.", getName()); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDelete.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDelete.java index 34449e5cd405..77be763d6b17 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDelete.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDelete.java @@ -65,7 +65,9 @@ public void run() { if (params().isForceDelete) { universe = forceLockUniverseForUpdate(-1 /* expectedUniverseVersion */); } else { - universe = lockUniverseForUpdate(params().expectedUniverseVersion); + universe = + lockAndFreezeUniverseForUpdate( + params().expectedUniverseVersion, null /* Txn callback */); } List roClusters = universe.getUniverseDetails().getReadOnlyClusters(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java index 177a65ead247..50916c210f8b 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterCreate.java @@ -39,7 +39,9 @@ public void run() { log.info("Started {} task for uuid={}", getName(), taskParams().getUniverseUUID()); try { verifyParams(UniverseOpType.CREATE); - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); preTaskActions(universe); addBasicPrecheckTasks(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterDelete.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterDelete.java index 7b90efaaa5d1..b807b21da454 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterDelete.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyKubernetesClusterDelete.java @@ -69,7 +69,7 @@ public void run() { if (params().isForceDelete) { universe = forceLockUniverseForUpdate(-1); } else { - universe = lockUniverseForUpdate(-1 /* expectedUniverseVersion */); + universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); } List roClusters = universe.getUniverseDetails().getReadOnlyClusters(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReplaceNodeInUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReplaceNodeInUniverse.java index 911722b79603..42789fc4d4b0 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReplaceNodeInUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReplaceNodeInUniverse.java @@ -66,7 +66,6 @@ public void run() { log.info("Started {} task for uuid={}", getName(), taskParams().getUniverseUUID()); checkUniverseVersion(); String errorString = null; - Universe universe = lockAndFreezeUniverseForUpdate( taskParams().expectedUniverseVersion, this::freezeUniverseInTxn); @@ -120,14 +119,13 @@ public void run() { // Run all the tasks. getRunnableTask().runSubTasks(); } catch (Throwable t) { - log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); errorString = t.getMessage(); + log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); throw t; } finally { // Mark the update of the universe as done. This will allow future edits/updates to the // universe to happen. - universe = unlockUniverseForUpdate(errorString); - + unlockUniverseForUpdate(errorString); log.info("Finished {} task.", getName()); } } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReprovisionNode.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReprovisionNode.java index e96af918b8d7..ec8619f3d14d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReprovisionNode.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReprovisionNode.java @@ -36,7 +36,9 @@ public void run() { checkUniverseVersion(); // Update the DB to prevent other changes from happening. - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); NodeDetails currentNode = universe.getNode(taskParams().nodeName); if (currentNode == null) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestartXClusterConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestartXClusterConfig.java index 9baab3b7978a..ea39947e29fc 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestartXClusterConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestartXClusterConfig.java @@ -34,10 +34,12 @@ public void run() { Universe targetUniverse = Universe.getOrBadRequest(xClusterConfig.getTargetUniverseUUID()); try { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); try { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), targetUniverse.getVersion(), null /* Txn callback */); createCheckXUniverseAutoFlag(sourceUniverse, targetUniverse) .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.PreflightChecks); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverse.java index e93b285c25dc..7785028d38f4 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverse.java @@ -58,7 +58,7 @@ public void run() { try { // Update the universe DB with the update to be performed and set the 'updateInProgress' flag // to prevent other updates from happening. - Universe universe = lockUniverseForUpdate(-1 /* expectedUniverseVersion */); + Universe universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); Collection nodes = universe.getNodes(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RunApiTriggeredHooks.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RunApiTriggeredHooks.java index c63bbd836c0b..ff86bb176ba6 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RunApiTriggeredHooks.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RunApiTriggeredHooks.java @@ -36,7 +36,7 @@ public void run() { log.info("Started {} task", getName()); try { - Universe universe = lockUniverseForUpdate(-1); // Check this + Universe universe = lockAndFreezeUniverseForUpdate(-1, null /* Txn callback */); Collection nodes = universe.getNodes(); int countBefore = nodes.size(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SetUniverseKey.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SetUniverseKey.java index 83b0afbfc427..ff80486d9a0a 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SetUniverseKey.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SetUniverseKey.java @@ -38,7 +38,7 @@ public void run() { checkUniverseVersion(); // Update the universe DB with the update to be performed and set the // 'updateInProgress' flag to prevent other updates from happening. - lockUniverseForUpdate(taskParams().expectedUniverseVersion); + lockAndFreezeUniverseForUpdate(taskParams().expectedUniverseVersion, null /* Txn callback */); preTaskActions(); // Manage encryption at rest diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNode.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNode.java index 9252d8db24af..16773af7154e 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNode.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNode.java @@ -51,7 +51,9 @@ public void run() { checkUniverseVersion(); // Update the DB to prevent other changes from happening. - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); currentNode = universe.getNode(taskParams().nodeName); if (currentNode == null) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SwitchoverDrConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SwitchoverDrConfig.java index 7e6824116364..c076bdd4e001 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SwitchoverDrConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SwitchoverDrConfig.java @@ -57,10 +57,12 @@ public void run() { Universe.getOrBadRequest(currentXClusterConfig.getTargetUniverseUUID()); try { // Lock the source universe. - lockUniverseForUpdate(sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + sourceUniverse.getUniverseUUID(), sourceUniverse.getVersion(), null /* Txn callback */); try { // Lock the target universe. - lockUniverseForUpdate(targetUniverse.getUniverseUUID(), targetUniverse.getVersion()); + lockAndFreezeUniverseForUpdate( + targetUniverse.getUniverseUUID(), targetUniverse.getVersion(), null /* Txn callback */); createSetDrStatesTask( currentXClusterConfig, diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfig.java index d655ecac4726..984f9269dd6a 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfig.java @@ -19,8 +19,7 @@ protected SyncXClusterConfig( @Override public void run() { log.info("Running {}", getName()); - - lockUniverseForUpdate(getUniverse().getVersion()); + lockAndFreezeUniverseForUpdate(getUniverse().getVersion(), null /* Txn callback */); try { createXClusterConfigSyncTask() .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.ConfigureUniverse); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java index e70ad524e473..991525330b1a 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java @@ -914,41 +914,6 @@ public String getName() { return super.getName() + "(" + taskParams().getUniverseUUID() + ")"; } - /** Similar to {@link #lockUniverseForUpdate(UUID, int, Consumer)} without the callback. */ - public Universe lockUniverseForUpdate(UUID universeUuid, int expectedUniverseVersion) { - return lockUniverseForUpdate(universeUuid, expectedUniverseVersion, null); - } - - /** - * Similar to {@link #lockUniverseForUpdate(UUID, int, Consumer)} with the universe UUID from this - * task and other parameters set to null. - */ - public Universe lockUniverseForUpdate(int expectedUniverseVersion) { - return lockUniverseForUpdate(taskParams().getUniverseUUID(), expectedUniverseVersion, null); - } - - /** - * It locks the universe for updates by setting the 'updateInProgress' flag. If the universe is - * already being modified, then throws an exception. - * - * @param universeUuid The UUID of the universe to lock - * @param expectedUniverseVersion Lock only if the current version of the universe is at this - * version; -1 implies always lock the universe - * @param callback Callback is invoked for any pre-processing to be done on the Universe before it - * is saved in transaction with 'updateInProgress' flag. - * @return - */ - private Universe lockUniverseForUpdate( - UUID universeUuid, int expectedUniverseVersion, @Nullable Consumer callback) { - UniverseUpdaterConfig updaterConfig = - UniverseUpdaterConfig.builder() - .callback(callback) - .checkSuccess(true) - .expectedUniverseVersion(expectedUniverseVersion) - .build(); - return lockUniverseForUpdate(universeUuid, getLockingUniverseUpdater(updaterConfig)); - } - public Universe forceLockUniverseForUpdate(int expectedUniverseVersion) { UniverseUpdaterConfig updaterConfig = UniverseUpdaterConfig.builder() @@ -976,11 +941,19 @@ public Universe lockUniverse(int expectedUniverseVersion) { return lockUniverseForUpdate(taskParams().getUniverseUUID(), updater); } + /** See {@link #lockAndFreezeUniverseForUpdate(UUID, int, Consumer)} */ + public Universe lockAndFreezeUniverseForUpdate( + int expectedUniverseVersion, @Nullable Consumer firstRunTxnCallback) { + return lockAndFreezeUniverseForUpdate( + taskParams().getUniverseUUID(), expectedUniverseVersion, firstRunTxnCallback); + } + /** * This method locks the universe, runs {@link #createPrecheckTasks(Universe)}, and freezes the * universe with the given txnCallback. By freezing, the association between the task and the * universe is set up such that the universe always has a reference to the task. * + * @param universeUuid the universe UUID. * @param expectedUniverseVersion Lock only if the current version of the universe is at this * version. -1 implies always lock the universe. * @param firstRunTxnCallback the callback to be invoked in transaction when the universe is @@ -988,27 +961,30 @@ public Universe lockUniverse(int expectedUniverseVersion) { * @return the universe. */ public Universe lockAndFreezeUniverseForUpdate( - int expectedUniverseVersion, @Nullable Consumer firstRunTxnCallback) { + UUID universeUuid, + int expectedUniverseVersion, + @Nullable Consumer firstRunTxnCallback) { UniverseUpdaterConfig updaterConfig = UniverseUpdaterConfig.builder() .expectedUniverseVersion(expectedUniverseVersion) .freezeUniverse(false) .build(); UniverseUpdater updater = getLockingUniverseUpdater(updaterConfig); - Universe universe = lockUniverseForUpdate(taskParams().getUniverseUUID(), updater); + Universe universe = lockUniverseForUpdate(universeUuid, updater); try { createPrecheckTasks(universe); if (isFirstTry()) { - createFreezeUniverseTask(firstRunTxnCallback) + createFreezeUniverseTask(universeUuid, firstRunTxnCallback) .setSubTaskGroupType(SubTaskGroupType.ValidateConfigurations); // Run to apply the change first before adding the rest of the subtasks. getRunnableTask().runSubTasks(); } else { - createFreezeUniverseTask().setSubTaskGroupType(SubTaskGroupType.ValidateConfigurations); + createFreezeUniverseTask(universeUuid) + .setSubTaskGroupType(SubTaskGroupType.ValidateConfigurations); } - return getUniverse(); + return Universe.getOrBadRequest(universeUuid); } catch (RuntimeException e) { - unlockUniverseForUpdate(); + unlockUniverseForUpdate(universeUuid); throw e; } } @@ -1016,25 +992,28 @@ public Universe lockAndFreezeUniverseForUpdate( /** * Similar to {@link #createFreezeUniverseTask(Consumer)} without the callback. * + * @param universeUuid the universe UUID. * @return */ - private SubTaskGroup createFreezeUniverseTask() { - return createFreezeUniverseTask(null); + private SubTaskGroup createFreezeUniverseTask(UUID universeUuid) { + return createFreezeUniverseTask(universeUuid, null); } /** * Creates a subtask to freeze the universe {@link #freezeUniverse(Consumer)}. * + * @param universeUuid the universe UUID. * @param callback the callback to be executed in transaction when the universe is frozen. * @return the subtask group. */ - private SubTaskGroup createFreezeUniverseTask(@Nullable Consumer callback) { + private SubTaskGroup createFreezeUniverseTask( + UUID universeUuid, @Nullable Consumer callback) { SubTaskGroup subTaskGroup = createSubTaskGroup( FreezeUniverse.class.getSimpleName(), SubTaskGroupType.ValidateConfigurations); FreezeUniverse task = createTask(FreezeUniverse.class); FreezeUniverse.Params params = new FreezeUniverse.Params(); - params.setUniverseUUID(taskParams().getUniverseUUID()); + params.setUniverseUUID(universeUuid); params.setCallback(callback); task.initialize(params); subTaskGroup.addSubTask(task); @@ -1053,23 +1032,22 @@ public Universe lockUniverseIfExist(UUID universeUuid, int expectedUniverseVersi } public Universe unlockUniverseForUpdate(UUID universeUuid) { - return unlockUniverseForUpdate(universeUuid, null /* error */); - } - - public Universe unlockUniverseForUpdate() { - return unlockUniverseForUpdate((String) null); + return unlockUniverseForUpdate(universeUuid, null, true); } public Universe unlockUniverseForUpdate(boolean updateTaskDetails) { - return unlockUniverseForUpdate( - taskParams().getUniverseUUID(), (String) null, updateTaskDetails); + return unlockUniverseForUpdate(taskParams().getUniverseUUID(), null, updateTaskDetails); } public Universe unlockUniverseForUpdate(String error) { - return unlockUniverseForUpdate(taskParams().getUniverseUUID(), error); + return unlockUniverseForUpdate(taskParams().getUniverseUUID(), error, true); + } + + public Universe unlockUniverseForUpdate() { + return unlockUniverseForUpdate(taskParams().getUniverseUUID(), null, true); } - public Universe unlockUniverseForUpdate( + private Universe unlockUniverseForUpdate( UUID universeUUID, String error, boolean updateTaskDetails) { ExecutionContext executionContext = getOrCreateExecutionContext(); if (!executionContext.isLocked(universeUUID)) { @@ -1110,10 +1088,6 @@ public Universe unlockUniverseForUpdate( return universe; } - public Universe unlockUniverseForUpdate(UUID universeUUID, String error) { - return unlockUniverseForUpdate(universeUUID, error, true); - } - public SubTaskGroup getAnsibleConfigureYbcServerTasks( AnsibleConfigureServers.Params params, Universe universe) { String subGroupDescription = diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateKubernetesDiskSize.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateKubernetesDiskSize.java index ed3a5377ba8e..3af6925536a8 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateKubernetesDiskSize.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateKubernetesDiskSize.java @@ -50,7 +50,9 @@ public void run() { verifyParams(UniverseOpType.EDIT); // additional verification about disk size increase is needed here - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); taskParams().useNewHelmNamingStyle = universe.getUniverseDetails().useNewHelmNamingStyle; preTaskActions(); addBasicPrecheckTasks(); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java index f36a7d2e8c54..3a38d638931f 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverse.java @@ -62,7 +62,9 @@ public void run() { // Update the universe DB with the update to be performed and set the 'updateInProgress' flag // to prevent other updates from happening. - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); kubernetesStatus.startYBUniverseEventStatus( universe, taskParams().getKubernetesResourceDetails(), diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverse.java index d0e2cadb4426..94d0df0911ee 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverse.java @@ -69,6 +69,7 @@ */ @Deprecated @Slf4j +// TODO This should be removed soon as the task type has no mapping. public class UpgradeUniverse extends UniverseDefinitionTaskBase { // Variable to mark if the loadbalancer state was changed. boolean loadbalancerOff = false; @@ -312,7 +313,9 @@ public void run() { checkUniverseVersion(); // Update the universe DB with the update to be performed and set the 'updateInProgress' flag // to prevent other updates from happening. - Universe universe = lockUniverseForUpdate(taskParams().expectedUniverseVersion); + Universe universe = + lockAndFreezeUniverseForUpdate( + taskParams().expectedUniverseVersion, null /* Txn callback */); Cluster primaryCluster = universe.getUniverseDetails().getPrimaryCluster(); UserIntent primIntent = primaryCluster.userIntent; diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java index 7cec9dedc84a..918d1597b3f8 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpgradeYbcGFlags.java @@ -119,11 +119,10 @@ public void run() { } catch (IllegalAccessException e) { throw new RuntimeException(e); } catch (Throwable t) { - log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); errorString = t.getMessage(); + log.error("Error executing task {} with error='{}'.", getName(), t.getMessage(), t); throw t; } finally { - // Mark the update of the universe as done. This will allow future updates to the universe. unlockUniverseForUpdate(errorString); } log.info("Finished {} task.", getName()); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java index e99bcfdf9d55..38291cfa1ea5 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java @@ -12,9 +12,9 @@ import static com.yugabyte.yw.common.ModelFactory.createUniverse; import static com.yugabyte.yw.models.TaskInfo.State.Failure; import static com.yugabyte.yw.models.TaskInfo.State.Success; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; @@ -353,6 +353,7 @@ private void setupCommon() { private static final List KUBERNETES_CREATE_UNIVERSE_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.KubernetesCommandExecutor, TaskType.KubernetesCommandExecutor, TaskType.KubernetesCommandExecutor, @@ -375,6 +376,7 @@ private void setupCommon() { // Cannot use defaultUniverse.universeUUID in a class field. private List getExpectedCreateUniverseTaskResults() { return ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", CREATE_NAMESPACE.name())), Json.toJson(ImmutableMap.of("commandType", APPLY_SECRET.name())), Json.toJson(ImmutableMap.of("commandType", HELM_INSTALL.name())), @@ -393,14 +395,15 @@ private List getExpectedCreateUniverseTaskResults() { } private List getTaskPositionsToSkip(boolean skipNamespace) { - // 3 is WAIT_FOR_PODS of type KubernetesCheckNumPod task. - // 0 is CREATE_NAMESPACE of type KubernetesCommandExecutor - return skipNamespace ? ImmutableList.of(0, 3) : ImmutableList.of(3); + // 4 is WAIT_FOR_PODS of type KubernetesCheckNumPod task. + // 1 is CREATE_NAMESPACE of type KubernetesCommandExecutor + // 0 is FreezeUniverse + return skipNamespace ? ImmutableList.of(1, 4) : ImmutableList.of(4); } private List getTaskCountPerPosition(int namespaceTasks, int parallelTasks) { return ImmutableList.of( - namespaceTasks, parallelTasks, parallelTasks, 0, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1); + 1, namespaceTasks, parallelTasks, parallelTasks, 0, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1); } private void assertTaskSequence( @@ -410,17 +413,17 @@ private void assertTaskSequence( List taskPositionsToSkip, List taskCountPerPosition) { int position = 0; - for (TaskType taskType : expectedTasks) { - if (taskPositionsToSkip.contains(position)) { - position++; + for (int i = 0; i < expectedTasks.size(); i++) { + if (taskPositionsToSkip.contains(i)) { + // Skip subtask from the expected ones as they are not run for some cases. continue; } - + TaskType taskType = expectedTasks.get(i); + JsonNode expectedResults = expectedTasksResult.get(i); + int expectedSize = taskCountPerPosition.get(i); List tasks = subTasksByPosition.get(position); - JsonNode expectedResults = expectedTasksResult.get(position); List taskDetails = tasks.stream().map(TaskInfo::getDetails).collect(Collectors.toList()); - int expectedSize = taskCountPerPosition.get(position); assertEquals(expectedSize, tasks.size()); assertEquals(taskType, tasks.get(0).getTaskType()); assertJsonEqual(expectedResults, taskDetails.get(0)); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfigTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfigTest.java index b4b0f0d61b99..c4f78a4be859 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfigTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateXClusterConfigTest.java @@ -484,8 +484,9 @@ public void testCreateXClusterSetupFailure() { assertNotNull(taskInfo); assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(TaskType.XClusterConfigSetup, taskInfo.getSubTasks().get(4).getTaskType()); - String taskErrMsg = taskInfo.getSubTasks().get(4).getDetails().get("errorString").asText(); + // Two FreeUniverse subtasks for source and target. + assertEquals(TaskType.XClusterConfigSetup, taskInfo.getSubTasks().get(6).getTaskType()); + String taskErrMsg = taskInfo.getSubTasks().get(6).getDetails().get("errorString").asText(); assertThat(taskErrMsg, containsString(setupErrMsg)); assertEquals(XClusterConfigStatusType.Failed, xClusterConfig.getStatus()); @@ -541,8 +542,9 @@ public void testCreateXClusterIsSetupDoneFailure() { assertNotNull(taskInfo); assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(TaskType.XClusterConfigSetup, taskInfo.getSubTasks().get(4).getTaskType()); - String taskErrMsg = taskInfo.getSubTasks().get(4).getDetails().get("errorString").asText(); + // Two FreeUniverse subtasks for source and target. + assertEquals(TaskType.XClusterConfigSetup, taskInfo.getSubTasks().get(6).getTaskType()); + String taskErrMsg = taskInfo.getSubTasks().get(6).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "XClusterConfig(%s) operation failed: code: %s\nmessage: \"%s\"", diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java index b9b33e6c7a3d..ba9801394c37 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java @@ -86,6 +86,8 @@ public class DeleteXClusterConfigTest extends CommissionerBaseTest { List DELETE_XCLUSTER_CONFIG_TASK_SEQUENCE = ImmutableList.of( + TaskType.FreezeUniverse, + TaskType.FreezeUniverse, TaskType.XClusterConfigSetStatus, TaskType.DeleteReplication, TaskType.DeleteBootstrapIds, @@ -378,7 +380,7 @@ public void testDeleteXClusterFailure() { assertNotNull(subtaskGroup); assertEquals(DELETE_XCLUSTER_CONFIG_TASK_SEQUENCE.get(i), subtaskGroup.getTaskType()); } - String taskErrMsg = taskInfo.getSubTasks().get(1).getDetails().get("errorString").asText(); + String taskErrMsg = taskInfo.getSubTasks().get(3).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "Failed to delete replication for XClusterConfig(%s): %s", diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java index b02f641a15db..275b6ca07492 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java @@ -162,14 +162,28 @@ private void setupUniverseMultiAZ(boolean updateInProgress, boolean skipProvider Json.toJson(ImmutableMap.of())); private void assertTaskSequence(Map> subTasksByPosition, int numTasks) { - assertTaskSequence(subTasksByPosition, numTasks, numTasks); + assertTaskSequence(subTasksByPosition, numTasks, numTasks, false); } private void assertTaskSequence( - Map> subTasksByPosition, int numTasks, int numNamespaceDelete) { + Map> subTasksByPosition, int numTasks, boolean forceDelete) { + assertTaskSequence(subTasksByPosition, numTasks, numTasks, forceDelete); + } + + private void assertTaskSequence( + Map> subTasksByPosition, + int numTasks, + int numNamespaceDelete, + boolean forceDelete) { int position = 0; - for (TaskType taskType : KUBERNETES_DESTROY_UNIVERSE_TASKS) { - JsonNode expectedResults = KUBERNETES_DESTROY_UNIVERSE_EXPECTED_RESULTS.get(position); + if (!forceDelete) { + // Shift by 1 subtask due to FreezeUniverse. + assertEquals( + TaskType.FreezeUniverse, subTasksByPosition.get(position++).get(0).getTaskType()); + } + for (int i = 0; i < KUBERNETES_DESTROY_UNIVERSE_TASKS.size(); i++) { + TaskType taskType = KUBERNETES_DESTROY_UNIVERSE_TASKS.get(i); + JsonNode expectedResults = KUBERNETES_DESTROY_UNIVERSE_EXPECTED_RESULTS.get(i); List tasks = subTasksByPosition.get(position); if (expectedResults.equals( @@ -351,7 +365,7 @@ public void testForceDestroyKubernetesUniverseWithUpdateInProgress() { List subTasks = taskInfo.getSubTasks(); Map> subTasksByPosition = subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - assertTaskSequence(subTasksByPosition, 1); + assertTaskSequence(subTasksByPosition, 1, true); assertEquals(Success, taskInfo.getTaskState()); assertFalse(defaultCustomer.getUniverseUUIDs().contains(defaultUniverse.getUniverseUUID())); } @@ -444,7 +458,7 @@ public void testDestroyKubernetesUniverseSuccessMultiAZWithNamespace() { List subTasks = taskInfo.getSubTasks(); Map> subTasksByPosition = subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - assertTaskSequence(subTasksByPosition, 3, 1); + assertTaskSequence(subTasksByPosition, 3, 1, false); assertEquals(Success, taskInfo.getTaskState()); assertFalse(defaultCustomer.getUniverseUUIDs().contains(defaultUniverse.getUniverseUUID())); } diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java index cab91779ae08..ad7d7414a852 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java @@ -151,6 +151,7 @@ private void setupUniverseSingleAZ(boolean setMasters) { private static final List KUBERNETES_ADD_POD_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.KubernetesCommandExecutor, TaskType.KubernetesCheckNumPod, TaskType.KubernetesCommandExecutor, @@ -164,7 +165,6 @@ private void setupUniverseSingleAZ(boolean setMasters) { private List getExpectedAddPodTaskResults() { return ImmutableList.of( - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), Json.toJson(ImmutableMap.of("commandType", WAIT_FOR_PODS.name())), @@ -180,6 +180,7 @@ private List getExpectedAddPodTaskResults() { private static final List KUBERNETES_REMOVE_POD_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.UpdatePlacementInfo, TaskType.WaitForDataMove, TaskType.KubernetesCommandExecutor, @@ -193,14 +194,13 @@ private List getExpectedAddPodTaskResults() { private List getExpectedRemovePodTaskResults() { return ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), Json.toJson(ImmutableMap.of("commandType", WAIT_FOR_PODS.name())), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", POD_INFO.name())), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), @@ -209,6 +209,7 @@ private List getExpectedRemovePodTaskResults() { private static final List KUBERNETES_CHANGE_INSTANCE_TYPE_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.UpdatePlacementInfo, TaskType.KubernetesCommandExecutor, TaskType.KubernetesWaitForPod, @@ -229,6 +230,7 @@ private List getExpectedRemovePodTaskResults() { private List getExpectedChangeInstaceTypeResults() { return ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), Json.toJson(ImmutableMap.of("commandType", WAIT_FOR_POD.name())), @@ -254,11 +256,6 @@ private void assertTaskSequence( List resultList, String type) { int position = 0; - // Since we create two empty subGroupTasks for add (namespace create and apply secret) - // they need to skipped over. - if (type.equals("add")) { - position = 2; - } for (TaskType task : taskList) { List tasks = subTasksByPosition.get(position); // In the case of adding and wait for server, we need to ensure that @@ -275,11 +272,6 @@ private void assertTaskSequence( tasks.stream().map(TaskInfo::getDetails).collect(Collectors.toList()); assertJsonEqual(expectedResults, taskDetails.get(0)); position++; - - // Similar to add, we expect two empty subGroupTasks (namespace and volume delete). - if (position == 3 && type.equals("remove")) { - position += 2; - } } } diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java index 5ad795079528..b5cfd3d33e98 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java @@ -111,6 +111,10 @@ public class EditXClusterConfigTest extends CommissionerBaseTest { List RENAME_FAILURE_TASK_SEQUENCE = ImmutableList.of( + // Freeze for source. + TaskType.FreezeUniverse, + // Freeze for target. + TaskType.FreezeUniverse, TaskType.XClusterConfigSetStatus, TaskType.XClusterConfigRename, TaskType.XClusterConfigSetStatus, @@ -119,6 +123,10 @@ public class EditXClusterConfigTest extends CommissionerBaseTest { List ADD_TABLE_IS_ALTER_DONE_FAILURE = ImmutableList.of( + // Freeze for source. + TaskType.FreezeUniverse, + // Freeze for target. + TaskType.FreezeUniverse, TaskType.XClusterConfigSetStatus, TaskType.XClusterConfigSetStatusForTables, TaskType.BootstrapProducer, @@ -473,7 +481,7 @@ public void testRenameFailure() { assertEquals(RENAME_FAILURE_TASK_SEQUENCE.get(i), subtaskGroup.getTaskType()); } - String taskErrMsg = taskInfo.getSubTasks().get(1).getDetails().get("errorString").asText(); + String taskErrMsg = taskInfo.getSubTasks().get(3).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "Failed to rename XClusterConfig(%s): %s", xClusterConfig.getUuid(), renameErrMsg); @@ -710,8 +718,8 @@ public void testPauseResumeFailure() { assertNotNull(taskInfo); assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(TaskType.SetReplicationPaused, taskInfo.getSubTasks().get(1).getTaskType()); - String taskErrMsg = taskInfo.getSubTasks().get(1).getDetails().get("errorString").asText(); + assertEquals(TaskType.SetReplicationPaused, taskInfo.getSubTasks().get(3).getTaskType()); + String taskErrMsg = taskInfo.getSubTasks().get(3).getDetails().get("errorString").asText(); assertThat(taskErrMsg, containsString("Failed to pause/enable XClusterConfig")); assertThat(taskErrMsg, containsString(pauseResumeErrMsg)); assertEquals(XClusterConfigStatusType.Running, xClusterConfig.getStatus()); @@ -957,7 +965,7 @@ public void testAddTablesAlterFailure() { assertEquals(ADD_TABLE_IS_ALTER_DONE_FAILURE.get(i), subtaskGroup.getTaskType()); } - String taskErrMsg = taskInfo.getSubTasks().get(3).getDetails().get("errorString").asText(); + String taskErrMsg = taskInfo.getSubTasks().get(5).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "Failed to add tables to XClusterConfig(%s): %s", @@ -1039,7 +1047,7 @@ public void testAddTablesIsAlterDoneFailure() { assertEquals(ADD_TABLE_IS_ALTER_DONE_FAILURE.get(i), subtaskGroup.getTaskType()); } - String taskErrMsg = taskInfo.getSubTasks().get(3).getDetails().get("errorString").asText(); + String taskErrMsg = taskInfo.getSubTasks().get(5).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "XClusterConfig(%s) operation failed: code: %s\nmessage: \"%s\"", @@ -1179,8 +1187,8 @@ public void testRemoveTablesAlterFailure() { assertNotNull(taskInfo); assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(TaskType.XClusterConfigModifyTables, taskInfo.getSubTasks().get(2).getTaskType()); - String taskErrMsg = taskInfo.getSubTasks().get(2).getDetails().get("errorString").asText(); + assertEquals(TaskType.XClusterConfigModifyTables, taskInfo.getSubTasks().get(4).getTaskType()); + String taskErrMsg = taskInfo.getSubTasks().get(4).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "Failed to remove tables from XClusterConfig(%s): %s", diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java index 1d3cde794b95..46598e441252 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java @@ -102,6 +102,7 @@ private void setupUniverse(boolean updateInProgress) { private static final List PAUSE_UNIVERSE_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.AnsibleClusterServerCtl, TaskType.SetNodeState, @@ -113,6 +114,7 @@ private void setupUniverse(boolean updateInProgress) { private static final List PAUSE_UNIVERSE_EXPECTED_RESULTS = ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("process", "tserver", "command", "stop")), Json.toJson(ImmutableMap.of()), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java index 4dbbdad30c9f..0da1b698def0 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java @@ -131,6 +131,7 @@ private Cluster addReadReplica(Region region) { private static final List CLUSTER_DELETE_TASK_SEQUENCE = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.CheckLeaderlessTablets, TaskType.SetNodeState, TaskType.AnsibleDestroyServer, @@ -147,6 +148,7 @@ private Cluster addReadReplica(Region region) { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of())); private void assertClusterDeleteSequence(Map> subTasksByPosition) { diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java index b655b14ba819..b4324debf0d1 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java @@ -124,6 +124,7 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { private static final List RESUME_UNIVERSE_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.ResumeServer, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleClusterServerCtl, @@ -138,6 +139,7 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { private static final List RESUME_ENCRYPTION_AT_REST_UNIVERSE_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.ResumeServer, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleClusterServerCtl, @@ -153,6 +155,7 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { private static final List RESUME_UNIVERSE_EXPECTED_RESULTS = ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("process", "master", "command", "start")), @@ -167,6 +170,7 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { private static final List RESUME_ENCRYPTION_AT_REST_UNIVERSE_EXPECTED_RESULTS = ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("process", "master", "command", "start")), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java index 6bde8b42ba20..7c884822a7d2 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java @@ -154,6 +154,7 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { // @formatter:off private static final List START_MASTER_TASK_SEQUENCE = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.CheckLeaderlessTablets, TaskType.SetNodeState, TaskType.WaitForClockSync, // Ensure clock skew is low enough @@ -175,6 +176,7 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { private static final List START_MASTER_TASK_EXPECTED_RESULTS = ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("state", "Starting")), Json.toJson(ImmutableMap.of()), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfigTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfigTest.java index 6ad2b07e6b28..36224e9e6a3f 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfigTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/SyncXClusterConfigTest.java @@ -427,8 +427,8 @@ public void testSyncError() { List configList = XClusterConfig.getByTargetUniverseUUID(targetUniverseUUID); assertEquals(0, configList.size()); - assertEquals(TaskType.XClusterConfigSync, taskInfo.getSubTasks().get(0).getTaskType()); - String taskErrMsg = taskInfo.getSubTasks().get(0).getDetails().get("errorString").asText(); + assertEquals(TaskType.XClusterConfigSync, taskInfo.getSubTasks().get(1).getTaskType()); + String taskErrMsg = taskInfo.getSubTasks().get(1).getDetails().get("errorString").asText(); String expectedErrMsg = String.format( "Failed to getMasterClusterConfig from target universe (%s): %s", diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java index f9bdfa0e45f6..ce8918db9d50 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java @@ -238,6 +238,7 @@ private void setupUniverseMultiAZ(boolean setMasters) { private static final List KUBERNETES_UPGRADE_SOFTWARE_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.KubernetesCommandExecutor, TaskType.KubernetesCommandExecutor, TaskType.KubernetesWaitForPod, @@ -271,6 +272,7 @@ private void setupUniverseMultiAZ(boolean setMasters) { private static final List KUBERNETES_UPGRADE_GFLAG_TASKS = ImmutableList.of( + TaskType.FreezeUniverse, TaskType.UpdateAndPersistGFlags, TaskType.KubernetesCommandExecutor, TaskType.KubernetesCommandExecutor, @@ -304,6 +306,7 @@ private void setupUniverseMultiAZ(boolean setMasters) { private static List createUpgradeSoftwareResult(boolean isSingleAZ) { String namespace = isSingleAZ ? "demo-universe" : "demo-universe-az-2"; return ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson( ImmutableMap.of("commandType", KubernetesCommandExecutor.CommandType.POD_INFO.name())), Json.toJson( @@ -378,6 +381,7 @@ private static List createUpgradeSoftwareResult(boolean isSingleAZ) { private static List createUpdateGflagsResult(boolean isSingleAZ) { String namespace = isSingleAZ ? "demo-universe" : "demo-universe-az-2"; return ImmutableList.of( + Json.toJson(ImmutableMap.of()), Json.toJson( ImmutableMap.of( "masterGFlags", Json.parse("{\"master-flag\":\"m1\"}"), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverseTest.java deleted file mode 100644 index cfa0be9233a1..000000000000 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeUniverseTest.java +++ /dev/null @@ -1,2443 +0,0 @@ -// Copyright (c) YugaByte, Inc. - -package com.yugabyte.yw.commissioner.tasks; - -import static com.yugabyte.yw.commissioner.UserTaskDetails.SubTaskGroupType.DownloadingSoftware; -import static com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType.MASTER; -import static com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType.TSERVER; -import static com.yugabyte.yw.common.ModelFactory.createUniverse; -import static com.yugabyte.yw.common.TestHelper.createTempFile; -import static com.yugabyte.yw.models.TaskInfo.State.Failure; -import static com.yugabyte.yw.models.TaskInfo.State.Success; -import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.net.HostAndPort; -import com.yugabyte.yw.commissioner.Common; -import com.yugabyte.yw.commissioner.UserTaskDetails; -import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType; -import com.yugabyte.yw.commissioner.tasks.params.NodeTaskParams; -import com.yugabyte.yw.commissioner.tasks.subtasks.CreateRootVolumes; -import com.yugabyte.yw.common.ApiUtils; -import com.yugabyte.yw.common.NodeManager.NodeCommandType; -import com.yugabyte.yw.common.PlacementInfoUtil; -import com.yugabyte.yw.common.ShellResponse; -import com.yugabyte.yw.common.TestHelper; -import com.yugabyte.yw.common.certmgmt.CertConfigType; -import com.yugabyte.yw.common.utils.Pair; -import com.yugabyte.yw.forms.CertificateParams; -import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; -import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster; -import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; -import com.yugabyte.yw.forms.UpgradeParams; -import com.yugabyte.yw.forms.UpgradeTaskParams; -import com.yugabyte.yw.forms.UpgradeTaskParams.UpgradeTaskType; -import com.yugabyte.yw.models.*; -import com.yugabyte.yw.models.helpers.*; -import io.ebean.DB; -import io.ebean.SqlUpdate; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import junitparams.JUnitParamsRunner; -import junitparams.Parameters; -import junitparams.naming.TestCaseName; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatcher; -import org.mockito.InjectMocks; -import org.mockito.MockitoAnnotations; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.yb.client.*; -import org.yb.master.CatalogEntityInfo.SysClusterConfigEntryPB; -import play.libs.Json; - -@RunWith(JUnitParamsRunner.class) -@Deprecated -public class UpgradeUniverseTest extends CommissionerBaseTest { - @Rule public MockitoRule rule = MockitoJUnit.rule(); - - private static class CreateRootVolumesMatcher implements ArgumentMatcher { - private UUID azUUID; - - public CreateRootVolumesMatcher(UUID azUUID) { - this.azUUID = azUUID; - } - - @Override - public boolean matches(NodeTaskParams right) { - if (!(right instanceof CreateRootVolumes.Params)) { - return false; - } - - return right.azUuid.equals(this.azUUID); - } - } - - @InjectMocks UpgradeUniverse upgradeUniverse; - - private YBClient mockClient; - private Universe defaultUniverse; - - private Region region; - private AvailabilityZone az1; - private AvailabilityZone az2; - private AvailabilityZone az3; - - private static final String CERT_1_CONTENTS = - "-----BEGIN CERTIFICATE-----\n" - + "MIIDEjCCAfqgAwIBAgIUEdzNoxkMLrZCku6H1jQ4pUgPtpQwDQYJKoZIhvcNAQEL\n" - + "BQAwLzERMA8GA1UECgwIWXVnYWJ5dGUxGjAYBgNVBAMMEUNBIGZvciBZdWdhYnl0\n" - + "ZURCMB4XDTIwMTIyMzA3MjU1MVoXDTIxMDEyMjA3MjU1MVowLzERMA8GA1UECgwI\n" - + "WXVnYWJ5dGUxGjAYBgNVBAMMEUNBIGZvciBZdWdhYnl0ZURCMIIBIjANBgkqhkiG\n" - + "9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuLPcCR1KpVSs3B2515xNAR8ntfhOM5JjLl6Y\n" - + "WjqoyRQ4wiOg5fGQpvjsearpIntr5t6uMevpzkDMYY4U21KbIW8Vvg/kXiASKMmM\n" - + "W4ToH3Q0NfgLUNb5zJ8df3J2JZ5CgGSpipL8VPWsuSZvrqL7V77n+ndjMTUBNf57\n" - + "eW4VjzYq+YQhyloOlXtlfWT6WaAsoaVOlbU0GK4dE2rdeo78p2mS2wQZSBDXovpp\n" - + "0TX4zhT6LsJaRBZe49GE4SMkxz74alK1ezRvFcrPiNKr5NOYYG9DUUqFHWg47Bmw\n" - + "KbiZ5dLdyxgWRDbanwl2hOMfExiJhHE7pqgr8XcizCiYuUzlDwIDAQABoyYwJDAO\n" - + "BgNVHQ8BAf8EBAMCAuQwEgYDVR0TAQH/BAgwBgEB/wIBATANBgkqhkiG9w0BAQsF\n" - + "AAOCAQEAVI3NTJVNX4XTcVAxXXGumqCbKu9CCLhXxaJb+J8YgmMQ+s9lpmBuC1eB\n" - + "38YFdSEG9dKXZukdQcvpgf4ryehtvpmk03s/zxNXC5237faQQqejPX5nm3o35E3I\n" - + "ZQqN3h+mzccPaUvCaIlvYBclUAt4VrVt/W66kLFPsfUqNKVxm3B56VaZuQL1ZTwG\n" - + "mrIYBoaVT/SmEeIX9PNjlTpprDN/oE25fOkOxwHyI9ydVFkMCpBNRv+NisQN9c+R\n" - + "/SBXfs+07aqFgrGTte6/I4VZ/6vz2cWMwZU+TUg/u0fc0Y9RzOuJrZBV2qPAtiEP\n" - + "YvtLjmJF//b3rsty6NFIonSVgq6Nqw==\n" - + "-----END CERTIFICATE-----\n"; - - private static final String CERT_2_CONTENTS = - "-----BEGIN CERTIFICATE-----\n" - + "MIIDAjCCAeqgAwIBAgIGAXVCiJ4gMA0GCSqGSIb3DQEBCwUAMC4xFjAUBgNVBAMM\n" - + "DXliLWFkbWluLXRlc3QxFDASBgNVBAoMC2V4YW1wbGUuY29tMB4XDTIwMTAxOTIw\n" - + "MjQxMVoXDTIxMTAxOTIwMjQxMVowLjEWMBQGA1UEAwwNeWItYWRtaW4tdGVzdDEU\n" - + "MBIGA1UECgwLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\n" - + "AoIBAQCw8Mk+/MK0mP67ZEKL7cGyTzggau57MzTApveLfGF1Snln/Y7wGzgbskaM\n" - + "0udz46es9HdaC/jT+PzMAAD9MCtAe5YYSL2+lmWT+WHdeJWF4XC/AVkjqj81N6OS\n" - + "Uxio6ww0S9cAoDmF3gZlmkRwQcsruiZ1nVyQ7l+5CerQ02JwYBIYolUu/1qMruDD\n" - + "pLsJ9LPWXPw2JsgYWyuEB5W1xEPDl6+QLTEVCFc9oN6wJOJgf0Y6OQODBrDRxddT\n" - + "8h0mgJ6yzmkerR8VA0bknPQFeruWNJ/4PKDO9Itk5MmmYU/olvT5zMJ79K8RSvhN\n" - + "+3gO8N7tcswaRP7HbEUmuVTtjFDlAgMBAAGjJjAkMBIGA1UdEwEB/wQIMAYBAf8C\n" - + "AQEwDgYDVR0PAQH/BAQDAgLkMA0GCSqGSIb3DQEBCwUAA4IBAQCB10NLTyuqSD8/\n" - + "HmbkdmH7TM1q0V/2qfrNQW86ywVKNHlKaZp9YlAtBcY5SJK37DJJH0yKM3GYk/ee\n" - + "4aI566aj65gQn+wte9QfqzeotfVLQ4ZlhhOjDVRqSJVUdiW7yejHQLnqexdOpPQS\n" - + "vwi73Fz0zGNxqnNjSNtka1rmduGwP0fiU3WKtHJiPL9CQFtRKdIlskKUlXg+WulM\n" - + "x9yw5oa6xpsbCzSoS31fxYg71KAxVvKJYumdKV3ElGU/+AK1y4loyHv/kPp+59fF\n" - + "9Q8gq/A6vGFjoZtVuuKUlasbMocle4Y9/nVxqIxWtc+aZ8mmP//J5oVXyzPs56dM\n" - + "E1pTE1HS\n" - + "-----END CERTIFICATE-----\n"; - - @Override - @Before - public void setUp() { - super.setUp(); - - MockitoAnnotations.initMocks(this); - upgradeUniverse.setUserTaskUUID(UUID.randomUUID()); - region = Region.create(defaultProvider, "region-1", "Region 1", "yb-image-1"); - az1 = AvailabilityZone.createOrThrow(region, "az-1", "AZ 1", "subnet-1"); - az2 = AvailabilityZone.createOrThrow(region, "az-2", "AZ 2", "subnet-2"); - az3 = AvailabilityZone.createOrThrow(region, "az-3", "AZ 3", "subnet-3"); - UUID certUUID = UUID.randomUUID(); - Date date = new Date(); - CertificateParams.CustomCertInfo customCertInfo = new CertificateParams.CustomCertInfo(); - customCertInfo.rootCertPath = "rootCertPath"; - customCertInfo.nodeCertPath = "nodeCertPath"; - customCertInfo.nodeKeyPath = "nodeKeyPath"; - createTempFile("upgrade_universe_test_ca.crt", CERT_1_CONTENTS); - try { - CertificateInfo.create( - certUUID, - defaultCustomer.getUuid(), - "test", - date, - date, - TestHelper.TMP_PATH + "/upgrade_universe_test_ca.crt", - customCertInfo); - } catch (IOException | NoSuchAlgorithmException e) { - } - - // create default universe - UniverseDefinitionTaskParams.UserIntent userIntent = - new UniverseDefinitionTaskParams.UserIntent(); - userIntent.numNodes = 3; - userIntent.ybSoftwareVersion = "old-version"; - userIntent.accessKeyCode = "demo-access"; - userIntent.regionList = ImmutableList.of(region.getUuid()); - defaultUniverse = createUniverse(defaultCustomer.getId(), certUUID); - PlacementInfo pi = new PlacementInfo(); - PlacementInfoUtil.addPlacementZone(az1.getUuid(), pi, 1, 1, false); - PlacementInfoUtil.addPlacementZone(az2.getUuid(), pi, 1, 1, true); - PlacementInfoUtil.addPlacementZone(az3.getUuid(), pi, 1, 1, false); - - defaultUniverse = - Universe.saveDetails( - defaultUniverse.getUniverseUUID(), ApiUtils.mockUniverseUpdater(userIntent, pi, true)); - - // Setup mocks - mockClient = mock(YBClient.class); - try { - when(mockClient.getMasterClusterConfig()) - .thenAnswer( - i -> { - SysClusterConfigEntryPB.Builder configBuilder = - SysClusterConfigEntryPB.newBuilder().setVersion(defaultUniverse.getVersion()); - return new GetMasterClusterConfigResponse(1111, "", configBuilder.build(), null); - }); - UpgradeYsqlResponse mockUpgradeYsqlResponse = new UpgradeYsqlResponse(1000, "", null); - when(mockClient.upgradeYsql(any(HostAndPort.class), anyBoolean())) - .thenReturn(mockUpgradeYsqlResponse); - IsInitDbDoneResponse mockIsInitDbDoneResponse = - new IsInitDbDoneResponse(1000, "", true, true, null, null); - when(mockClient.getIsInitDbDone()).thenReturn(mockIsInitDbDoneResponse); - } catch (Exception ignored) { - fail(); - } - when(mockYBClient.getClient(any(), any())).thenReturn(mockClient); - when(mockYBClient.getClientWithConfig(any())).thenReturn(mockClient); - when(mockClient.waitForServer(any(HostAndPort.class), anyLong())).thenReturn(true); - when(mockClient.getLeaderMasterHostAndPort()) - .thenReturn(HostAndPort.fromString("10.0.0.2").withDefaultPort(7000)); - IsServerReadyResponse okReadyResp = new IsServerReadyResponse(0, "", null, 0, 0); - try { - when(mockClient.waitForMaster(any(HostAndPort.class), anyLong())).thenReturn(true); - when(mockClient.isServerReady(any(HostAndPort.class), anyBoolean())).thenReturn(okReadyResp); - when(mockClient.setFlag(any(HostAndPort.class), anyString(), anyString(), anyBoolean())) - .thenReturn(true); - ListMastersResponse listMastersResponse = mock(ListMastersResponse.class); - when(listMastersResponse.getMasters()).thenReturn(Collections.emptyList()); - when(mockClient.listMasters()).thenReturn(listMastersResponse); - } catch (Exception ignored) { - fail(); - } - ShellResponse dummyShellResponse = new ShellResponse(); - when(mockNodeManager.nodeCommand(any(), any())).thenReturn(dummyShellResponse); - ShellResponse successResponse = new ShellResponse(); - successResponse.message = "YSQL successfully upgraded to the latest version"; - when(mockNodeUniverseManager.runYbAdminCommand(any(), any(), any(), anyList(), anyLong())) - .thenReturn(successResponse); - - setFollowerLagMock(); - setLeaderlessTabletsMock(); - } - - private TaskInfo submitTask(UpgradeUniverse.Params taskParams, UpgradeTaskType taskType) { - return submitTask(taskParams, taskType, 2); - } - - private TaskInfo submitTask( - UpgradeUniverse.Params taskParams, UpgradeTaskType taskType, int expectedVersion) { - taskParams.setUniverseUUID(defaultUniverse.getUniverseUUID()); - taskParams.taskType = taskType; - taskParams.expectedUniverseVersion = expectedVersion; - // Need not sleep for default 3min in tests. - taskParams.sleepAfterMasterRestartMillis = 5; - taskParams.sleepAfterTServerRestartMillis = 5; - - try { - UUID taskUUID = commissioner.submit(TaskType.UpgradeUniverse, taskParams); - return waitForTask(taskUUID); - } catch (InterruptedException e) { - assertNull(e.getMessage()); - } - return null; - } - - private static final List PROPERTY_KEYS = ImmutableList.of("processType", "taskSubType"); - - private static final List NON_NODE_TASKS = - ImmutableList.of( - TaskType.LoadBalancerStateChange, - TaskType.UpdateAndPersistGFlags, - TaskType.UpdateSoftwareVersion, - TaskType.UnivSetCertificate, - TaskType.UniverseSetTlsParams, - TaskType.UniverseUpdateSucceeded); - - private static final List GFLAGS_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.AnsibleConfigureServers, - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.SetNodeState, - TaskType.WaitForServer); - - private static final List GFLAGS_ROLLING_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.WaitForEncryptionKeyInMemory, - TaskType.SetNodeState); - - private static final List CERTS_ROLLING_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.WaitForEncryptionKeyInMemory, - TaskType.SetNodeState); - - private static final List CERTS_NON_ROLLING_TASK_SEQUENCE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.SetNodeState, - TaskType.WaitForServer); - - private static final List GFLAGS_NON_ROLLING_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.AnsibleConfigureServers, - TaskType.SetNodeState, - TaskType.SetFlagInMemory, - TaskType.SetNodeState); - - private static final List SOFTWARE_FULL_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.SetNodeState, - TaskType.WaitForServer); - - private static final List SOFTWARE_ROLLING_UPGRADE_TASK_SEQUENCE_ACTIVE_ROLE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.WaitForEncryptionKeyInMemory, - TaskType.SetNodeState); - - private static final List SOFTWARE_ROLLING_UPGRADE_TASK_SEQUENCE_INACTIVE_ROLE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleConfigureServers, - TaskType.SetNodeState); - - private static final List ROLLING_RESTART_TASK_SEQUENCE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.WaitForEncryptionKeyInMemory, - TaskType.SetNodeState); - - private static final List VM_IMAGE_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.ReplaceRootVolume, - TaskType.AnsibleSetupServer, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.WaitForEncryptionKeyInMemory, - TaskType.UpdateNodeDetails); - - private static final List TOGGLE_TLS_ROLLING_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.WaitForServerReady, - TaskType.WaitForEncryptionKeyInMemory, - TaskType.SetNodeState); - - private static final List TOGGLE_TLS_NON_ROLLING_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.AnsibleConfigureServers, - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.SetNodeState, - TaskType.WaitForServer); - - private static final List TOGGLE_TLS_NON_RESTART_UPGRADE_TASK_SEQUENCE = - ImmutableList.of( - TaskType.AnsibleConfigureServers, - TaskType.SetNodeState, - TaskType.SetFlagInMemory, - TaskType.SetNodeState); - - private static final List RESIZE_NODE_UPGRADE_TASK_SEQUENCE_NO_MASTER = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.ChangeInstanceType, - TaskType.UpdateNodeDetails, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.SetNodeState); - - private static final List RESIZE_NODE_UPGRADE_TASK_SEQUENCE_IS_MASTER = - ImmutableList.of( - TaskType.SetNodeState, - TaskType.AnsibleClusterServerCtl, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForMasterLeader, - TaskType.ChangeMasterConfig, - TaskType.ChangeInstanceType, - TaskType.UpdateNodeDetails, - TaskType.AnsibleConfigureServers, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.ChangeMasterConfig, - TaskType.CheckFollowerLag, - TaskType.AnsibleClusterServerCtl, - TaskType.WaitForServer, - TaskType.SetNodeState); - - private int assertRollingRestartSequence( - Map> subTasksByPosition, ServerType serverType, int startPosition) { - int position = startPosition; - List taskSequence = ROLLING_RESTART_TASK_SEQUENCE; - List nodeOrder = getRollingUpgradeNodeOrder(serverType); - for (int nodeIdx : nodeOrder) { - String nodeName = String.format("host-n%d", nodeIdx); - for (TaskType type : taskSequence) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = tasks.get(0).getTaskType(); - - assertEquals(1, tasks.size()); - assertEquals(type, taskType); - if (!NON_NODE_TASKS.contains(taskType)) { - Map assertValues = - new HashMap<>(ImmutableMap.of("nodeName", nodeName, "nodeCount", 1)); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - return position; - } - - private int assertCertsRotateSequence( - Map> subTasksByPosition, - ServerType serverType, - int startPosition, - boolean isRollingUpgrade) { - int position = startPosition; - if (isRollingUpgrade) { - List taskSequence = CERTS_ROLLING_UPGRADE_TASK_SEQUENCE; - List nodeOrder = getRollingUpgradeNodeOrder(serverType); - for (int nodeIdx : nodeOrder) { - String nodeName = String.format("host-n%d", nodeIdx); - for (TaskType type : taskSequence) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = tasks.get(0).getTaskType(); - - assertEquals(1, tasks.size()); - assertEquals(type, taskType); - if (!NON_NODE_TASKS.contains(taskType)) { - Map assertValues = - new HashMap<>(ImmutableMap.of("nodeName", nodeName, "nodeCount", 1)); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - } else { - for (TaskType type : CERTS_NON_ROLLING_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = assertTaskType(tasks, type); - - if (NON_NODE_TASKS.contains(taskType)) { - assertEquals(1, tasks.size()); - } else { - Map assertValues = - ImmutableMap.of( - "nodeNames", - (Object) ImmutableList.of("host-n1", "host-n2", "host-n3"), - "nodeCount", - 3); - assertEquals(3, tasks.size()); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - - return position; - } - - private int assertSoftwareUpgradeSequence( - Map> subTasksByPosition, - ServerType serverType, - int startPosition, - boolean isRollingUpgrade, - boolean activeRole) { - int position = startPosition; - if (isRollingUpgrade) { - List taskSequence = - activeRole - ? SOFTWARE_ROLLING_UPGRADE_TASK_SEQUENCE_ACTIVE_ROLE - : SOFTWARE_ROLLING_UPGRADE_TASK_SEQUENCE_INACTIVE_ROLE; - List nodeOrder = getRollingUpgradeNodeOrderForSWUpgrade(serverType, activeRole); - for (int nodeIdx : nodeOrder) { - String nodeName = String.format("host-n%d", nodeIdx); - for (TaskType type : taskSequence) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = tasks.get(0).getTaskType(); - UserTaskDetails.SubTaskGroupType subTaskGroupType = tasks.get(0).getSubTaskGroupType(); - assertEquals(1, tasks.size()); - assertEquals(type, taskType); - if (!NON_NODE_TASKS.contains(taskType)) { - Map assertValues = - new HashMap<>(ImmutableMap.of("nodeName", nodeName, "nodeCount", 1)); - - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - String version = "new-version"; - String taskSubType = - subTaskGroupType.equals(DownloadingSoftware) ? "Download" : "Install"; - assertValues.putAll( - ImmutableMap.of( - "ybSoftwareVersion", version, - "processType", serverType.toString(), - "taskSubType", taskSubType)); - } - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - } else { - for (TaskType type : SOFTWARE_FULL_UPGRADE_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = assertTaskType(tasks, type); - - if (NON_NODE_TASKS.contains(taskType)) { - assertEquals(1, tasks.size()); - } else { - Map assertValues = - new HashMap<>( - ImmutableMap.of( - "nodeNames", - (Object) ImmutableList.of("host-n1", "host-n2", "host-n3"), - "nodeCount", - 3)); - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - String version = "new-version"; - assertValues.putAll( - ImmutableMap.of( - "ybSoftwareVersion", version, "processType", serverType.toString())); - } - assertEquals(3, tasks.size()); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - return position; - } - - private int assertGFlagsUpgradeSequence( - Map> subTasksByPosition, - ServerType serverType, - int startPosition, - UpgradeParams.UpgradeOption option) { - return assertGFlagsUpgradeSequence( - subTasksByPosition, serverType, startPosition, option, false); - } - - private int assertGFlagsUpgradeSequence( - Map> subTasksByPosition, - ServerType serverType, - int startPosition, - UpgradeParams.UpgradeOption option, - boolean isEdit) { - return assertGFlagsUpgradeSequence( - subTasksByPosition, serverType, startPosition, option, isEdit, false); - } - - private int assertGFlagsUpgradeSequence( - Map> subTasksByPosition, - ServerType serverType, - int startPosition, - UpgradeParams.UpgradeOption option, - boolean isEdit, - boolean isDelete) { - int position = startPosition; - switch (option) { - case ROLLING_UPGRADE: - List taskSequence = GFLAGS_ROLLING_UPGRADE_TASK_SEQUENCE; - List nodeOrder = getRollingUpgradeNodeOrder(serverType); - for (int nodeIdx : nodeOrder) { - String nodeName = String.format("host-n%d", nodeIdx); - for (TaskType type : taskSequence) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = tasks.get(0).getTaskType(); - assertEquals(1, tasks.size()); - assertEquals(type, taskType); - if (!NON_NODE_TASKS.contains(taskType)) { - Map assertValues = - new HashMap<>(ImmutableMap.of("nodeName", nodeName, "nodeCount", 1)); - - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - if (!isDelete) { - JsonNode gflagValue = - serverType.equals(MASTER) - ? Json.parse("{\"master-flag\":" + (isEdit ? "\"m2\"}" : "\"m1\"}")) - : Json.parse("{\"tserver-flag\":" + (isEdit ? "\"t2\"}" : "\"t1\"}")); - assertValues.putAll(ImmutableMap.of("gflags", gflagValue)); - } - } - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - break; - case NON_ROLLING_UPGRADE: - for (TaskType value : GFLAGS_UPGRADE_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = assertTaskType(tasks, value); - - if (NON_NODE_TASKS.contains(taskType)) { - assertEquals(1, tasks.size()); - } else { - Map assertValues = - new HashMap<>( - ImmutableMap.of( - "nodeNames", - (Object) ImmutableList.of("host-n1", "host-n2", "host-n3"), - "nodeCount", - 3)); - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - if (!isDelete) { - JsonNode gflagValue = - serverType.equals(MASTER) - ? Json.parse("{\"master-flag\":" + (isEdit ? "\"m2\"}" : "\"m1\"}")) - : Json.parse("{\"tserver-flag\":" + (isEdit ? "\"t2\"}" : "\"t1\"}")); - assertValues.putAll(ImmutableMap.of("gflags", gflagValue)); - } - assertValues.put("processType", serverType.toString()); - } - assertEquals(3, tasks.size()); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - break; - - case NON_RESTART_UPGRADE: - for (TaskType type : GFLAGS_NON_ROLLING_UPGRADE_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = assertTaskType(tasks, type); - - if (NON_NODE_TASKS.contains(taskType)) { - assertEquals(1, tasks.size()); - } else { - Map assertValues = - new HashMap<>( - ImmutableMap.of( - "nodeNames", - (Object) ImmutableList.of("host-n1", "host-n2", "host-n3"), - "nodeCount", - 3)); - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - if (!isDelete) { - JsonNode gflagValue = - serverType.equals(MASTER) - ? Json.parse("{\"master-flag\":" + (isEdit ? "\"m2\"}" : "\"m1\"}")) - : Json.parse("{\"tserver-flag\":" + (isEdit ? "\"t2\"}" : "\"t1\"}")); - assertValues.putAll(ImmutableMap.of("gflags", gflagValue)); - } - } - assertEquals(3, tasks.size()); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - break; - } - - return position; - } - - private int assertToggleTlsSequence( - Map> subTasksByPosition, - ServerType serverType, - int startPosition, - UpgradeParams.UpgradeOption option) { - int position = startPosition; - if (option == UpgradeParams.UpgradeOption.ROLLING_UPGRADE) { - List taskSequence = TOGGLE_TLS_ROLLING_UPGRADE_TASK_SEQUENCE; - List nodeOrder = getRollingUpgradeNodeOrder(serverType); - - for (int nodeIdx : nodeOrder) { - String nodeName = String.format("host-n%d", nodeIdx); - for (TaskType type : taskSequence) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = tasks.get(0).getTaskType(); - assertEquals(1, tasks.size()); - assertEquals(type, taskType); - if (!NON_NODE_TASKS.contains(taskType)) { - Map assertValues = - new HashMap<>(ImmutableMap.of("nodeName", nodeName, "nodeCount", 1)); - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - assertValues.putAll(ImmutableMap.of("processType", serverType.toString())); - } - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - } else if (option == UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE) { - for (TaskType type : TOGGLE_TLS_NON_ROLLING_UPGRADE_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = assertTaskType(tasks, type); - - if (NON_NODE_TASKS.contains(taskType)) { - assertEquals(1, tasks.size()); - } else { - Map assertValues = - new HashMap<>( - ImmutableMap.of( - "nodeNames", - (Object) ImmutableList.of("host-n1", "host-n2", "host-n3"), - "nodeCount", - 3)); - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - assertValues.putAll(ImmutableMap.of("processType", serverType.toString())); - } - assertEquals(3, tasks.size()); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } else { - for (TaskType type : TOGGLE_TLS_NON_RESTART_UPGRADE_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position); - TaskType taskType = assertTaskType(tasks, type); - - if (NON_NODE_TASKS.contains(taskType)) { - assertEquals(1, tasks.size()); - } else { - Map assertValues = - new HashMap<>( - ImmutableMap.of( - "nodeNames", - (Object) ImmutableList.of("host-n1", "host-n2", "host-n3"), - "nodeCount", - 3)); - if (taskType.equals(TaskType.AnsibleConfigureServers)) { - assertValues.putAll(ImmutableMap.of("processType", serverType.toString())); - } - assertEquals(3, tasks.size()); - assertNodeSubTask(tasks, assertValues); - } - position++; - } - } - - return position; - } - - public enum UpgradeType { - ROLLING_UPGRADE, - ROLLING_UPGRADE_MASTER_ONLY, - ROLLING_UPGRADE_TSERVER_ONLY, - FULL_UPGRADE, - FULL_UPGRADE_MASTER_ONLY, - FULL_UPGRADE_TSERVER_ONLY - } - - private int assertRollingRestartCommonTasks( - Map> subTasksByPosition, int startPosition) { - int position = startPosition; - List commonNodeTasks = - new ArrayList<>( - ImmutableList.of(TaskType.LoadBalancerStateChange, TaskType.UniverseUpdateSucceeded)); - for (TaskType commonNodeTask : commonNodeTasks) { - assertTaskType(subTasksByPosition.get(position), commonNodeTask); - position++; - } - return position; - } - - private int assertGFlagsCommonTasks( - Map> subTasksByPosition, - int startPosition, - UpgradeType type, - boolean isFinalStep) { - int position = startPosition; - List commonNodeTasks = new ArrayList<>(); - if (type.name().equals("ROLLING_UPGRADE") - || type.name().equals("ROLLING_UPGRADE_TSERVER_ONLY")) { - commonNodeTasks.add(TaskType.LoadBalancerStateChange); - } - - if (isFinalStep) { - commonNodeTasks.addAll( - ImmutableList.of(TaskType.UpdateAndPersistGFlags, TaskType.UniverseUpdateSucceeded)); - } - for (TaskType commonNodeTask : commonNodeTasks) { - assertTaskType(subTasksByPosition.get(position), commonNodeTask); - position++; - } - return position; - } - - private int assertSoftwareCommonTasks( - Map> subTasksByPosition, - int startPosition, - UpgradeType type, - boolean isFinalStep) { - int position = startPosition; - List commonNodeTasks = new ArrayList<>(); - - if (isFinalStep) { - if (type.name().equals("ROLLING_UPGRADE") - || type.name().equals("ROLLING_UPGRADE_TSERVER_ONLY")) { - commonNodeTasks.add(TaskType.LoadBalancerStateChange); - } - - commonNodeTasks.addAll( - ImmutableList.of( - TaskType.RunYsqlUpgrade, - TaskType.UpdateSoftwareVersion, - TaskType.UniverseUpdateSucceeded)); - } - for (TaskType commonNodeTask : commonNodeTasks) { - assertTaskType(subTasksByPosition.get(position), commonNodeTask); - position++; - } - return position; - } - - private int assertCertsRotateCommonTasks( - Map> subTasksByPosition, - int startPosition, - UpgradeType type, - boolean isFinalStep) { - int position = startPosition; - List commonNodeTasks = new ArrayList<>(); - - if (isFinalStep) { - if (type.name().equals("ROLLING_UPGRADE")) { - commonNodeTasks.add(TaskType.LoadBalancerStateChange); - } - - commonNodeTasks.addAll( - ImmutableList.of(TaskType.UnivSetCertificate, TaskType.UniverseUpdateSucceeded)); - } - for (TaskType commonNodeTask : commonNodeTasks) { - assertTaskType(subTasksByPosition.get(position), commonNodeTask); - position++; - } - return position; - } - - private int assertToggleTlsCommonTasks( - Map> subTasksByPosition, - int startPosition, - UpgradeParams.UpgradeOption upgradeOption, - boolean isMetadataUpdateStep) { - int position = startPosition; - List commonNodeTasks = new ArrayList<>(); - if (upgradeOption == UpgradeParams.UpgradeOption.ROLLING_UPGRADE) { - commonNodeTasks.add(TaskType.LoadBalancerStateChange); - } - if (isMetadataUpdateStep) { - commonNodeTasks.addAll(ImmutableList.of(TaskType.UniverseSetTlsParams)); - } - - for (TaskType commonNodeTask : commonNodeTasks) { - assertTaskType(subTasksByPosition.get(position), commonNodeTask); - position++; - } - - return position; - } - - private void assertNodeSubTask(List subTasks, Map assertValues) { - List nodeNames = - subTasks.stream() - .map(t -> t.getDetails().get("nodeName").textValue()) - .collect(Collectors.toList()); - int nodeCount = (int) assertValues.getOrDefault("nodeCount", 1); - assertEquals(nodeCount, nodeNames.size()); - if (nodeCount == 1) { - assertEquals(assertValues.get("nodeName"), nodeNames.get(0)); - } else { - List assertNodeNames = (List) assertValues.get("nodeNames"); - assertTrue(nodeNames.containsAll(assertNodeNames)); - assertEquals(assertNodeNames.size(), nodeNames.size()); - } - - List subTaskDetails = - subTasks.stream().map(TaskInfo::getDetails).collect(Collectors.toList()); - assertValues.forEach( - (expectedKey, expectedValue) -> { - if (!ImmutableList.of("nodeName", "nodeNames", "nodeCount").contains(expectedKey)) { - List values = - subTaskDetails.stream() - .map( - t -> { - JsonNode data = - PROPERTY_KEYS.contains(expectedKey) - ? t.get("properties").get(expectedKey) - : t.get(expectedKey); - return data.isObject() ? data : data.textValue(); - }) - .collect(Collectors.toList()); - values.forEach( - actualValue -> - assertEquals( - "Unexpected value for key " + expectedKey, expectedValue, actualValue)); - } - }); - } - - public void testResizeNodeUpgrade(int rf, int numInvocations) { - String intendedInstanceType = "c5.2xlarge"; - int intendedVolumeSize = 300; - - // Seed the database to have the intended type since we cannot mock static methods - String updateQuery = - "INSERT INTO instance_type (" - + "provider_uuid, instance_type_code, active, num_cores, mem_size_gb," - + "instance_type_details )" - + "VALUES (" - + ":providerUUID, :typeCode, true, :numCores, :memSize, :details)"; - SqlUpdate update = DB.sqlUpdate(updateQuery); - update.setParameter("providerUUID", defaultProvider.getUuid()); - update.setParameter("typeCode", intendedInstanceType); - update.setParameter("numCores", 8); - update.setParameter("memSize", 16); - update.setParameter("details", "{\"volumeDetailsList\":[],\"tenancy\":\"Shared\"}"); - int modifiedCount = DB.getDefault().execute(update); - assertEquals(1, modifiedCount); - - Region secondRegion = Region.create(defaultProvider, "region-2", "Region 2", "yb-image-1"); - AvailabilityZone.createOrThrow(secondRegion, "az-4", "AZ 4", "subnet-4"); - - Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - Cluster primaryCluster = universeDetails.getPrimaryCluster(); - UserIntent userIntent = primaryCluster.userIntent; - userIntent.providerType = Common.CloudType.aws; - userIntent.provider = defaultProvider.getUuid().toString(); - if (rf == 1) { - userIntent.numNodes = 1; - userIntent.replicationFactor = 1; - } - userIntent.instanceType = "c5.large"; - DeviceInfo deviceInfo = new DeviceInfo(); - deviceInfo.volumeSize = 250; - deviceInfo.numVolumes = 1; - userIntent.deviceInfo = deviceInfo; - - if (rf == 1) { - for (NodeDetails nodeDetail : universeDetails.nodeDetailsSet) { - if (nodeDetail.nodeIdx != 1) { - nodeDetail.isMaster = false; - } - nodeDetail.cloudInfo.private_ip = "1.2.3." + nodeDetail.nodeIdx; - } - } else { - for (int idx = userIntent.numNodes + 1; idx <= userIntent.numNodes + 2; idx++) { - NodeDetails node = new NodeDetails(); - node.nodeIdx = idx; - node.placementUuid = primaryCluster.uuid; - node.nodeName = "host-n" + idx; - node.isMaster = false; - node.isTserver = true; - node.cloudInfo = new CloudSpecificInfo(); - node.cloudInfo.instance_type = userIntent.getInstanceTypeForNode(node); - node.cloudInfo.private_ip = "1.2.3." + idx; - universeDetails.nodeDetailsSet.add(node); - } - } - - for (NodeDetails node : universeDetails.nodeDetailsSet) { - node.nodeUuid = UUID.randomUUID(); - } - userIntent.numNodes += 2; - universe.setUniverseDetails(universeDetails); - }; - defaultUniverse = Universe.saveDetails(defaultUniverse.getUniverseUUID(), updater); - - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.upgradeOption = UpgradeParams.UpgradeOption.ROLLING_UPGRADE; - taskParams.clusters.add(defaultUniverse.getUniverseDetails().getPrimaryCluster()); - taskParams.forceResizeNode = false; - DeviceInfo deviceInfo = new DeviceInfo(); - deviceInfo.volumeSize = intendedVolumeSize; - taskParams.getPrimaryCluster().userIntent.deviceInfo = deviceInfo; - taskParams.getPrimaryCluster().userIntent.instanceType = intendedInstanceType; - TaskInfo taskInfo = - submitTask(taskParams, UpgradeTaskType.ResizeNode, defaultUniverse.getVersion()); - verify(mockNodeManager, times(numInvocations)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - List changeDiskSize = subTasksByPosition.get(position++); - assertEquals( - changeDiskSize.size(), - defaultUniverse.getUniverseDetails().getPrimaryCluster().userIntent.numNodes); - assertTaskType(changeDiskSize, TaskType.InstanceActions); - - changeDiskSize.forEach( - task -> { - JsonNode details = task.getDetails(); - assertEquals(intendedVolumeSize, details.get("deviceInfo").get("volumeSize").asInt()); - assertEquals(1, details.get("deviceInfo").get("numVolumes").asInt()); - assertNotNull(details.get("instanceType")); - }); - - List persistChangeDiskSize = subTasksByPosition.get(position++); - assertEquals(1, persistChangeDiskSize.size()); - assertTaskType(persistChangeDiskSize, TaskType.PersistResizeNode); - - // Find start position of each node's subtasks - // nodeName to startPosition - Map nodeTasksStartPosition = new HashMap<>(); - String lastNode = null; - for (int j = position; j < subTasksByPosition.size(); j++) { - List tasks = subTasksByPosition.get(j); - assertEquals(1, tasks.size()); - - JsonNode nodeNameJson = tasks.get(0).getDetails().get("nodeName"); - if (nodeNameJson == null) { - continue; - } - String nodeName = nodeNameJson.asText("NoNodeName"); - if (lastNode == null || !lastNode.equals(nodeName)) { - nodeTasksStartPosition.put(nodeName, j); - if (nodeTasksStartPosition.size() - == defaultUniverse.getUniverseDetails().getPrimaryCluster().userIntent.numNodes) { - break; - } - lastNode = nodeName; - } - } - - for (NodeDetails node : defaultUniverse.getUniverseDetails().nodeDetailsSet) { - String nodeName = node.nodeName; - - int tmpPosition = nodeTasksStartPosition.get(nodeName); - - if (node.isMaster) { - for (int j = 0; - j < RESIZE_NODE_UPGRADE_TASK_SEQUENCE_IS_MASTER.size() - && tmpPosition < subTasksByPosition.size(); - j++) { - if (rf == 1) { - // Don't change master config for RF1 - if (RESIZE_NODE_UPGRADE_TASK_SEQUENCE_IS_MASTER.get(j) == TaskType.ChangeMasterConfig - || RESIZE_NODE_UPGRADE_TASK_SEQUENCE_IS_MASTER.get(j) == TaskType.CheckFollowerLag - || RESIZE_NODE_UPGRADE_TASK_SEQUENCE_IS_MASTER.get(j) - == TaskType.WaitForMasterLeader) { - continue; - } - } - - List tasks = subTasksByPosition.get(tmpPosition++); - assertEquals(1, tasks.size()); - TaskInfo task = tasks.get(0); - TaskType taskType = task.getTaskType(); - assertEquals(RESIZE_NODE_UPGRADE_TASK_SEQUENCE_IS_MASTER.get(j), taskType); - if (taskType == TaskType.ChangeInstanceType) { - JsonNode details = task.getDetails(); - assertNotNull(details.get("instanceType").asText().equals(intendedInstanceType)); - } - - if (position < tmpPosition) { - position = tmpPosition; - } - } - } else { - for (int j = 0; - j < RESIZE_NODE_UPGRADE_TASK_SEQUENCE_NO_MASTER.size() - && tmpPosition < subTasksByPosition.size(); - j++) { - List tasks = subTasksByPosition.get(tmpPosition++); - assertEquals(1, tasks.size()); - TaskInfo task = tasks.get(0); - TaskType taskType = task.getTaskType(); - assertEquals(RESIZE_NODE_UPGRADE_TASK_SEQUENCE_NO_MASTER.get(j), taskType); - if (taskType == TaskType.ChangeInstanceType) { - JsonNode details = task.getDetails(); - assertNotNull(details.get("instanceType").asText().equals(intendedInstanceType)); - } - - if (position < tmpPosition) { - position = tmpPosition; - } - } - } - } - - List persistChangeInstanceType = subTasksByPosition.get(position); - assertEquals(1, persistChangeInstanceType.size()); - assertTaskType(persistChangeInstanceType, TaskType.PersistResizeNode); - - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testResizeNodeUpgradeRF3() { - RuntimeConfigEntry.upsertGlobal("yb.checks.change_master_config.enabled", "false"); - testResizeNodeUpgrade(3, 29); - } - - @Test - public void testResizeNodeUpgradeRF1() { - testResizeNodeUpgrade(1, 15); - } - - @Test - public void testVMImageUpgrade() { - Region secondRegion = Region.create(defaultProvider, "region-2", "Region 2", "yb-image-1"); - AvailabilityZone az4 = AvailabilityZone.createOrThrow(secondRegion, "az-4", "AZ 4", "subnet-4"); - - Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - Cluster primaryCluster = universeDetails.getPrimaryCluster(); - UserIntent userIntent = primaryCluster.userIntent; - userIntent.regionList = ImmutableList.of(region.getUuid(), secondRegion.getUuid()); - userIntent.provider = defaultProvider.getUuid().toString(); - - PlacementInfo pi = primaryCluster.placementInfo; - PlacementInfoUtil.addPlacementZone(az4.getUuid(), pi, 1, 2, false); - universe.setUniverseDetails(universeDetails); - - for (int idx = userIntent.numNodes + 1; idx <= userIntent.numNodes + 2; idx++) { - NodeDetails node = new NodeDetails(); - node.nodeIdx = idx; - node.placementUuid = primaryCluster.uuid; - node.nodeName = "host-n" + idx; - node.isMaster = true; - node.isTserver = true; - node.cloudInfo = new CloudSpecificInfo(); - node.cloudInfo.private_ip = "1.2.3." + idx; - node.cloudInfo.az = az4.getCode(); - node.azUuid = az4.getUuid(); - universeDetails.nodeDetailsSet.add(node); - } - - for (NodeDetails node : universeDetails.nodeDetailsSet) { - node.nodeUuid = UUID.randomUUID(); - } - - userIntent.numNodes += 2; - }; - - defaultUniverse = Universe.saveDetails(defaultUniverse.getUniverseUUID(), updater); - - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; - taskParams.machineImages.put(region.getUuid(), "test-vm-image-1"); - taskParams.machineImages.put(secondRegion.getUuid(), "test-vm-image-2"); - - // expect a CreateRootVolume for each AZ - final int expectedRootVolumeCreationTasks = 4; - - Map> createVolumeOutput = - Arrays.asList(az1, az2, az3).stream() - .collect( - Collectors.toMap( - az -> az.getUuid(), - az -> - Collections.singletonList(String.format("root-volume-%s", az.getCode())))); - // AZ 4 has 2 nodes so return 2 volumes here - createVolumeOutput.put(az4.getUuid(), Arrays.asList("root-volume-4", "root-volume-5")); - - // Use output for verification and response is the raw string that parses into output. - Map createVolumeOutputResponse = - Stream.of(az1, az2, az3) - .collect( - Collectors.toMap( - az -> az.getUuid(), - az -> - String.format( - "{\"boot_disks_per_zone\":[\"root-volume-%s\"], " - + "\"root_device_name\":\"/dev/sda1\"}", - az.getCode()))); - createVolumeOutputResponse.put( - az4.getUuid(), - "{\"boot_disks_per_zone\":[\"root-volume-4\", \"root-volume-5\"], " - + "\"root_device_name\":\"/dev/sda1\"}"); - - ObjectMapper om = new ObjectMapper(); - for (Map.Entry e : createVolumeOutputResponse.entrySet()) { - when(mockNodeManager.nodeCommand( - eq(NodeCommandType.Create_Root_Volumes), - argThat(new CreateRootVolumesMatcher(e.getKey())))) - .thenReturn(ShellResponse.create(0, e.getValue())); - } - - TaskInfo taskInfo = - submitTask( - taskParams, UpgradeTaskParams.UpgradeTaskType.VMImage, defaultUniverse.getVersion()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - List createRootVolumeTasks = subTasksByPosition.get(position++); - assertTaskType(createRootVolumeTasks, TaskType.CreateRootVolumes); - assertEquals(expectedRootVolumeCreationTasks, createRootVolumeTasks.size()); - - createRootVolumeTasks.forEach( - task -> { - JsonNode details = task.getDetails(); - UUID azUuid = UUID.fromString(details.get("azUuid").asText()); - AvailabilityZone zone = - AvailabilityZone.find.query().fetch("region").where().idEq(azUuid).findOne(); - String machineImage = details.get("machineImage").asText(); - assertEquals(taskParams.machineImages.get(zone.getRegion().getUuid()), machineImage); - - String azUUID = details.get("azUuid").asText(); - if (azUUID.equals(az4.getUuid().toString())) { - assertEquals(2, details.get("numVolumes").asInt()); - } - }); - - List nodeOrder = Arrays.asList(1, 3, 4, 5, 2); - - Map replaceRootVolumeParams = new HashMap<>(); - - for (int nodeIdx : nodeOrder) { - String nodeName = String.format("host-n%d", nodeIdx); - - for (TaskType type : VM_IMAGE_UPGRADE_TASK_SEQUENCE) { - List tasks = subTasksByPosition.get(position++); - - assertEquals(1, tasks.size()); - - TaskInfo task = tasks.get(0); - TaskType taskType = task.getTaskType(); - - assertEquals(type, taskType); - - if (!NON_NODE_TASKS.contains(taskType)) { - Map assertValues = - new HashMap<>(ImmutableMap.of("nodeName", nodeName, "nodeCount", 1)); - - assertNodeSubTask(tasks, assertValues); - } - - if (taskType == TaskType.ReplaceRootVolume) { - JsonNode details = task.getDetails(); - UUID az = UUID.fromString(details.get("azUuid").asText()); - replaceRootVolumeParams.compute(az, (k, v) -> v == null ? 1 : v + 1); - } - } - } - - assertEquals(createVolumeOutput.keySet(), replaceRootVolumeParams.keySet()); - createVolumeOutput - .entrySet() - .forEach( - e -> assertEquals(e.getValue().size(), (int) replaceRootVolumeParams.get(e.getKey()))); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testSoftwareUpgradeWithSameVersion() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.ybSoftwareVersion = "old-version"; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Software); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - defaultUniverse.refresh(); - assertEquals(2, defaultUniverse.getVersion()); - // In case of an exception, no task should be queued. - assertEquals(0, taskInfo.getSubTasks().size()); - } - - @Test - public void testSoftwareUpgradeWithoutVersion() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Software); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - defaultUniverse.refresh(); - assertEquals(2, defaultUniverse.getVersion()); - // In case of an exception, no task should be queued. - assertEquals(0, taskInfo.getSubTasks().size()); - } - - @Test - public void testSoftwareUpgrade() { - // create default universe - UniverseDefinitionTaskParams.UserIntent userIntent = - new UniverseDefinitionTaskParams.UserIntent(); - userIntent.numNodes = 5; - userIntent.replicationFactor = 3; - userIntent.ybSoftwareVersion = "old-version"; - userIntent.accessKeyCode = "demo-access"; - userIntent.regionList = ImmutableList.of(region.getUuid()); - - PlacementInfo pi = new PlacementInfo(); - PlacementInfoUtil.addPlacementZone(az1.getUuid(), pi, 1, 2, false); - PlacementInfoUtil.addPlacementZone(az2.getUuid(), pi, 1, 1, true); - PlacementInfoUtil.addPlacementZone(az3.getUuid(), pi, 1, 2, false); - - defaultUniverse = - Universe.saveDetails( - defaultUniverse.getUniverseUUID(), - ApiUtils.mockUniverseUpdater( - userIntent, "host", true /* setMasters */, false /* updateInProgress */, pi)); - - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.ybSoftwareVersion = "new-version"; - TaskInfo taskInfo = - submitTask( - taskParams, UpgradeTaskParams.UpgradeTaskType.Software, defaultUniverse.getVersion()); - verify(mockNodeManager, times(33)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - List downloadTasks = subTasksByPosition.get(position++); - assertTaskType(downloadTasks, TaskType.AnsibleConfigureServers); - assertEquals(5, downloadTasks.size()); - position = assertSoftwareUpgradeSequence(subTasksByPosition, MASTER, position, true, true); - position = assertSoftwareUpgradeSequence(subTasksByPosition, MASTER, position, true, false); - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - position = - assertSoftwareCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, false); - position = assertSoftwareUpgradeSequence(subTasksByPosition, TSERVER, position, true, true); - assertSoftwareCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, true); - assertEquals(74, position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testSoftwareUpgradeWithReadReplica() { - // Update default universe. - UniverseDefinitionTaskParams.UserIntent userIntent = - new UniverseDefinitionTaskParams.UserIntent(); - userIntent.numNodes = 5; - userIntent.replicationFactor = 3; - userIntent.ybSoftwareVersion = "old-version"; - userIntent.accessKeyCode = "demo-access"; - userIntent.regionList = ImmutableList.of(region.getUuid()); - - PlacementInfo pi = new PlacementInfo(); - PlacementInfoUtil.addPlacementZone(az1.getUuid(), pi, 1, 2, false); - PlacementInfoUtil.addPlacementZone(az2.getUuid(), pi, 1, 1, true); - PlacementInfoUtil.addPlacementZone(az3.getUuid(), pi, 1, 2, false); - - defaultUniverse = - Universe.saveDetails( - defaultUniverse.getUniverseUUID(), - ApiUtils.mockUniverseUpdater( - userIntent, "host", true /* setMasters */, false /* updateInProgress */, pi)); - - pi = new PlacementInfo(); - AvailabilityZone az4 = AvailabilityZone.createOrThrow(region, "az-4", "AZ 4", "subnet-1"); - AvailabilityZone az5 = AvailabilityZone.createOrThrow(region, "az-5", "AZ 5", "subnet-2"); - AvailabilityZone az6 = AvailabilityZone.createOrThrow(region, "az-6", "AZ 6", "subnet-3"); - - // Currently read replica zones are always affinitized. - PlacementInfoUtil.addPlacementZone(az4.getUuid(), pi, 1, 1, false); - PlacementInfoUtil.addPlacementZone(az5.getUuid(), pi, 1, 1, true); - PlacementInfoUtil.addPlacementZone(az6.getUuid(), pi, 1, 1, false); - - userIntent.numNodes = 3; - - defaultUniverse = - Universe.saveDetails( - defaultUniverse.getUniverseUUID(), - ApiUtils.mockUniverseUpdaterWithReadReplica(userIntent, pi)); - - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.ybSoftwareVersion = "new-version"; - TaskInfo taskInfo = - submitTask( - taskParams, UpgradeTaskParams.UpgradeTaskType.Software, defaultUniverse.getVersion()); - verify(mockNodeManager, times(45)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - List downloadTasks = subTasksByPosition.get(position++); - assertTaskType(downloadTasks, TaskType.AnsibleConfigureServers); - assertEquals(8, downloadTasks.size()); - position = assertSoftwareUpgradeSequence(subTasksByPosition, MASTER, position, true, true); - position = assertSoftwareUpgradeSequence(subTasksByPosition, MASTER, position, true, false); - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - position = - assertSoftwareCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, false); - position = assertSoftwareUpgradeSequence(subTasksByPosition, TSERVER, position, true, true); - assertSoftwareCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, true); - assertEquals(98, position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testSoftwareNonRollingUpgrade() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.ybSoftwareVersion = "new-version"; - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Software); - ArgumentCaptor commandParams = ArgumentCaptor.forClass(NodeTaskParams.class); - verify(mockNodeManager, times(21)).nodeCommand(any(), commandParams.capture()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - int position = 0; - List downloadTasks = subTasksByPosition.get(position++); - assertTaskType(downloadTasks, TaskType.AnsibleConfigureServers); - assertEquals(3, downloadTasks.size()); - position = assertSoftwareUpgradeSequence(subTasksByPosition, MASTER, position, false, true); - position = assertSoftwareUpgradeSequence(subTasksByPosition, TSERVER, position, false, true); - assertSoftwareCommonTasks(subTasksByPosition, position, UpgradeType.FULL_UPGRADE, true); - assertEquals(13, position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testGFlagsNonRollingUpgrade() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = ImmutableMap.of("master-flag", "m1"); - taskParams.tserverGFlags = ImmutableMap.of("tserver-flag", "t1"); - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(18)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, MASTER, position, UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE); - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, TSERVER, position, UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks(subTasksByPosition, position, UpgradeType.FULL_UPGRADE, true); - assertEquals(14, position); - } - - @Test - public void testGFlagsNonRollingMasterOnlyUpgrade() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = ImmutableMap.of("master-flag", "m1"); - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(9)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, MASTER, position, UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.FULL_UPGRADE_MASTER_ONLY, true); - assertEquals(8, position); - } - - @Test - public void testGFlagsNonRollingTServerOnlyUpgrade() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.tserverGFlags = ImmutableMap.of("tserver-flag", "t1"); - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(9)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, TSERVER, position, UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.FULL_UPGRADE_TSERVER_ONLY, true); - - assertEquals(8, position); - } - - @Test - public void testGFlagsUpgradeWithMasterGFlags() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = ImmutableMap.of("master-flag", "m1"); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(9)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, MASTER, position, UpgradeParams.UpgradeOption.ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE_MASTER_ONLY, true); - assertEquals(26, position); - } - - @Test - public void testGFlagsUpgradeWithTServerGFlags() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.tserverGFlags = ImmutableMap.of("tserver-flag", "t1"); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - assertEquals(Success, taskInfo.getTaskState()); - ArgumentCaptor commandParams = ArgumentCaptor.forClass(NodeTaskParams.class); - verify(mockNodeManager, times(9)).nodeCommand(any(), commandParams.capture()); - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, TSERVER, position, UpgradeParams.UpgradeOption.ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE_TSERVER_ONLY, true); - assertEquals(28, position); - } - - @Test - public void testGFlagsUpgrade() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = ImmutableMap.of("master-flag", "m1"); - taskParams.tserverGFlags = ImmutableMap.of("tserver-flag", "t1"); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(18)).nodeCommand(any(), any()); - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, MASTER, position, UpgradeParams.UpgradeOption.ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, false); - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, TSERVER, position, UpgradeParams.UpgradeOption.ROLLING_UPGRADE); - position = - assertGFlagsCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, true); - assertEquals(52, position); - } - - @Test - public void testGFlagsUpgradeWithEmptyFlags() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - defaultUniverse.refresh(); - assertEquals(2, defaultUniverse.getVersion()); - // In case of an exception, no task should be queued. - assertEquals(0, taskInfo.getSubTasks().size()); - } - - @Test - public void testGFlagsUpgradeWithSameMasterFlags() { - SysClusterConfigEntryPB.Builder configBuilder = - SysClusterConfigEntryPB.newBuilder().setVersion(3); - GetMasterClusterConfigResponse mockConfigResponse = - new GetMasterClusterConfigResponse(1111, "", configBuilder.build(), null); - try { - when(mockClient.getMasterClusterConfig()).thenReturn(mockConfigResponse); - } catch (Exception e) { - } - when(mockYBClient.getClient(any(), any())).thenReturn(mockClient); - // Simulate universe created with master flags and tserver flags. - final Map masterFlags = ImmutableMap.of("master-flag", "m123"); - Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - UserIntent userIntent = universeDetails.getPrimaryCluster().userIntent; - userIntent.masterGFlags = masterFlags; - userIntent.tserverGFlags = ImmutableMap.of("tserver-flag", "t1"); - universe.setUniverseDetails(universeDetails); - }; - Universe.saveDetails(defaultUniverse.getUniverseUUID(), updater); - - // Upgrade with same master flags but different tserver flags should not run master tasks. - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = masterFlags; - taskParams.tserverGFlags = ImmutableMap.of("tserver-flag", "t2"); - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags, 3); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(9)).nodeCommand(any(), any()); - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, - TSERVER, - position, - UpgradeParams.UpgradeOption.ROLLING_UPGRADE, - true); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE_TSERVER_ONLY, true); - assertEquals(28, position); - } - - @Test - public void testGFlagsUpgradeWithSameTserverFlags() { - SysClusterConfigEntryPB.Builder configBuilder = - SysClusterConfigEntryPB.newBuilder().setVersion(3); - GetMasterClusterConfigResponse mockConfigResponse = - new GetMasterClusterConfigResponse(1111, "", configBuilder.build(), null); - try { - when(mockClient.getMasterClusterConfig()).thenReturn(mockConfigResponse); - } catch (Exception e) { - } - when(mockYBClient.getClient(any(), any())).thenReturn(mockClient); - // Simulate universe created with master flags and tserver flags. - final Map tserverFlags = ImmutableMap.of("tserver-flag", "m123"); - Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - UserIntent userIntent = universeDetails.getPrimaryCluster().userIntent; - userIntent.masterGFlags = ImmutableMap.of("master-flag", "m1"); - userIntent.tserverGFlags = tserverFlags; - universe.setUniverseDetails(universeDetails); - }; - Universe.saveDetails(defaultUniverse.getUniverseUUID(), updater); - - // Upgrade with same master flags but different tserver flags should not run master tasks. - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = ImmutableMap.of("master-flag", "m2"); - taskParams.tserverGFlags = tserverFlags; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags, 3); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(9)).nodeCommand(any(), any()); - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, - MASTER, - position, - UpgradeParams.UpgradeOption.ROLLING_UPGRADE, - true); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE_MASTER_ONLY, true); - assertEquals(26, position); - } - - @Test - public void testRemoveFlags() { - for (ServerType serverType : ImmutableList.of(MASTER, TSERVER)) { - if (serverType.equals(MASTER)) { - SysClusterConfigEntryPB.Builder configBuilder = - SysClusterConfigEntryPB.newBuilder().setVersion(3); - GetMasterClusterConfigResponse mockConfigResponse = - new GetMasterClusterConfigResponse(1111, "", configBuilder.build(), null); - try { - when(mockClient.getMasterClusterConfig()).thenReturn(mockConfigResponse); - } catch (Exception e) { - } - when(mockYBClient.getClient(any(), any())).thenReturn(mockClient); - } else if (serverType.equals(TSERVER)) { - SysClusterConfigEntryPB.Builder configBuilder = - SysClusterConfigEntryPB.newBuilder().setVersion(4); - GetMasterClusterConfigResponse mockConfigResponse = - new GetMasterClusterConfigResponse(1111, "", configBuilder.build(), null); - try { - when(mockClient.getMasterClusterConfig()).thenReturn(mockConfigResponse); - } catch (Exception e) { - } - when(mockYBClient.getClient(any(), any())).thenReturn(mockClient); - } - // Simulate universe created with master flags and tserver flags. - final Map tserverFlags = ImmutableMap.of("tserver-flag", "t1"); - final Map masterGFlags = ImmutableMap.of("master-flag", "m1"); - Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - UserIntent userIntent = universeDetails.getPrimaryCluster().userIntent; - userIntent.masterGFlags = masterGFlags; - userIntent.tserverGFlags = tserverFlags; - universe.setUniverseDetails(universeDetails); - }; - Universe.saveDetails(defaultUniverse.getUniverseUUID(), updater); - - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - // This is a delete operation on the master flags. - if (serverType == MASTER) { - taskParams.masterGFlags = new HashMap<>(); - taskParams.tserverGFlags = tserverFlags; - } else { - taskParams.masterGFlags = masterGFlags; - taskParams.tserverGFlags = new HashMap<>(); - } - - int expectedVersion = serverType == MASTER ? 3 : 4; - TaskInfo taskInfo = - submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags, expectedVersion); - assertEquals(Success, taskInfo.getTaskState()); - - int numInvocations = serverType == MASTER ? 9 : 18; - verify(mockNodeManager, times(numInvocations)).nodeCommand(any(), any()); - - List subTasks = new ArrayList<>(taskInfo.getSubTasks()); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - int position = 0; - if (serverType != MASTER) { - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - } - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, - serverType, - position, - UpgradeParams.UpgradeOption.ROLLING_UPGRADE, - true, - true); - position = - assertGFlagsCommonTasks( - subTasksByPosition, - position, - serverType == MASTER - ? UpgradeType.ROLLING_UPGRADE_MASTER_ONLY - : UpgradeType.ROLLING_UPGRADE_TSERVER_ONLY, - true); - assertEquals(serverType == MASTER ? 26 : 28, position); - } - } - - public void testGFlagsUpgradeNonRestart() throws Exception { - // Simulate universe created with master flags and tserver flags. - final Map tserverFlags = ImmutableMap.of("tserver-flag", "t1"); - Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - UserIntent userIntent = universeDetails.getPrimaryCluster().userIntent; - userIntent.masterGFlags = ImmutableMap.of("master-flag", "m1"); - userIntent.tserverGFlags = tserverFlags; - universe.setUniverseDetails(universeDetails); - }; - Universe.saveDetails(defaultUniverse.getUniverseUUID(), updater); - - // SetFlagResponse response = new SetFlagResponse(0, "", null); - when(mockClient.setFlag(any(), any(), any(), anyBoolean())).thenReturn(true); - - // Upgrade with same master flags but different tserver flags should not run master tasks. - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.masterGFlags = ImmutableMap.of("master-flag", "m2"); - taskParams.tserverGFlags = tserverFlags; - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_RESTART_UPGRADE; - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.GFlags, 3); - assertEquals(Success, taskInfo.getTaskState()); - verify(mockNodeManager, times(3)).nodeCommand(any(), any()); - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - int position = 0; - position = - assertGFlagsUpgradeSequence( - subTasksByPosition, MASTER, position, taskParams.upgradeOption, true); - position = - assertGFlagsCommonTasks( - subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE_MASTER_ONLY, true); - assertEquals(6, position); - } - - @Test - public void testRollingRestart() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Restart); - verify(mockNodeManager, times(12)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - position = assertRollingRestartSequence(subTasksByPosition, MASTER, position); - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - position = assertRollingRestartSequence(subTasksByPosition, TSERVER, position); - assertRollingRestartCommonTasks(subTasksByPosition, position); - assertEquals(43, position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testCertUpdateRolling() { - defaultUniverse.save(); - UUID certUUID = UUID.randomUUID(); - Date date = new Date(); - CertificateParams.CustomCertInfo customCertInfo = new CertificateParams.CustomCertInfo(); - customCertInfo.rootCertPath = "rootCertPath1"; - customCertInfo.nodeCertPath = "nodeCertPath1"; - customCertInfo.nodeKeyPath = "nodeKeyPath1"; - createTempFile("upgrade_universe_test_ca2.crt", CERT_1_CONTENTS); - - try { - CertificateInfo.create( - certUUID, - defaultCustomer.getUuid(), - "test2", - date, - date, - TestHelper.TMP_PATH + "/upgrade_universe_test_ca2.crt", - customCertInfo); - } catch (IOException | NoSuchAlgorithmException e) { - } - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.certUUID = certUUID; - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Certs); - verify(mockNodeManager, times(15)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - List downloadTasks = subTasksByPosition.get(position++); - assertTaskType(downloadTasks, TaskType.AnsibleConfigureServers); - assertEquals(3, downloadTasks.size()); - position = assertCertsRotateSequence(subTasksByPosition, MASTER, position, true); - assertTaskType(subTasksByPosition.get(position++), TaskType.LoadBalancerStateChange); - position = assertCertsRotateSequence(subTasksByPosition, TSERVER, position, true); - assertCertsRotateCommonTasks(subTasksByPosition, position, UpgradeType.ROLLING_UPGRADE, true); - assertEquals(44, position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testCertUpdateNonRolling() { - defaultUniverse.save(); - UUID certUUID = UUID.randomUUID(); - Date date = new Date(); - CertificateParams.CustomCertInfo customCertInfo = new CertificateParams.CustomCertInfo(); - customCertInfo.rootCertPath = "rootCertPath1"; - customCertInfo.nodeCertPath = "nodeCertPath1"; - customCertInfo.nodeKeyPath = "nodeKeyPath1"; - createTempFile("upgrade_universe_test_ca2.crt", CERT_1_CONTENTS); - try { - CertificateInfo.create( - certUUID, - defaultCustomer.getUuid(), - "test2", - date, - date, - TestHelper.TMP_PATH + "/upgrade_universe_test_ca2.crt", - customCertInfo); - } catch (IOException | NoSuchAlgorithmException e) { - } - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.certUUID = certUUID; - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE; - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Certs); - verify(mockNodeManager, times(15)).nodeCommand(any(), any()); - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - List downloadTasks = subTasksByPosition.get(position++); - assertTaskType(downloadTasks, TaskType.AnsibleConfigureServers); - assertEquals(3, downloadTasks.size()); - position = assertCertsRotateSequence(subTasksByPosition, MASTER, position, false); - position = assertCertsRotateSequence(subTasksByPosition, TSERVER, position, false); - assertCertsRotateCommonTasks(subTasksByPosition, position, UpgradeType.FULL_UPGRADE, true); - assertEquals(11, position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - } - - @Test - public void testCertUpdateFailureDifferentCerts() { - defaultUniverse.save(); - UUID certUUID = UUID.randomUUID(); - Date date = new Date(); - CertificateParams.CustomCertInfo customCertInfo = new CertificateParams.CustomCertInfo(); - customCertInfo.rootCertPath = "rootCertPath1"; - customCertInfo.nodeCertPath = "nodeCertPath1"; - customCertInfo.nodeKeyPath = "nodeKeyPath1"; - createTempFile("upgrade_universe_test_ca2.crt", CERT_2_CONTENTS); - try { - CertificateInfo.create( - certUUID, - defaultCustomer.getUuid(), - "test2", - date, - date, - TestHelper.TMP_PATH + "/upgrade_universe_test_ca2.crt", - customCertInfo); - } catch (IOException | NoSuchAlgorithmException e) { - } - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.certUUID = certUUID; - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.Certs); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - defaultUniverse.refresh(); - assertEquals(2, defaultUniverse.getVersion()); - // In case of an exception, no task should be queued. - assertEquals(0, taskInfo.getSubTasks().size()); - } - - private int getNodeToNodeChangeForToggleTls(boolean enableNodeToNodeEncrypt) { - return defaultUniverse - .getUniverseDetails() - .getPrimaryCluster() - .userIntent - .enableNodeToNodeEncrypt - != enableNodeToNodeEncrypt - ? (enableNodeToNodeEncrypt ? 1 : -1) - : 0; - } - - private void prepareUniverseForToggleTls( - boolean nodeToNode, - boolean clientToNode, - boolean rootAndClientRootCASame, - UUID rootCA, - UUID clientRootCA) - throws IOException, NoSuchAlgorithmException { - createTempFile("upgrade_universe_test_ca.crt", "test content"); - - CertificateInfo.create( - rootCA, - defaultCustomer.getUuid(), - "test1", - new Date(), - new Date(), - "privateKey", - TestHelper.TMP_PATH + "/upgrade_universe_test_ca.crt", - CertConfigType.SelfSigned); - - if (!rootAndClientRootCASame && !rootCA.equals(clientRootCA)) { - CertificateInfo.create( - clientRootCA, - defaultCustomer.getUuid(), - "test1", - new Date(), - new Date(), - "privateKey", - TestHelper.TMP_PATH + "/upgrade_universe_test_ca.crt", - CertConfigType.SelfSigned); - } - - defaultUniverse = - Universe.saveDetails( - defaultUniverse.getUniverseUUID(), - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); - PlacementInfo placementInfo = universeDetails.getPrimaryCluster().placementInfo; - UserIntent userIntent = universeDetails.getPrimaryCluster().userIntent; - userIntent.enableNodeToNodeEncrypt = nodeToNode; - userIntent.enableClientToNodeEncrypt = clientToNode; - universeDetails.allowInsecure = true; - universeDetails.rootAndClientRootCASame = rootAndClientRootCASame; - if (nodeToNode || (rootAndClientRootCASame && clientToNode)) { - universeDetails.rootCA = rootCA; - } - if (clientToNode) { - universeDetails.setClientRootCA(clientRootCA); - } - if (nodeToNode || clientToNode) { - universeDetails.allowInsecure = false; - } - universeDetails.upsertPrimaryCluster(userIntent, placementInfo); - universe.setUniverseDetails(universeDetails); - }, - false); - } - - private UpgradeUniverse.Params getTaskParamsForToggleTls( - boolean nodeToNode, - boolean clientToNode, - boolean rootAndClientRootCASame, - UUID rootCA, - UUID clientRootCA, - UpgradeParams.UpgradeOption upgradeOption) { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.upgradeOption = upgradeOption; - taskParams.enableNodeToNodeEncrypt = nodeToNode; - taskParams.enableClientToNodeEncrypt = clientToNode; - taskParams.rootAndClientRootCASame = rootAndClientRootCASame; - taskParams.rootCA = rootCA; - taskParams.setClientRootCA(clientRootCA); - return taskParams; - } - - private Pair - getUpgradeOptionsForToggleTls(int nodeToNodeChange, boolean isRolling) { - if (isRolling) { - return new Pair<>( - nodeToNodeChange < 0 - ? UpgradeParams.UpgradeOption.NON_RESTART_UPGRADE - : UpgradeParams.UpgradeOption.ROLLING_UPGRADE, - nodeToNodeChange > 0 - ? UpgradeParams.UpgradeOption.NON_RESTART_UPGRADE - : UpgradeParams.UpgradeOption.ROLLING_UPGRADE); - } else { - return new Pair<>( - nodeToNodeChange < 0 - ? UpgradeParams.UpgradeOption.NON_RESTART_UPGRADE - : UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE, - nodeToNodeChange > 0 - ? UpgradeParams.UpgradeOption.NON_RESTART_UPGRADE - : UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE); - } - } - - private Pair getExpectedValuesForToggleTls(UpgradeUniverse.Params taskParams) { - int nodeToNodeChange = getNodeToNodeChangeForToggleTls(taskParams.enableNodeToNodeEncrypt); - int expectedPosition = 1; - int expectedNumberOfInvocations = 0; - - if (taskParams.enableNodeToNodeEncrypt || taskParams.enableClientToNodeEncrypt) { - expectedPosition += 1; - expectedNumberOfInvocations += 3; - } - - if (taskParams.upgradeOption == UpgradeParams.UpgradeOption.ROLLING_UPGRADE) { - if (nodeToNodeChange != 0) { - expectedPosition += 58; - expectedNumberOfInvocations += 24; - } else { - expectedPosition += 50; - expectedNumberOfInvocations += 18; - } - } else { - if (nodeToNodeChange != 0) { - expectedPosition += 20; - expectedNumberOfInvocations += 24; - } else { - expectedPosition += 12; - expectedNumberOfInvocations += 18; - } - } - - return new Pair<>(expectedPosition, expectedNumberOfInvocations); - } - - @Test - public void testToggleTlsUpgradeInvalidUpgradeOption() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.enableNodeToNodeEncrypt = true; - taskParams.upgradeOption = UpgradeParams.UpgradeOption.NON_RESTART_UPGRADE; - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.ToggleTls, -1); - if (taskInfo == null) { - fail(); - } - - defaultUniverse.refresh(); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(0, taskInfo.getSubTasks().size()); - } - - @Test - public void testToggleTlsUpgradeWithoutChangeInParams() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.ToggleTls, -1); - if (taskInfo == null) { - fail(); - } - - defaultUniverse.refresh(); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(0, taskInfo.getSubTasks().size()); - } - - @Test - public void testToggleTlsUpgradeWithoutRootCa() { - UpgradeUniverse.Params taskParams = new UpgradeUniverse.Params(); - taskParams.enableNodeToNodeEncrypt = true; - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.ToggleTls, -1); - if (taskInfo == null) { - fail(); - } - - defaultUniverse.refresh(); - verify(mockNodeManager, times(0)).nodeCommand(any(), any()); - assertEquals(Failure, taskInfo.getTaskState()); - assertEquals(0, taskInfo.getSubTasks().size()); - } - - @Test - @Parameters({ - // "true, true, false, false, true, true",//both cannot be same, when rootCA is disabled - "true, true, false, false, false, true", - // "true, false, false, false, true, true",//both cannot be same, when root CA is disabled - "true, false, false, false, false, true", - "false, true, false, true, true, true", - "false, true, false, true, false, true", - "false, false, false, true, true, true", - "false, false, false, true, false, true", - "true, true, false, true, false, true", - "true, false, false, true, true, true", - "false, true, false, false, false, true", - // "false, false, false, false, true, true",//both cannot be same when rootCA is disabled - // "true, true, true, false, true, true",//both cannot be same when rootCA is disabled - "true, true, true, false, false, true", - // "true, false, true, false, true, true",//both cannot be same when rootCA is disabled - "true, false, true, false, false, true", - "false, true, true, true, true, true", - "false, true, true, true, false, true", - "false, false, true, true, true, true", - "false, false, true, true, false, true", - "true, true, true, true, false, true", - "true, false, true, true, true, true", - "false, true, true, false, false, true", - // "false, false, true, false, true, true",//both cannot be same when rootCA is disabled - "true, true, true, false, true, false", - "true, true, true, false, false, false", - "true, false, true, false, true, false", - "true, false, true, false, false, false", - "false, true, true, true, true, false", - "false, true, true, true, false, false", - "false, false, true, true, true, false", - "false, false, true, true, false, false", - "true, true, true, true, false, false", - "true, false, true, true, true, false", - "false, true, true, false, false, false", - "false, false, true, false, true, false", - "true, true, false, false, true, false", - "true, true, false, false, false, false", - "true, false, false, false, true, false", - "true, false, false, false, false, false", - "false, true, false, true, true, false", - "false, true, false, true, false, false", - "false, false, false, true, true, false", - "false, false, false, true, false, false", - "true, true, false, true, false, false", - "true, false, false, true, true, false", - "false, true, false, false, false, false", - "false, false, false, false, true, false" - }) - @TestCaseName( - "testToggleTlsNonRollingUpgradeWhen" - + "CurrNodeToNode:{0}_CurrClientToNode:{1}_CurrRootAndClientRootCASame:{2}" - + "_NodeToNode:{3}_ClientToNode:{4}_RootAndClientRootCASame:{5}") - public void testToggleTlsNonRollingUpgrade( - boolean currentNodeToNode, - boolean currentClientToNode, - boolean currRootAndClientRootCASame, - boolean nodeToNode, - boolean clientToNode, - boolean rootAndClientRootCASame) - throws IOException, NoSuchAlgorithmException { - - if (rootAndClientRootCASame && (!nodeToNode || !clientToNode)) { - // bothCASame cannot be true when either one of nodeToNode or clientToNode is disabled - rootAndClientRootCASame = false; - } - - UUID rootCA = UUID.randomUUID(); - UUID clientRootCA = UUID.randomUUID(); - prepareUniverseForToggleTls( - currentNodeToNode, currentClientToNode, currRootAndClientRootCASame, rootCA, clientRootCA); - UpgradeUniverse.Params taskParams = - getTaskParamsForToggleTls( - nodeToNode, - clientToNode, - rootAndClientRootCASame, - rootCA, - clientRootCA, - UpgradeParams.UpgradeOption.NON_ROLLING_UPGRADE); - - int nodeToNodeChange = getNodeToNodeChangeForToggleTls(nodeToNode); - Pair upgrade = - getUpgradeOptionsForToggleTls(nodeToNodeChange, false); - Pair expectedValues = getExpectedValuesForToggleTls(taskParams); - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.ToggleTls, -1); - if (taskInfo == null) { - fail(); - } - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - if (taskParams.enableNodeToNodeEncrypt || taskParams.enableClientToNodeEncrypt) { - // Cert update tasks will be non rolling - List certUpdateTasks = subTasksByPosition.get(position++); - assertTaskType(certUpdateTasks, TaskType.AnsibleConfigureServers); - assertEquals(3, certUpdateTasks.size()); - } - // First round gflag update tasks - position = assertToggleTlsSequence(subTasksByPosition, MASTER, position, upgrade.getFirst()); - position = assertToggleTlsSequence(subTasksByPosition, TSERVER, position, upgrade.getFirst()); - position = assertToggleTlsCommonTasks(subTasksByPosition, position, upgrade.getFirst(), true); - if (nodeToNodeChange != 0) { - // Second round gflag update tasks - position = assertToggleTlsSequence(subTasksByPosition, MASTER, position, upgrade.getSecond()); - position = - assertToggleTlsSequence(subTasksByPosition, TSERVER, position, upgrade.getSecond()); - } - - assertEquals((int) expectedValues.getFirst(), position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - Universe universe = Universe.getOrBadRequest(defaultUniverse.getUniverseUUID()); - if (nodeToNode || (rootAndClientRootCASame && clientToNode)) - assertEquals(taskParams.rootCA, universe.getUniverseDetails().rootCA); - else assertNull(universe.getUniverseDetails().rootCA); - if (clientToNode) - assertEquals(taskParams.getClientRootCA(), universe.getUniverseDetails().getClientRootCA()); - else assertNull(universe.getUniverseDetails().getClientRootCA()); - assertEquals( - nodeToNode, - universe.getUniverseDetails().getPrimaryCluster().userIntent.enableNodeToNodeEncrypt); - assertEquals( - clientToNode, - universe.getUniverseDetails().getPrimaryCluster().userIntent.enableClientToNodeEncrypt); - assertEquals(rootAndClientRootCASame, universe.getUniverseDetails().rootAndClientRootCASame); - verify(mockNodeManager, times(expectedValues.getSecond())).nodeCommand(any(), any()); - } - - @Test - @Parameters({ - // "true, true, false, false, true, true",//both cannot be same when rootCA is disabled - "true, true, false, false, false, true", - // "true, false, false, false, true, true",//both cannot be same when rootCA is disabled - "true, false, false, false, false, true", - "false, true, false, true, true, true", - "false, true, false, true, false, true", - "false, false, false, true, true, true", - "false, false, false, true, false, true", - "true, true, false, true, false, true", - "true, false, false, true, true, true", - "false, true, false, false, false, true", - // "false, false, false, false, true, true",//both cannot be same when rootCA is disabled - // "true, true, true, false, true, true",//both cannot be same when rootCA is disabled - "true, true, true, false, false, true", - // "true, false, true, false, true, true",//both cannot be same when rootCA is disabled - "true, false, true, false, false, true", - "false, true, true, true, true, true", - "false, true, true, true, false, true", - "false, false, true, true, true, true", - "false, false, true, true, false, true", - "true, true, true, true, false, true", - "true, false, true, true, true, true", - "false, true, true, false, false, true", - // "false, false, true, false, true, true",//both cannot be same when rootCA is disabled - "true, true, true, false, true, false", - "true, true, true, false, false, false", - "true, false, true, false, true, false", - "true, false, true, false, false, false", - "false, true, true, true, true, false", - "false, true, true, true, false, false", - "false, false, true, true, true, false", - "false, false, true, true, false, false", - "true, true, true, true, false, false", - "true, false, true, true, true, false", - "false, true, true, false, false, false", - "false, false, true, false, true, false", - "true, true, false, false, true, false", - "true, true, false, false, false, false", - "true, false, false, false, true, false", - "true, false, false, false, false, false", - "false, true, false, true, true, false", - "false, true, false, true, false, false", - "false, false, false, true, true, false", - "false, false, false, true, false, false", - "true, true, false, true, false, false", - "true, false, false, true, true, false", - "false, true, false, false, false, false", - "false, false, false, false, true, false" - }) - @TestCaseName( - "testToggleTlsRollingUpgradeWhen" - + "CurrNodeToNode:{0}_CurrClientToNode:{1}_CurrRootAndClientRootCASame:{2}" - + "_NodeToNode:{3}_ClientToNode:{4}_RootAndClientRootCASame:{5}") - public void testToggleTlsRollingUpgrade( - boolean currentNodeToNode, - boolean currentClientToNode, - boolean currRootAndClientRootCASame, - boolean nodeToNode, - boolean clientToNode, - boolean rootAndClientRootCASame) - throws IOException, NoSuchAlgorithmException { - - if (rootAndClientRootCASame && (!nodeToNode || !clientToNode)) { - // bothCASame cannot be true when either of nodeToNode or clientToNode are false - rootAndClientRootCASame = false; - } - UUID rootCA = UUID.randomUUID(); - UUID clientRootCA = UUID.randomUUID(); - prepareUniverseForToggleTls( - currentNodeToNode, currentClientToNode, currRootAndClientRootCASame, rootCA, clientRootCA); - UpgradeUniverse.Params taskParams = - getTaskParamsForToggleTls( - nodeToNode, - clientToNode, - rootAndClientRootCASame, - rootCA, - rootAndClientRootCASame ? rootCA : clientRootCA, - UpgradeParams.UpgradeOption.ROLLING_UPGRADE); - - int nodeToNodeChange = getNodeToNodeChangeForToggleTls(nodeToNode); - Pair upgrade = - getUpgradeOptionsForToggleTls(nodeToNodeChange, true); - Pair expectedValues = getExpectedValuesForToggleTls(taskParams); - - TaskInfo taskInfo = submitTask(taskParams, UpgradeTaskParams.UpgradeTaskType.ToggleTls, -1); - if (taskInfo == null) { - fail(); - } - - List subTasks = taskInfo.getSubTasks(); - Map> subTasksByPosition = - subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); - - int position = 0; - if (taskParams.enableNodeToNodeEncrypt || taskParams.enableClientToNodeEncrypt) { - // Cert update tasks will be non rolling - List certUpdateTasks = subTasksByPosition.get(position++); - assertTaskType(certUpdateTasks, TaskType.AnsibleConfigureServers); - assertEquals(3, certUpdateTasks.size()); - } - // First round gflag update tasks - position = assertToggleTlsSequence(subTasksByPosition, MASTER, position, upgrade.getFirst()); - position = assertToggleTlsCommonTasks(subTasksByPosition, position, upgrade.getFirst(), false); - position = assertToggleTlsSequence(subTasksByPosition, TSERVER, position, upgrade.getFirst()); - position = assertToggleTlsCommonTasks(subTasksByPosition, position, upgrade.getFirst(), true); - if (nodeToNodeChange != 0) { - // Second round gflag update tasks - position = assertToggleTlsSequence(subTasksByPosition, MASTER, position, upgrade.getSecond()); - position = - assertToggleTlsCommonTasks(subTasksByPosition, position, upgrade.getSecond(), false); - position = - assertToggleTlsSequence(subTasksByPosition, TSERVER, position, upgrade.getSecond()); - position = - assertToggleTlsCommonTasks(subTasksByPosition, position, upgrade.getSecond(), false); - } - - assertEquals((int) expectedValues.getFirst(), position); - assertEquals(100.0, taskInfo.getPercentCompleted(), 0); - assertEquals(Success, taskInfo.getTaskState()); - Universe universe = Universe.getOrBadRequest(defaultUniverse.getUniverseUUID()); - if (nodeToNode || (rootAndClientRootCASame && clientToNode)) - assertEquals(rootCA, universe.getUniverseDetails().rootCA); - else assertNull(universe.getUniverseDetails().rootCA); - if (clientToNode) - assertEquals(taskParams.getClientRootCA(), universe.getUniverseDetails().getClientRootCA()); - else assertNull(universe.getUniverseDetails().getClientRootCA()); - assertEquals( - nodeToNode, - universe.getUniverseDetails().getPrimaryCluster().userIntent.enableNodeToNodeEncrypt); - assertEquals( - clientToNode, - universe.getUniverseDetails().getPrimaryCluster().userIntent.enableClientToNodeEncrypt); - assertEquals(rootAndClientRootCASame, universe.getUniverseDetails().rootAndClientRootCASame); - verify(mockNodeManager, times(expectedValues.getSecond())).nodeCommand(any(), any()); - } - - private List getRollingUpgradeNodeOrder(ServerType serverType) { - return serverType == MASTER - ? - // We need to check that the master leader is upgraded last. - Arrays.asList(1, 3, 2) - : - // We need to check that isAffinitized zone node is upgraded getFirst(). - defaultUniverse.getUniverseDetails().getReadOnlyClusters().isEmpty() - ? Arrays.asList(2, 1, 3) - : - // Primary cluster getFirst(), then read replica. - Arrays.asList(2, 1, 3, 6, 4, 5); - } - - // Software upgrade has more nodes - 3 masters + 2 tservers only. - private List getRollingUpgradeNodeOrderForSWUpgrade( - ServerType serverType, boolean activeRole) { - return serverType == MASTER - ? - // We need to check that the master leader is upgraded last. - (activeRole ? Arrays.asList(1, 3, 2) : Arrays.asList(4, 5)) - : - // We need to check that isAffinitized zone node is upgraded getFirst(). - defaultUniverse.getUniverseDetails().getReadOnlyClusters().isEmpty() - ? Arrays.asList(3, 1, 2, 4, 5) - : - // Primary cluster getFirst(), then read replica. - Arrays.asList(3, 1, 2, 4, 5, 8, 6, 7); - } -}