From cf8656023afd36bd1f0d9f598d022c24882ee54d Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Tue, 14 Oct 2025 22:19:49 -0700 Subject: [PATCH 1/7] PHOENIX-7566 ZK to SystemTable Sync and Event Reactor for Failover --- .../phoenix/jdbc/ClusterRoleRecord.java | 4 +- .../phoenix/jdbc/HAGroupStateListener.java | 9 +- .../phoenix/jdbc/HAGroupStoreClient.java | 555 ++++++++++++++---- .../phoenix/jdbc/HAGroupStoreManager.java | 326 ++++++++-- .../phoenix/jdbc/HAGroupStoreRecord.java | 80 ++- .../apache/phoenix/query/QueryServices.java | 18 +- .../phoenix/query/QueryServicesOptions.java | 12 +- .../RegionServerEndpointService.proto | 1 - .../PhoenixRegionServerEndpoint.java | 3 +- .../hbase/index/IndexRegionObserver.java | 24 +- ...rverEndpointWithConsistentFailoverIT.java} | 40 +- ...IndexRegionObserverMutationBlockingIT.java | 100 +++- .../jdbc/HAGroupStateSubscriptionIT.java | 167 +++--- .../phoenix/jdbc/HAGroupStoreClientIT.java | 219 +++++-- .../phoenix/jdbc/HAGroupStoreManagerIT.java | 401 +++++++++++-- .../apache/phoenix/jdbc/PhoenixHAAdminIT.java | 66 ++- .../phoenix/jdbc/HAGroupStoreRecordTest.java | 67 ++- 17 files changed, 1641 insertions(+), 451 deletions(-) rename phoenix-core/src/it/java/org/apache/phoenix/end2end/{PhoenixRegionServerEndpointITWithConsistentFailover.java => PhoenixRegionServerEndpointWithConsistentFailoverIT.java} (85%) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java index 3b71f8f9cd9..40963802ef8 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java @@ -95,7 +95,7 @@ public HAGroupStoreRecord.HAGroupState getDefaultHAGroupState() { case OFFLINE: return HAGroupStoreRecord.HAGroupState.OFFLINE; case ACTIVE_TO_STANDBY: - return HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY; + return HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY; case STANDBY_TO_ACTIVE: return HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE; case UNKNOWN: @@ -352,6 +352,4 @@ public String toPrettyString() { return toString(); } } - - } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStateListener.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStateListener.java index 634f50e0d0c..1446514f8cb 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStateListener.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStateListener.java @@ -43,15 +43,22 @@ public interface HAGroupStateListener { * consider delegating to a separate thread.

* * @param haGroupName the name of the HA group that transitioned + * @param fromState the previous state before the transition + * can be null for initial state. + * Also, can be inaccurate in case there is + * connection loss to ZK and multiple state changes happen in between. * @param toState the new state after the transition * @param modifiedTime the time the state transition occurred * @param clusterType whether this transition occurred on the local or peer cluster + * @param lastSyncStateTimeInMs the time we were in sync state, can be null. * * @throws Exception implementations may throw exceptions, but they will be * logged and will not prevent other listeners from being notified */ void onStateChange(String haGroupName, + HAGroupState fromState, HAGroupState toState, long modifiedTime, - ClusterType clusterType); + ClusterType clusterType, + Long lastSyncStateTimeInMs); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java index e4743bc7fdd..eeae13f311c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -33,6 +34,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; @@ -70,6 +74,8 @@ import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK; +import static org.apache.phoenix.query.QueryServices.HA_GROUP_STORE_SYNC_INTERVAL_SECONDS; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS; /** @@ -79,14 +85,15 @@ */ public class HAGroupStoreClient implements Closeable { - public static final String ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE = "phoenix" - + ZKPaths.PATH_SEPARATOR + "consistentHA" - + ZKPaths.PATH_SEPARATOR + "groupState"; + public static final String ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE = "phoenix" + + ZKPaths.PATH_SEPARATOR + "consistentHA"; private static final long HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS = 30000L; // Multiplier for ZK session timeout to account for time it will take for HMaster to abort // the region server in case ZK connection is lost from the region server. @VisibleForTesting static final double ZK_SESSION_TIMEOUT_MULTIPLIER = 1.1; + // Maximum jitter in seconds for sync job start time (10 seconds) + private static final long SYNC_JOB_MAX_JITTER_SECONDS = 10; private PhoenixHAAdmin phoenixHaAdmin; private PhoenixHAAdmin peerPhoenixHaAdmin; private static final Logger LOGGER = LoggerFactory.getLogger(HAGroupStoreClient.class); @@ -105,8 +112,6 @@ public class HAGroupStoreClient implements Closeable { private final Configuration conf; // ZK URL for the current cluster and HAGroupName private String zkUrl; - // Peer ZK URL for peer cluster and HAGroupName - private String peerZKUrl = null; // Peer Custom Event Listener private final PathChildrenCacheListener peerCustomPathChildrenCacheListener; // Wait time for sync mode @@ -119,13 +124,8 @@ public class HAGroupStoreClient implements Closeable { // Map key format: "clusterType:targetState" -> Set private final ConcurrentHashMap> targetStateSubscribers = new ConcurrentHashMap<>(); - // Policy for the HA group - private HighAvailabilityPolicy policy; - private ClusterRole clusterRole; - private ClusterRole peerClusterRole; - private String clusterUrl; - private String peerClusterUrl; - private long clusterRoleRecordVersion; + // Scheduled executor for periodic sync job + private ScheduledExecutorService syncExecutor; public static HAGroupStoreClient getInstance(Configuration conf, String haGroupName) throws SQLException { return getInstanceForZkUrl(conf, haGroupName, null); @@ -188,8 +188,14 @@ public static List getHAGroupNames(String zkUrl) throws SQLException { while (rs.next()) { String zkUrl1 = rs.getString(ZK_URL_1); String zkUrl2 = rs.getString(ZK_URL_2); - String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK); - String formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK); + String formattedZkUrl1 = null; + String formattedZkUrl2 = null; + if (StringUtils.isNotBlank(zkUrl1)) { + formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK); + } + if (StringUtils.isNotBlank(zkUrl2)) { + formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK); + } String formattedZkUrl = JDBCUtil.formatUrl(zkUrl, RegistryType.ZK); if (StringUtils.equals(formattedZkUrl1, formattedZkUrl) || StringUtils.equals(formattedZkUrl2, formattedZkUrl)) { @@ -214,11 +220,9 @@ public static List getHAGroupNames(String zkUrl) throws SQLException { // Custom Event Listener this.peerCustomPathChildrenCacheListener = peerPathChildrenCacheListener; try { - // Initialize HAGroupStoreClient attributes - initializeHAGroupStoreClientAttributes(haGroupName); // Initialize Phoenix HA Admin this.phoenixHaAdmin = new PhoenixHAAdmin(this.zkUrl, - conf, ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + conf, ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); // Initialize local cache this.pathChildrenCache = initializePathChildrenCache(phoenixHaAdmin, pathChildrenCacheListener, ClusterType.LOCAL); @@ -229,6 +233,8 @@ public static List getHAGroupNames(String zkUrl) throws SQLException { // Initialize peer cache maybeInitializePeerPathChildrenCache(); } + // Start periodic sync job + startPeriodicSyncJob(); } catch (Exception e) { this.isHealthy = false; @@ -249,9 +255,7 @@ public void rebuild() throws Exception { throw new IOException("HAGroupStoreClient is not healthy"); } LOGGER.info("Rebuilding HAGroupStoreClient for HA group {}", haGroupName); - initializeHAGroupStoreClientAttributes(haGroupName); initializeZNodeIfNeeded(); - maybeInitializePeerPathChildrenCache(); // NOTE: this is a BLOCKING method. // Completely rebuild the internal cache by querying for all needed data @@ -275,7 +279,7 @@ public HAGroupStoreRecord getHAGroupStoreRecord() throws IOException { if (!isHealthy) { throw new IOException("HAGroupStoreClient is not healthy"); } - return fetchCacheRecord(this.pathChildrenCache, ClusterType.LOCAL).getLeft(); + return fetchCacheRecordAndPopulateZKIfNeeded(this.pathChildrenCache, ClusterType.LOCAL).getLeft(); } /** @@ -286,15 +290,17 @@ public HAGroupStoreRecord getHAGroupStoreRecord() throws IOException { * @throws IOException if the client is not healthy or the operation fails * @throws StaleHAGroupStoreRecordVersionException if the version is stale * @throws InvalidClusterRoleTransitionException when transition is not valid + * @throws SQLException */ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupState) - throws IOException, StaleHAGroupStoreRecordVersionException, - InvalidClusterRoleTransitionException { + throws IOException, + InvalidClusterRoleTransitionException, + SQLException, StaleHAGroupStoreRecordVersionException { Preconditions.checkNotNull(haGroupState, "haGroupState cannot be null"); if (!isHealthy) { throw new IOException("HAGroupStoreClient is not healthy"); } - Pair cacheRecord = fetchCacheRecord( + Pair cacheRecord = fetchCacheRecordAndPopulateZKIfNeeded( this.pathChildrenCache, ClusterType.LOCAL); HAGroupStoreRecord currentHAGroupStoreRecord = cacheRecord.getLeft(); Stat currentHAGroupStoreRecordStat = cacheRecord.getRight(); @@ -310,7 +316,6 @@ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupStat // Once state changes back to ACTIVE_IN_SYNC or the role is // NOT ACTIVE or ACTIVE_TO_STANDBY // set the time to null to mark that we are current(or we don't have any reader). - // TODO: Verify that for reader this is the correct approach. Long lastSyncTimeInMs = currentHAGroupStoreRecord .getLastSyncStateTimeInMs(); ClusterRole clusterRole = haGroupState.getClusterRole(); @@ -327,15 +332,80 @@ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupStat currentHAGroupStoreRecord.getProtocolVersion(), currentHAGroupStoreRecord.getHaGroupName(), haGroupState, - lastSyncTimeInMs + lastSyncTimeInMs, + currentHAGroupStoreRecord.getPolicy(), + currentHAGroupStoreRecord.getPeerZKUrl(), + currentHAGroupStoreRecord.getClusterUrl(), + currentHAGroupStoreRecord.getPeerClusterUrl(), + currentHAGroupStoreRecord.getAdminCRRVersion() ); - // TODO: Check if cluster role is changing, if so, we need to update - // the system table first - // Lock the row in System Table and make sure update is reflected - // in all regionservers - // It should automatically update the ZK record as well. - phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, + try { + phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, newHAGroupStoreRecord, currentHAGroupStoreRecordStat.getVersion()); + } catch (StaleHAGroupStoreRecordVersionException e) { + LOGGER.debug("Failed to update HAGroupStoreRecord for HA group " + + haGroupName + " with cached stat version " + + currentHAGroupStoreRecordStat.getVersion() + + " checking if state is already updated in local cache", e); + // Check against current cached record, + // hoping that record is updated in local cache. + Pair cachedRecord + = fetchCacheRecordAndPopulateZKIfNeeded(pathChildrenCache, ClusterType.LOCAL); + currentHAGroupStoreRecord = cachedRecord.getLeft(); + Stat previousHAGroupStoreRecordStat = currentHAGroupStoreRecordStat; + currentHAGroupStoreRecordStat = cachedRecord.getRight(); + if (currentHAGroupStoreRecord != null + && currentHAGroupStoreRecord.getHAGroupState() + == haGroupState) { + LOGGER.debug("HAGroupStoreRecord for HA group {} is already updated" + + "with state {}, no need to update", + haGroupName, haGroupState); + return; + // Check if the cached version is not updated, only then check with ZK. + } else if (currentHAGroupStoreRecordStat != null + && currentHAGroupStoreRecordStat.getVersion() + == previousHAGroupStoreRecordStat.getVersion()) { + try { + // Check against record in ZK, if it is still what we don't want, + // throw an exception. + currentHAGroupStoreRecord + = phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName) + .getLeft(); + if (currentHAGroupStoreRecord != null + && currentHAGroupStoreRecord.getHAGroupState() + == haGroupState) { + LOGGER.debug("HAGroupStoreRecord for HA group {} is already " + + "updated with state {}, no need to update", + haGroupName, haGroupState); + return; + } + throw e; + } catch (StaleHAGroupStoreRecordVersionException e2) { + throw e2; + } + } + } + // If cluster role is changing, if so, we update, + // the system table on best effort basis. + // We also have a periodic job which syncs the ZK + // state with System Table periodically. + if (currentHAGroupStoreRecord.getClusterRole() != clusterRole) { + HAGroupStoreRecord peerZkRecord = getHAGroupStoreRecordFromPeer(); + ClusterRoleRecord.ClusterRole peerClusterRole = peerZkRecord != null + ? peerZkRecord.getClusterRole() + : ClusterRoleRecord.ClusterRole.UNKNOWN; + SystemTableHAGroupRecord systemTableRecord = new SystemTableHAGroupRecord( + HighAvailabilityPolicy.valueOf(newHAGroupStoreRecord.getPolicy()), + clusterRole, + peerClusterRole, + newHAGroupStoreRecord.getClusterUrl(), + newHAGroupStoreRecord.getPeerClusterUrl(), + this.zkUrl, + newHAGroupStoreRecord.getPeerZKUrl(), + newHAGroupStoreRecord.getAdminCRRVersion() + ); + updateSystemTableHAGroupRecordSilently(haGroupName, systemTableRecord); + } } else { LOGGER.info("Not updating HAGroupStoreRecord for HA group {} with state {}", haGroupName, haGroupState); @@ -351,17 +421,19 @@ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupStat * @throws IOException */ public ClusterRoleRecord getClusterRoleRecord() throws IOException { + HAGroupStoreRecord currentHAGroupStoreRecord = getHAGroupStoreRecord(); HAGroupStoreRecord peerHAGroupStoreRecord = getHAGroupStoreRecordFromPeer(); ClusterRoleRecord.ClusterRole peerClusterRole = peerHAGroupStoreRecord != null ? peerHAGroupStoreRecord.getClusterRole() : ClusterRole.UNKNOWN; return new ClusterRoleRecord(this.haGroupName, - this.policy, - this.clusterUrl, - this.clusterRole, - this.peerClusterUrl, + HighAvailabilityPolicy.valueOf( + currentHAGroupStoreRecord.getPolicy()), + currentHAGroupStoreRecord.getClusterUrl(), + currentHAGroupStoreRecord.getHAGroupState().getClusterRole(), + currentHAGroupStoreRecord.getPeerClusterUrl(), peerClusterRole, - this.clusterRoleRecordVersion); + currentHAGroupStoreRecord.getAdminCRRVersion()); } /** @@ -374,41 +446,108 @@ public HAGroupStoreRecord getHAGroupStoreRecordFromPeer() throws IOException { if (!isHealthy) { throw new IOException("HAGroupStoreClient is not healthy"); } - return fetchCacheRecord(this.peerPathChildrenCache, ClusterType.PEER).getLeft(); + return fetchCacheRecordAndPopulateZKIfNeeded(this.peerPathChildrenCache, ClusterType.PEER).getLeft(); } - private void initializeZNodeIfNeeded() throws IOException, - StaleHAGroupStoreRecordVersionException { + private void initializeZNodeIfNeeded() throws IOException, SQLException { // Sometimes the ZNode might not be available in local cache yet, so better to check // in ZK directly if we need to initialize Pair cacheRecordFromZK = phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(this.haGroupName); HAGroupStoreRecord haGroupStoreRecord = cacheRecordFromZK.getLeft(); - HAGroupState defaultHAGroupState = this.clusterRole.getDefaultHAGroupState(); - // Initialize lastSyncTimeInMs only if we start in ACTIVE_NOT_IN_SYNC state - // and ZNode is not already present - Long lastSyncTimeInMs = defaultHAGroupState.equals(HAGroupState.ACTIVE_NOT_IN_SYNC) - ? System.currentTimeMillis() - : null; - HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord( - HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, - haGroupName, - this.clusterRole.getDefaultHAGroupState(), - lastSyncTimeInMs - ); - // Only update current ZNode if it doesn't have the same role as present in System Table. - // If not exists, then create ZNode - // TODO: Discuss if this approach is what reader needs. - if (haGroupStoreRecord != null && - !haGroupStoreRecord.getClusterRole().equals(this.clusterRole)) { - phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, - newHAGroupStoreRecord, cacheRecordFromZK.getRight().getVersion()); - } else if (haGroupStoreRecord == null) { + // Only if the ZNode is not present, we need to create it from System Table + if (haGroupStoreRecord == null) { + SystemTableHAGroupRecord systemTableRecord + = getSystemTableHAGroupRecord(this.haGroupName); + Preconditions.checkNotNull(systemTableRecord, + "System Table HAGroupRecord cannot be null"); + HAGroupStoreRecord.HAGroupState defaultHAGroupState + = systemTableRecord.getClusterRole().getDefaultHAGroupState(); + Long lastSyncTimeInMs + = defaultHAGroupState.equals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC) + ? System.currentTimeMillis() + : null; + HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord( + HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, + haGroupName, + defaultHAGroupState, + lastSyncTimeInMs, + systemTableRecord.getPolicy().toString(), + systemTableRecord.getPeerZKUrl(), + systemTableRecord.getClusterUrl(), + systemTableRecord.getPeerClusterUrl(), + systemTableRecord.getAdminCRRVersion() + ); phoenixHaAdmin.createHAGroupStoreRecordInZooKeeper(newHAGroupStoreRecord); } } - private void initializeHAGroupStoreClientAttributes(String haGroupName) throws SQLException { + /** + * Inner class to hold system table HA group record + */ + public static class SystemTableHAGroupRecord { + private final HighAvailabilityPolicy policy; + private final ClusterRoleRecord.ClusterRole clusterRole; + private final ClusterRoleRecord.ClusterRole peerClusterRole; + private final String clusterUrl; + private final String peerClusterUrl; + private final String zkUrl; + private final String peerZKUrl; + private final long adminCRRVersion; + + public SystemTableHAGroupRecord(HighAvailabilityPolicy policy, + ClusterRoleRecord.ClusterRole clusterRole, + ClusterRoleRecord.ClusterRole peerClusterRole, + String clusterUrl, + String peerClusterUrl, + String zkUrl, + String peerZKUrl, + long adminCRRVersion) { + this.policy = policy; + this.clusterRole = clusterRole; + this.peerClusterRole = peerClusterRole; + this.clusterUrl = clusterUrl; + this.peerClusterUrl = peerClusterUrl; + this.zkUrl = zkUrl; + this.peerZKUrl = peerZKUrl; + this.adminCRRVersion = adminCRRVersion; + } + + public HighAvailabilityPolicy getPolicy() { + return policy; + } + + public ClusterRoleRecord.ClusterRole getClusterRole() { + return clusterRole; + } + + public ClusterRoleRecord.ClusterRole getPeerClusterRole() { + return peerClusterRole; + } + + public String getClusterUrl() { + return clusterUrl; + } + + public String getPeerClusterUrl() { + return peerClusterUrl; + } + + public String getZkUrl() { + return zkUrl; + } + + public String getPeerZKUrl() { + return peerZKUrl; + } + + public long getAdminCRRVersion() { + return adminCRRVersion; + } + } + + private SystemTableHAGroupRecord getSystemTableHAGroupRecord(String haGroupName) + throws SQLException { String queryString = String.format("SELECT * FROM %s WHERE %s = '%s'", SYSTEM_HA_GROUP_NAME, HA_GROUP_NAME, haGroupName); try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection( @@ -416,68 +555,260 @@ private void initializeHAGroupStoreClientAttributes(String haGroupName) throws S Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(queryString)) { if (rs.next()) { - this.policy = HighAvailabilityPolicy.valueOf(rs.getString(POLICY)); + HighAvailabilityPolicy policy + = HighAvailabilityPolicy.valueOf(rs.getString(POLICY)); String zkUrl1 = rs.getString(ZK_URL_1); String zkUrl2 = rs.getString(ZK_URL_2); String clusterRole1 = rs.getString(CLUSTER_ROLE_1); String clusterRole2 = rs.getString(CLUSTER_ROLE_2); String clusterUrl1 = rs.getString(CLUSTER_URL_1); String clusterUrl2 = rs.getString(CLUSTER_URL_2); - this.clusterRoleRecordVersion = rs.getLong(VERSION); - Preconditions.checkNotNull(zkUrl1, "ZK_URL_1 in System Table cannot be null"); - Preconditions.checkNotNull(zkUrl2, "ZK_URL_2 in System Table cannot be null"); - String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK); - String formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK); + long adminCRRVersion = rs.getLong(VERSION); + String formattedZkUrl1 = null; + String formattedZkUrl2 = null; + if (StringUtils.isNotBlank(zkUrl1)) { + formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK); + } + if (StringUtils.isNotBlank(zkUrl2)) { + formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK); + } String formattedZkUrl = JDBCUtil.formatUrl(this.zkUrl, RegistryType.ZK); + String peerZKUrl; + ClusterRoleRecord.ClusterRole clusterRole; + ClusterRoleRecord.ClusterRole peerClusterRole; + String clusterUrl; + String peerClusterUrl; + if (StringUtils.equals(formattedZkUrl1, formattedZkUrl)) { - this.peerZKUrl = formattedZkUrl2; - this.clusterRole = ClusterRoleRecord.ClusterRole.from( + peerZKUrl = formattedZkUrl2; + clusterRole = ClusterRoleRecord.ClusterRole.from( clusterRole1.getBytes(StandardCharsets.UTF_8)); - this.peerClusterRole = ClusterRoleRecord.ClusterRole.from( + peerClusterRole = ClusterRoleRecord.ClusterRole.from( clusterRole2.getBytes(StandardCharsets.UTF_8)); - this.clusterUrl = clusterUrl1; - this.peerClusterUrl = clusterUrl2; + clusterUrl = clusterUrl1; + peerClusterUrl = clusterUrl2; } else if (StringUtils.equals(formattedZkUrl2, formattedZkUrl)) { - this.peerZKUrl = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK); - this.clusterRole = ClusterRoleRecord.ClusterRole.from( + peerZKUrl = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK); + clusterRole = ClusterRoleRecord.ClusterRole.from( clusterRole2.getBytes(StandardCharsets.UTF_8)); - this.peerClusterRole = ClusterRoleRecord.ClusterRole.from( + peerClusterRole = ClusterRoleRecord.ClusterRole.from( clusterRole1.getBytes(StandardCharsets.UTF_8)); - this.clusterUrl = clusterUrl2; - this.peerClusterUrl = clusterUrl1; + clusterUrl = clusterUrl2; + peerClusterUrl = clusterUrl1; + } else { + throw new SQLException("Current zkUrl does not match" + + "any zkUrl in System Table for HA group: " + haGroupName); } + + Preconditions.checkNotNull(clusterRole, + "Cluster role in System Table cannot be null"); + Preconditions.checkNotNull(peerClusterRole, + "Peer cluster role in System Table cannot be null"); + Preconditions.checkNotNull(clusterUrl, + "Cluster URL in System Table cannot be null"); + Preconditions.checkNotNull(peerZKUrl, + "Peer ZK URL in System Table cannot be null"); + Preconditions.checkNotNull(peerClusterUrl, + "Peer Cluster URL in System Table cannot be null"); + + return new SystemTableHAGroupRecord(policy, clusterRole, peerClusterRole, + clusterUrl, peerClusterUrl, formattedZkUrl, peerZKUrl, adminCRRVersion); } else { throw new SQLException("HAGroupStoreRecord not found for HA group name: " + haGroupName + " in System Table " + SYSTEM_HA_GROUP_NAME); } } - Preconditions.checkNotNull(this.clusterRole, - "Cluster role in System Table cannot be null"); - Preconditions.checkNotNull(this.peerClusterRole, - "Peer cluster role in System Table cannot be null"); - Preconditions.checkNotNull(this.clusterUrl, - "Cluster URL in System Table cannot be null"); - Preconditions.checkNotNull(this.peerZKUrl, - "Peer ZK URL in System Table cannot be null"); - Preconditions.checkNotNull(this.peerClusterUrl, - "Peer Cluster URL in System Table cannot be null"); - Preconditions.checkNotNull(this.clusterRoleRecordVersion, - "Cluster role record version in System Table cannot be null"); } - private void maybeInitializePeerPathChildrenCache() { - if (StringUtils.isNotBlank(this.peerZKUrl)) { + // Update the system table on best effort basis for HA group + // In case of failure, we will log the error and continue. + private void updateSystemTableHAGroupRecordSilently(String haGroupName, + SystemTableHAGroupRecord record) + throws SQLException { + StringBuilder updateQuery = new StringBuilder("UPSERT INTO " + SYSTEM_HA_GROUP_NAME + " ("); + StringBuilder valuesClause = new StringBuilder(" VALUES ("); + List parameters = new ArrayList<>(); + + // Always include HA_GROUP_NAME as it's the key + updateQuery.append(HA_GROUP_NAME); + valuesClause.append("?"); + parameters.add(haGroupName); + + // Update non-null fields only. + if (record.getPolicy() != null) { + updateQuery.append(", ").append(POLICY); + valuesClause.append(", ?"); + parameters.add(record.getPolicy().toString()); + } + + if (record.getClusterRole() != null) { + updateQuery.append(", ").append(CLUSTER_ROLE_1); + valuesClause.append(", ?"); + parameters.add(record.getClusterRole().name()); + } + + if (record.getPeerClusterRole() != null) { + updateQuery.append(", ").append(CLUSTER_ROLE_2); + valuesClause.append(", ?"); + parameters.add(record.getPeerClusterRole().name()); + } + + if (record.getClusterUrl() != null) { + updateQuery.append(", ").append(CLUSTER_URL_1); + valuesClause.append(", ?"); + parameters.add(record.getClusterUrl()); + } + + if (record.getPeerClusterUrl() != null) { + updateQuery.append(", ").append(CLUSTER_URL_2); + valuesClause.append(", ?"); + parameters.add(record.getPeerClusterUrl()); + } + + if (record.getZkUrl() != null) { + updateQuery.append(", ").append(ZK_URL_1); + valuesClause.append(", ?"); + parameters.add(record.getZkUrl()); + } + + if (record.getPeerZKUrl() != null) { + updateQuery.append(", ").append(ZK_URL_2); + valuesClause.append(", ?"); + parameters.add(record.getPeerZKUrl()); + } + + if (record.getAdminCRRVersion() > 0) { + updateQuery.append(", ").append(VERSION); + valuesClause.append(", ?"); + parameters.add(record.getAdminCRRVersion()); + } + + updateQuery.append(")").append(valuesClause).append(")"); + + try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection( + JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl); + PreparedStatement pstmt = conn.prepareStatement(updateQuery.toString())) { + + for (int i = 0; i < parameters.size(); i++) { + pstmt.setObject(i + 1, parameters.get(i)); + } + + pstmt.executeUpdate(); + conn.commit(); + } catch (Exception e) { + LOGGER.error("Failed to update system table on best" + + "effort basis for HA group {}, error: {}", haGroupName, e); + } + } + + /** + * Starts the periodic sync job that syncs ZooKeeper data (source of truth) to system table. + * The job runs at configurable intervals with a random jitter for the initial delay. + */ + private void startPeriodicSyncJob() { + if (syncExecutor == null) { + syncExecutor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "HAGroupStoreClient-SyncJob-" + haGroupName); + t.setDaemon(true); + return t; + }); + + // Get sync interval from configuration (in seconds) + long syncIntervalSeconds = conf.getLong(HA_GROUP_STORE_SYNC_INTERVAL_SECONDS, + DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS); + + // Add jitter to initial delay + long jitterSeconds + = ThreadLocalRandom.current().nextLong(0, SYNC_JOB_MAX_JITTER_SECONDS + 1); + + LOGGER.info("Starting periodic sync job for HA group {} " + + "with initial delay of {} seconds, " + + "then every {} seconds", + haGroupName, + jitterSeconds, + syncIntervalSeconds); + + syncExecutor.scheduleAtFixedRate( + this::syncZKToSystemTable, + jitterSeconds, + syncIntervalSeconds, + TimeUnit.SECONDS + ); + } + } + + /** + * Syncs data from ZooKeeper (source of truth) to the system table. + * This method is called periodically to ensure consistency. + */ + private void syncZKToSystemTable() { + if (!isHealthy) { + LOGGER.debug("HAGroupStoreClient is not healthy," + + "skipping sync for HA group {}", haGroupName); + return; + } + + LOGGER.debug("Starting periodic sync from ZK to" + + "system table for HA group {}", haGroupName); + // Get current data from ZooKeeper + try { + HAGroupStoreRecord zkRecord = phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getLeft(); + if (zkRecord == null) { + LOGGER.warn("No ZK record found for HA group {}, skipping sync", haGroupName); + return; + } + // Get peer record for complete information + HAGroupStoreRecord peerZkRecord = getHAGroupStoreRecordFromPeer(); + ClusterRoleRecord.ClusterRole peerClusterRole = peerZkRecord != null + ? peerZkRecord.getClusterRole() + : ClusterRoleRecord.ClusterRole.UNKNOWN; + // Create SystemTableHAGroupRecord from ZK data + SystemTableHAGroupRecord systemTableRecord = new SystemTableHAGroupRecord( + HighAvailabilityPolicy.valueOf(zkRecord.getPolicy()), + zkRecord.getClusterRole(), + peerClusterRole, + zkRecord.getClusterUrl(), + zkRecord.getPeerClusterUrl(), + this.zkUrl, + zkRecord.getPeerZKUrl(), + zkRecord.getAdminCRRVersion() + ); + // Update system table with ZK data + updateSystemTableHAGroupRecordSilently(haGroupName, systemTableRecord); + LOGGER.info("Successfully synced ZK data to system table for HA group {}", haGroupName); + } catch (IOException | SQLException e) { + long syncIntervalSeconds = conf.getLong(HA_GROUP_STORE_SYNC_INTERVAL_SECONDS, + DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS); + LOGGER.error("Failed to sync ZK data to system " + + "table for HA group on best effort basis {}," + + "retrying in {} seconds", + haGroupName, syncIntervalSeconds); + } + } + + private void maybeInitializePeerPathChildrenCache() throws IOException { + // There is an edge case when the cache is not initialized yet, but we get CHILD_ADDED event + // so we need to get the record from ZK. + HAGroupStoreRecord currentHAGroupStoreRecord + = phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getLeft(); + if (currentHAGroupStoreRecord == null) { + LOGGER.error("Current HAGroupStoreRecord is null," + + "skipping peer path children cache initialization"); + return; + } + String peerZKUrl = currentHAGroupStoreRecord.getPeerZKUrl(); + if (StringUtils.isNotBlank(peerZKUrl)) { try { // Setup peer connection if needed (first time or ZK Url changed) if (peerPathChildrenCache == null || peerPhoenixHaAdmin != null - && !StringUtils.equals(this.peerZKUrl, peerPhoenixHaAdmin.getZkUrl())) { + && !StringUtils.equals(peerZKUrl, peerPhoenixHaAdmin.getZkUrl())) { // Clean up existing peer connection if it exists closePeerConnection(); // Setup new peer connection this.peerPhoenixHaAdmin - = new PhoenixHAAdmin(this.peerZKUrl, conf, - ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + = new PhoenixHAAdmin(peerZKUrl, conf, + ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); // Create new PeerPathChildrenCache this.peerPathChildrenCache = initializePathChildrenCache(peerPhoenixHaAdmin, this.peerCustomPathChildrenCacheListener, ClusterType.PEER); @@ -543,6 +874,10 @@ private PathChildrenCacheListener createCacheListener(CountDownLatch latch, case CHILD_UPDATED: if (eventRecord != null) { handleStateChange(eventRecord, eventStat, cacheType); + // Reinitialize peer path children cache if peer url is added or updated. + if (cacheType == ClusterType.LOCAL) { + maybeInitializePeerPathChildrenCache(); + } } break; case CHILD_REMOVED: @@ -572,8 +907,8 @@ private PathChildrenCacheListener createCacheListener(CountDownLatch latch, } - private Pair fetchCacheRecord(PathChildrenCache cache, - ClusterType cacheType) { + private Pair fetchCacheRecordAndPopulateZKIfNeeded(PathChildrenCache cache, + ClusterType cacheType) { if (cache == null) { LOGGER.warn("{} HAGroupStoreClient cache is null, returning null", cacheType); return Pair.of(null, null); @@ -644,10 +979,30 @@ private void closePeerConnection() { } } + /** + * Shuts down the periodic sync executor gracefully. + */ + private void shutdownSyncExecutor() { + if (syncExecutor != null) { + syncExecutor.shutdown(); + try { + if (!syncExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + syncExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + syncExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + syncExecutor = null; + } + } + @Override public void close() { try { LOGGER.info("Closing HAGroupStoreClient"); + // Shutdown sync executor + shutdownSyncExecutor(); if (pathChildrenCache != null) { pathChildrenCache.close(); pathChildrenCache = null; @@ -752,7 +1107,8 @@ private void handleStateChange(HAGroupStoreRecord newRecord, if (oldState == null || !oldState.equals(newState)) { LOGGER.info("Detected state transition for HA group {} from {} to {} on {} cluster", haGroupName, oldState, newState, clusterType); - notifySubscribers(oldState, newState, newStat.getMtime(), clusterType); + notifySubscribers(oldState, newState, newStat.getMtime(), clusterType, + newRecord.getLastSyncStateTimeInMs()); } } @@ -766,7 +1122,8 @@ private void handleStateChange(HAGroupStoreRecord newRecord, private void notifySubscribers(HAGroupState fromState, HAGroupState toState, long modifiedTime, - ClusterType clusterType) { + ClusterType clusterType, + Long lastSyncStateTimeInMs) { LOGGER.debug("Notifying subscribers of state transition " + "for HA group {} from {} to {} on {} cluster", haGroupName, fromState, toState, clusterType); @@ -790,8 +1147,8 @@ private void notifySubscribers(HAGroupState fromState, for (HAGroupStateListener listener : listenersToNotify) { try { - listener.onStateChange(haGroupName, - toState, modifiedTime, clusterType); + listener.onStateChange(haGroupName, fromState, + toState, modifiedTime, clusterType, lastSyncStateTimeInMs); } catch (Exception e) { LOGGER.error("Error notifying listener of state transition " + "for HA group {} from {} to {} on {} cluster", diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java index 7045a301f32..f829354c650 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java @@ -20,15 +20,26 @@ import org.apache.hadoop.conf.Configuration; import org.apache.phoenix.exception.InvalidClusterRoleTransitionException; import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException; +import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState; +import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import static org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC; +import static org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY; +import static org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC; +import static org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE; +import static org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState.STANDBY; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl; import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED; import static org.apache.phoenix.query.QueryServicesOptions @@ -45,6 +56,87 @@ public class HAGroupStoreManager { private final boolean mutationBlockEnabled; private final String zkUrl; private final Configuration conf; + /** + * Concurrent set to track HA groups that have already had failover management set up. + * Prevents duplicate subscriptions for the same HA group. + * Thread-safe without requiring external synchronization. + */ + private final Set failoverManagedHAGroups = ConcurrentHashMap.newKeySet(); + + /** + * Functional interface for resolving target local states based on current local state + * when peer cluster transitions occur. + */ + @FunctionalInterface + private interface TargetStateResolver { + HAGroupStoreRecord.HAGroupState determineTarget( + HAGroupStoreRecord.HAGroupState currentLocalState); + } + + /** + * Static mapping of peer state transitions to local target state resolvers. + * Defines all supported peer-to-local state transitions for failover management. + */ + private static final Map + PEER_STATE_TRANSITIONS = createPeerStateTransitions(); + + /** + * Static mapping of local state transitions to local target state resolvers. + * Defines all supported local-to-local state transitions for failover management. + */ + private static final Map + LOCAL_STATE_TRANSITIONS = createLocalStateTransitions(); + + private static Map + createPeerStateTransitions() { + Map transitions = new HashMap<>(); + + // Simple transition (no condition check) + transitions.put(ACTIVE_IN_SYNC_TO_STANDBY, currentLocal + -> STANDBY_TO_ACTIVE); + + // Conditional transitions with state validation + transitions.put(ACTIVE_IN_SYNC, currentLocal -> { + if (currentLocal == ACTIVE_IN_SYNC_TO_STANDBY) { + return HAGroupStoreRecord.HAGroupState.STANDBY; + } + if (currentLocal == HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY) { + return HAGroupStoreRecord.HAGroupState.STANDBY; + } + return null; // No transition + }); + + transitions.put(ACTIVE_NOT_IN_SYNC, currentLocal -> { + if (currentLocal == ACTIVE_IN_SYNC_TO_STANDBY) { + return HAGroupStoreRecord.HAGroupState.STANDBY; + } + if (currentLocal == HAGroupStoreRecord.HAGroupState.STANDBY) { + return HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY; + } + return null; // No transition + }); + + transitions.put(HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY, currentLocal -> + currentLocal == ACTIVE_IN_SYNC_TO_STANDBY + ? HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_IN_SYNC + : null); + + return transitions; + } + + private static Map + createLocalStateTransitions() { + Map transitions = new HashMap<>(); + // Local abort transitions - these are simple transitions (no condition check) + transitions.put(HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY, + currentLocal -> STANDBY); + transitions.put(HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_IN_SYNC, + currentLocal -> ACTIVE_IN_SYNC); + transitions.put(HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_NOT_IN_SYNC, + currentLocal -> ACTIVE_NOT_IN_SYNC); + return transitions; + } + /** * Creates/gets an instance of HAGroupStoreManager. @@ -63,11 +155,13 @@ public static HAGroupStoreManager getInstance(final Configuration conf) { return haGroupStoreManagerInstance; } - private HAGroupStoreManager(final Configuration conf) { + @VisibleForTesting + HAGroupStoreManager(final Configuration conf) { this.mutationBlockEnabled = conf.getBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED); this.zkUrl = getLocalZkUrl(conf); this.conf = conf; + this.failoverManagedHAGroups.clear(); } /** @@ -89,7 +183,8 @@ public List getHAGroupNames() throws SQLException { */ public boolean isMutationBlocked(String haGroupName) throws IOException { if (mutationBlockEnabled) { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); HAGroupStoreRecord recordWithMetadata = haGroupStoreClient.getHAGroupStoreRecord(); return recordWithMetadata != null @@ -103,16 +198,14 @@ public boolean isMutationBlocked(String haGroupName) throws IOException { /** * Force rebuilds the HAGroupStoreClient instance for all HA groups. * If any HAGroupStoreClient instance is not created, it will be created. - * @param broadcastUpdate if true, the update will be broadcasted to all - * regionserver endpoints. * @throws Exception in case of an error with dependencies or table. */ - public void invalidateHAGroupStoreClient(boolean broadcastUpdate) throws Exception { + public void invalidateHAGroupStoreClient() throws Exception { List haGroupNames = HAGroupStoreClient.getHAGroupNames(this.zkUrl); List failedHAGroupNames = new ArrayList<>(); for (String haGroupName : haGroupNames) { try { - invalidateHAGroupStoreClient(haGroupName, broadcastUpdate); + invalidateHAGroupStoreClient(haGroupName); } catch (Exception e) { failedHAGroupNames.add(haGroupName); LOGGER.error("Failed to invalidate HAGroupStoreClient for " + haGroupName, e); @@ -130,13 +223,11 @@ public void invalidateHAGroupStoreClient(boolean broadcastUpdate) throws Excepti * Force rebuilds the HAGroupStoreClient for a specific HA group. * * @param haGroupName name of the HA group, null for default HA group and tracks all HA groups. - * @param broadcastUpdate if true, the update will be broadcasted to all - * regionserver endpoints. * @throws Exception in case of an error with dependencies or table. */ - public void invalidateHAGroupStoreClient(final String haGroupName, - boolean broadcastUpdate) throws Exception { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + public void invalidateHAGroupStoreClient(final String haGroupName) throws Exception { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); haGroupStoreClient.rebuild(); } @@ -150,7 +241,8 @@ public void invalidateHAGroupStoreClient(final String haGroupName, */ public Optional getHAGroupStoreRecord(final String haGroupName) throws IOException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); return Optional.ofNullable(haGroupStoreClient.getHAGroupStoreRecord()); } @@ -158,13 +250,14 @@ public Optional getHAGroupStoreRecord(final String haGroupNa * Returns the HAGroupStoreRecord for a specific HA group from peer cluster. * * @param haGroupName name of the HA group - * @return Optional HAGroupStoreRecord for the HA group from peer cluster can be empty if + * @return Optional HAGroupStoreRecord for the HA group from peer cluster can be empty if * the HA group is not found or peer cluster is not available. * @throws IOException when HAGroupStoreClient is not healthy. */ public Optional getPeerHAGroupStoreRecord(final String haGroupName) throws IOException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); return Optional.ofNullable(haGroupStoreClient.getHAGroupStoreRecordFromPeer()); } @@ -176,8 +269,9 @@ public Optional getPeerHAGroupStoreRecord(final String haGro */ public void setHAGroupStatusToStoreAndForward(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, - InvalidClusterRoleTransitionException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + InvalidClusterRoleTransitionException, SQLException { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); haGroupStoreClient.setHAGroupStatusIfNeeded( HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC); } @@ -190,12 +284,51 @@ public void setHAGroupStatusToStoreAndForward(final String haGroupName) */ public void setHAGroupStatusToSync(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, - InvalidClusterRoleTransitionException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + InvalidClusterRoleTransitionException, SQLException { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); haGroupStoreClient.setHAGroupStatusIfNeeded( HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); } + /** + * Sets the HAGroupStoreRecord to transition from ACTIVE_IN_SYNC to STANDBY in local cluster. + * This initiates the failover process by moving the active cluster to a transitional state. + * + * @param haGroupName name of the HA group + * @throws IOException when HAGroupStoreClient is not healthy. + * @throws StaleHAGroupStoreRecordVersionException when the version is stale + * @throws InvalidClusterRoleTransitionException when the transition is not valid + * @throws SQLException when there is an error with the database operation + */ + public void setHAGroupStatusToActiveInSyncToStandby(final String haGroupName) + throws IOException, StaleHAGroupStoreRecordVersionException, + InvalidClusterRoleTransitionException, SQLException { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); + haGroupStoreClient.setHAGroupStatusIfNeeded( + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + } + + /** + * Sets the HAGroupStoreRecord to abort failover and return to STANDBY in local cluster. + * This aborts an ongoing failover process by moving the standby cluster to abort state. + * + * @param haGroupName name of the HA group + * @throws IOException when HAGroupStoreClient is not healthy. + * @throws StaleHAGroupStoreRecordVersionException when the version is stale + * @throws InvalidClusterRoleTransitionException when the transition is not valid + * @throws SQLException when there is an error with the database operation + */ + public void setHAGroupStatusToAbortToStandby(final String haGroupName) + throws IOException, StaleHAGroupStoreRecordVersionException, + InvalidClusterRoleTransitionException, SQLException { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); + haGroupStoreClient.setHAGroupStatusIfNeeded( + HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY); + } + /** * Sets the HAGroupStoreRecord to degrade reader functionality in local cluster. * Transitions from STANDBY to DEGRADED_STANDBY_FOR_READER or from @@ -208,8 +341,9 @@ public void setHAGroupStatusToSync(final String haGroupName) */ public void setReaderToDegraded(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, - InvalidClusterRoleTransitionException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + InvalidClusterRoleTransitionException, SQLException { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); @@ -217,16 +351,8 @@ public void setReaderToDegraded(final String haGroupName) throw new IOException("Current HAGroupStoreRecord is null for HA group: " + haGroupName); } - - HAGroupStoreRecord.HAGroupState currentState = currentRecord.getHAGroupState(); - HAGroupStoreRecord.HAGroupState targetState - = HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER; - - if (currentState == HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER) { - targetState = HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY; - } - - haGroupStoreClient.setHAGroupStatusIfNeeded(targetState); + haGroupStoreClient.setHAGroupStatusIfNeeded( + HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY); } /** @@ -241,8 +367,9 @@ public void setReaderToDegraded(final String haGroupName) */ public void setReaderToHealthy(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, - InvalidClusterRoleTransitionException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + InvalidClusterRoleTransitionException, SQLException { + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); @@ -251,14 +378,7 @@ public void setReaderToHealthy(final String haGroupName) + "for HA group: " + haGroupName); } - HAGroupStoreRecord.HAGroupState currentState = currentRecord.getHAGroupState(); - HAGroupStoreRecord.HAGroupState targetState = HAGroupStoreRecord.HAGroupState.STANDBY; - - if (currentState == HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY) { - targetState = HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER; - } - - haGroupStoreClient.setHAGroupStatusIfNeeded(targetState); + haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.STANDBY); } /** @@ -272,7 +392,8 @@ public void setReaderToHealthy(final String haGroupName) */ public ClusterRoleRecord getClusterRoleRecord(String haGroupName) throws IOException { - HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + HAGroupStoreClient haGroupStoreClient + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); return haGroupStoreClient.getClusterRoleRecord(); } @@ -289,7 +410,8 @@ public void subscribeToTargetState(String haGroupName, HAGroupStoreRecord.HAGroupState targetState, ClusterType clusterType, HAGroupStateListener listener) throws IOException { - HAGroupStoreClient client = getHAGroupStoreClient(haGroupName); + HAGroupStoreClient client + = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); client.subscribeToTargetState(targetState, clusterType, listener); LOGGER.debug("Delegated subscription to target state {} " + "for HA group {} on {} cluster to client", @@ -309,7 +431,7 @@ public void unsubscribeFromTargetState(String haGroupName, ClusterType clusterType, HAGroupStateListener listener) { try { - HAGroupStoreClient client = getHAGroupStoreClient(haGroupName); + HAGroupStoreClient client = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); client.unsubscribeFromTargetState(targetState, clusterType, listener); LOGGER.debug("Delegated unsubscription from target state {} " + "for HA group {} on {} cluster to client", @@ -337,4 +459,122 @@ private HAGroupStoreClient getHAGroupStoreClient(final String haGroupName) } return haGroupStoreClient; } + + + /** + * Helper method to get HAGroupStoreClient instance and setup failover management. + * NOTE: As soon as the HAGroupStoreClient is initialized, + * it will setup the failover management as well. + * Failover management is only set up once per HA group + * to prevent duplicate subscriptions. + * + * @param haGroupName name of the HA group + * @return HAGroupStoreClient instance for the specified HA group + * @throws IOException when HAGroupStoreClient is not initialized + */ + private synchronized HAGroupStoreClient + getHAGroupStoreClientAndSetupFailoverManagement(final String haGroupName) + throws IOException { + HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + + // Only setup failover management once per HA group using atomic add operation + if (failoverManagedHAGroups.add(haGroupName)) { + // add() returns true if the element was not already present + setupPeerFailoverManagement(haGroupName); + setupLocalFailoverManagement(haGroupName); + LOGGER.info("Failover management setup completed for HA group: {}", haGroupName); + } else { + LOGGER.debug("Failover management already configured for HA group: {}", haGroupName); + } + + return haGroupStoreClient; + } + + + // ===== Failover Management Related Methods ===== + + public void setupLocalFailoverManagement(String haGroupName) throws IOException { + HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + + // Generic subscription loop using static local transition mapping + for (Map.Entry entry + : LOCAL_STATE_TRANSITIONS.entrySet()) { + subscribeToTargetState(haGroupName, entry.getKey(), ClusterType.LOCAL, + new FailoverManagementListener(haGroupStoreClient, + entry.getValue())); + } + + LOGGER.info("Setup local failover management for HA group: {} with {} state transitions", + haGroupName, LOCAL_STATE_TRANSITIONS.size()); + } + + /** + * Listener implementation for handling peer failover management state transitions. + * Subscribes to peer state changes and triggers appropriate local state transitions. + */ + private static class FailoverManagementListener implements HAGroupStateListener { + private final HAGroupStoreClient client; + private final TargetStateResolver resolver; + + FailoverManagementListener(HAGroupStoreClient client, + TargetStateResolver resolver) { + this.client = client; + this.resolver = resolver; + } + + @Override + public void onStateChange(String haGroupName, + HAGroupState fromState, + HAGroupState toState, + long modifiedTime, + ClusterType clusterType, + Long lastSyncStateTimeInMs) { + HAGroupStoreRecord.HAGroupState targetState = null; + HAGroupStoreRecord.HAGroupState currentLocalState = null; + + try { + // Get current local state + HAGroupStoreRecord currentRecord = client.getHAGroupStoreRecord(); + if (currentRecord == null) { + LOGGER.error("Current HAGroupStoreRecord is null for HA group: {} " + + "in Failover Management, failover may be stalled", haGroupName); + return; + } + + // Resolve target state using TargetStateResolver + currentLocalState = currentRecord.getHAGroupState(); + targetState = resolver.determineTarget(currentLocalState); + + if (targetState == null) { + return; + } + + // Execute transition if valid + client.setHAGroupStatusIfNeeded(targetState); + + LOGGER.info("Failover management transition: peer {} -> {}, " + + "local {} -> {} for HA group: {}", + toState, toState, currentLocalState, targetState, haGroupName); + + } catch (Exception e) { + LOGGER.error("Failed to set HAGroupStatusIfNeeded for HA group: {} " + + "in Failover Management, event reaction/failover may be stalled", + haGroupName, e); + } + } + } + + public void setupPeerFailoverManagement(String haGroupName) throws IOException { + HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); + + // Generic subscription loop using static transition mapping + for (Map.Entry entry + : PEER_STATE_TRANSITIONS.entrySet()) { + subscribeToTargetState(haGroupName, entry.getKey(), ClusterType.PEER, + new FailoverManagementListener(haGroupStoreClient, entry.getValue())); + } + + LOGGER.info("Setup peer failover management for HA group: {} with {} state transitions", + haGroupName, PEER_STATE_TRANSITIONS.size()); + } } \ No newline at end of file diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java index 6891d93c46d..1eca48cc822 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java @@ -60,8 +60,6 @@ public enum HAGroupState { ACTIVE_IN_SYNC_TO_STANDBY, ACTIVE_WITH_OFFLINE_PEER, DEGRADED_STANDBY, - DEGRADED_STANDBY_FOR_READER, - DEGRADED_STANDBY_FOR_WRITER, OFFLINE, STANDBY, STANDBY_TO_ACTIVE, @@ -87,8 +85,6 @@ public ClusterRoleRecord.ClusterRole getClusterRole() { return ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY; case ABORT_TO_STANDBY: case DEGRADED_STANDBY: - case DEGRADED_STANDBY_FOR_READER: - case DEGRADED_STANDBY_FOR_WRITER: case STANDBY: return ClusterRoleRecord.ClusterRole.STANDBY; case STANDBY_TO_ACTIVE: @@ -113,7 +109,7 @@ public ClusterRoleRecord.ClusterRole getClusterRole() { ); STANDBY.allowedTransitions = ImmutableSet.of(STANDBY_TO_ACTIVE, - DEGRADED_STANDBY_FOR_READER, DEGRADED_STANDBY_FOR_WRITER); + DEGRADED_STANDBY); // This needs to be manually recovered by operator OFFLINE.allowedTransitions = ImmutableSet.of(); // This needs to be manually recovered by operator @@ -126,11 +122,7 @@ public ClusterRoleRecord.ClusterRole getClusterRole() { STANDBY_TO_ACTIVE.allowedTransitions = ImmutableSet.of(ABORT_TO_STANDBY, ACTIVE_IN_SYNC); DEGRADED_STANDBY.allowedTransitions - = ImmutableSet.of(DEGRADED_STANDBY_FOR_READER, DEGRADED_STANDBY_FOR_WRITER); - DEGRADED_STANDBY_FOR_WRITER.allowedTransitions = ImmutableSet.of(STANDBY, - DEGRADED_STANDBY); - DEGRADED_STANDBY_FOR_READER.allowedTransitions = ImmutableSet.of(STANDBY, - DEGRADED_STANDBY); + = ImmutableSet.of(STANDBY); ACTIVE_WITH_OFFLINE_PEER.allowedTransitions = ImmutableSet.of(ACTIVE_NOT_IN_SYNC); ABORT_TO_ACTIVE_IN_SYNC.allowedTransitions = ImmutableSet.of(ACTIVE_IN_SYNC); ABORT_TO_ACTIVE_NOT_IN_SYNC.allowedTransitions = ImmutableSet.of(ACTIVE_NOT_IN_SYNC); @@ -164,27 +156,35 @@ public static HAGroupState from(byte[] bytes) { private final String haGroupName; private final HAGroupState haGroupState; private final Long lastSyncStateTimeInMs; + private final String policy; + private final String peerZKUrl; + private final String clusterUrl; + private final String peerClusterUrl; + private final long adminCRRVersion; @JsonCreator public HAGroupStoreRecord(@JsonProperty("protocolVersion") String protocolVersion, @JsonProperty("haGroupName") String haGroupName, @JsonProperty("haGroupState") HAGroupState haGroupState, - @JsonProperty("lastSyncStateTimeInMs") Long lastSyncStateTimeInMs) { + @JsonProperty("lastSyncStateTimeInMs") Long lastSyncStateTimeInMs, + @JsonProperty("policy") String policy, + @JsonProperty("peerZKUrl") String peerZKUrl, + @JsonProperty("clusterUrl") String clusterUrl, + @JsonProperty("peerClusterUrl") String peerClusterUrl, + @JsonProperty("adminCRRVersion") + long adminCRRVersion) { Preconditions.checkNotNull(haGroupName, "HA group name cannot be null!"); Preconditions.checkNotNull(haGroupState, "HA group state cannot be null!"); this.protocolVersion = Objects.toString(protocolVersion, DEFAULT_PROTOCOL_VERSION); this.haGroupName = haGroupName; this.haGroupState = haGroupState; + this.policy = policy; this.lastSyncStateTimeInMs = lastSyncStateTimeInMs; - } - - /** - * Convenience constructor for backward compatibility without lastSyncStateTimeInMs. - */ - public HAGroupStoreRecord(String protocolVersion, - String haGroupName, HAGroupState haGroupState) { - this(protocolVersion, haGroupName, haGroupState, null); + this.peerZKUrl = peerZKUrl; + this.clusterUrl = clusterUrl; + this.peerClusterUrl = peerClusterUrl; + this.adminCRRVersion = adminCRRVersion; } public static Optional fromJson(byte[] bytes) { @@ -209,7 +209,12 @@ public boolean hasSameInfo(HAGroupStoreRecord other) { return haGroupName.equals(other.haGroupName) && haGroupState.equals(other.haGroupState) && protocolVersion.equals(other.protocolVersion) - && Objects.equals(lastSyncStateTimeInMs, other.lastSyncStateTimeInMs); + && Objects.equals(lastSyncStateTimeInMs, other.lastSyncStateTimeInMs) + && Objects.equals(policy, other.policy) + && Objects.equals(peerZKUrl, other.peerZKUrl) + && Objects.equals(clusterUrl, other.clusterUrl) + && Objects.equals(peerClusterUrl, other.peerClusterUrl) + && adminCRRVersion == other.adminCRRVersion; } public String getProtocolVersion() { @@ -229,6 +234,26 @@ public Long getLastSyncStateTimeInMs() { return lastSyncStateTimeInMs; } + public String getPeerZKUrl() { + return peerZKUrl; + } + + public String getPolicy() { + return policy; + } + + public String getClusterUrl() { + return clusterUrl; + } + + public String getPeerClusterUrl() { + return peerClusterUrl; + } + + public long getAdminCRRVersion() { + return adminCRRVersion; + } + @JsonIgnore public ClusterRoleRecord.ClusterRole getClusterRole() { return haGroupState.getClusterRole(); @@ -241,6 +266,11 @@ public int hashCode() { .append(haGroupName) .append(haGroupState) .append(lastSyncStateTimeInMs) + .append(policy) + .append(peerZKUrl) + .append(clusterUrl) + .append(peerClusterUrl) + .append(adminCRRVersion) .hashCode(); } @@ -259,6 +289,11 @@ public boolean equals(Object other) { .append(haGroupName, otherRecord.haGroupName) .append(haGroupState, otherRecord.haGroupState) .append(lastSyncStateTimeInMs, otherRecord.lastSyncStateTimeInMs) + .append(policy, otherRecord.policy) + .append(peerZKUrl, otherRecord.peerZKUrl) + .append(clusterUrl, otherRecord.clusterUrl) + .append(peerClusterUrl, otherRecord.peerClusterUrl) + .append(adminCRRVersion, otherRecord.adminCRRVersion) .isEquals(); } } @@ -270,6 +305,11 @@ public String toString() { + ", haGroupName='" + haGroupName + '\'' + ", haGroupState=" + haGroupState + ", lastSyncStateTimeInMs=" + lastSyncStateTimeInMs + + ", policy='" + policy + '\'' + + ", peerZKUrl='" + peerZKUrl + '\'' + + ", clusterUrl='" + clusterUrl + '\'' + + ", peerClusterUrl='" + peerClusterUrl + '\'' + + ", adminCRRVersion=" + adminCRRVersion + '}'; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java index 0ba347918ce..93aae479145 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -107,10 +107,10 @@ public interface QueryServices extends SQLCloseable { public static final String MUTATE_BATCH_SIZE_BYTES_ATTRIB = "phoenix.mutate.batchSizeBytes"; public static final String MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB = "phoenix.coprocessor.maxServerCacheTimeToLiveMs"; public static final String MAX_SERVER_CACHE_PERSISTENCE_TIME_TO_LIVE_MS_ATTRIB = "phoenix.coprocessor.maxServerCachePersistenceTimeToLiveMs"; - + @Deprecated // Use FORCE_ROW_KEY_ORDER instead. public static final String ROW_KEY_ORDER_SALTED_TABLE_ATTRIB = "phoenix.query.rowKeyOrderSaltedTable"; - + public static final String USE_INDEXES_ATTRIB = "phoenix.query.useIndexes"; @Deprecated // use the IMMUTABLE keyword while creating the table public static final String IMMUTABLE_ROWS_ATTRIB = "phoenix.mutate.immutableRows"; @@ -161,7 +161,7 @@ public interface QueryServices extends SQLCloseable { "phoenix.index.failure.handling.rebuild.interval"; public static final String INDEX_REBUILD_TASK_INITIAL_DELAY = "phoenix.index.rebuild.task.initial.delay"; public static final String START_TRUNCATE_TASK_DELAY = "phoenix.start.truncate.task.delay"; - + public static final String INDEX_FAILURE_HANDLING_REBUILD_NUMBER_OF_BATCHES_PER_TABLE = "phoenix.index.rebuild.batch.perTable"; // If index disable timestamp is older than this threshold, then index rebuild task won't attempt to rebuild it public static final String INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD = "phoenix.index.rebuild.disabletimestamp.threshold"; @@ -215,7 +215,7 @@ public interface QueryServices extends SQLCloseable { public static final String STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB = "phoenix.stats.guidepost.width"; public static final String STATS_GUIDEPOST_PER_REGION_ATTRIB = "phoenix.stats.guidepost.per.region"; public static final String STATS_USE_CURRENT_TIME_ATTRIB = "phoenix.stats.useCurrentTime"; - + public static final String RUN_UPDATE_STATS_ASYNC = "phoenix.update.stats.command.async"; public static final String STATS_SERVER_POOL_SIZE = "phoenix.stats.pool.size"; public static final String COMMIT_STATS_ASYNC = "phoenix.stats.commit.async"; @@ -244,7 +244,7 @@ public interface QueryServices extends SQLCloseable { // Tag Name to determine the Phoenix Client Type public static final String CLIENT_METRICS_TAG = "phoenix.client.metrics.tag"; - + // Transaction related configs public static final String TRANSACTIONS_ENABLED = "phoenix.transactions.enabled"; // Controls whether or not uncommitted data is automatically sent to HBase @@ -263,7 +263,7 @@ public interface QueryServices extends SQLCloseable { public static final String ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE = "phoenix.view.allowNewColumnFamily"; public static final String RETURN_SEQUENCE_VALUES_ATTRIB = "phoenix.sequence.returnValues"; public static final String EXTRA_JDBC_ARGUMENTS_ATTRIB = "phoenix.jdbc.extra.arguments"; - + public static final String MAX_VERSIONS_TRANSACTIONAL_ATTRIB = "phoenix.transactions.maxVersions"; // metadata configs @@ -281,7 +281,7 @@ public interface QueryServices extends SQLCloseable { public static final String INDEX_POPULATION_SLEEP_TIME = "phoenix.index.population.wait.time"; public static final String LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB = "phoenix.client.localIndexUpgrade"; public static final String LIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.limited.query.serial.threshold"; - + //currently BASE64 and ASCII is supported public static final String UPLOAD_BINARY_DATA_TYPE_ENCODING = "phoenix.upload.binaryDataType.encoding"; // Toggle for server-written updates to SYSTEM.CATALOG @@ -404,7 +404,7 @@ public interface QueryServices extends SQLCloseable { // Also, before 4.15 when we added a column to a base table we would have to propagate the // column metadata to all its child views. After PHOENIX-3534 we no longer propagate metadata // changes from a parent to its children (we just resolve its ancestors and include their columns) - // + // // The following config is used to continue writing the parent table column metadata while // creating a view and also prevent metadata changes to a parent table/view that needs to be // propagated to its children. This is done to allow rollback of the splittable SYSTEM.CATALOG @@ -558,6 +558,8 @@ public interface QueryServices extends SQLCloseable { public static final String SYNCHRONOUS_REPLICATION_ENABLED = "phoenix.synchronous.replication.enabled"; + // HA Group Store sync job interval in seconds + String HA_GROUP_STORE_SYNC_INTERVAL_SECONDS = "phoenix.ha.group.store.sync.interval.seconds"; /** * Get executor service used for parallel scans diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 5a88e452706..6d59e2fb758 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -123,6 +123,7 @@ import static org.apache.phoenix.query.QueryServices.PHOENIX_TTL_SERVER_SIDE_MASKING_ENABLED; import static org.apache.phoenix.query.QueryServices.MAX_IN_LIST_SKIP_SCAN_SIZE; import static org.apache.phoenix.query.QueryServices.WAL_EDIT_CODEC_ATTRIB; +import static org.apache.phoenix.query.QueryServices.HA_GROUP_STORE_SYNC_INTERVAL_SECONDS; import java.util.Map.Entry; @@ -225,7 +226,7 @@ public class QueryServicesOptions { public static final int DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES = 1000; public static final int DEFAULT_CLOCK_SKEW_INTERVAL = 2000; public static final boolean DEFAULT_INDEX_FAILURE_HANDLING_REBUILD = true; // auto rebuild on - public static final boolean DEFAULT_INDEX_FAILURE_BLOCK_WRITE = false; + public static final boolean DEFAULT_INDEX_FAILURE_BLOCK_WRITE = false; public static final boolean DEFAULT_INDEX_FAILURE_DISABLE_INDEX = true; public static final boolean DEFAULT_INDEX_FAILURE_THROW_EXCEPTION = true; public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL = 60000; // 60 secs @@ -375,7 +376,7 @@ public class QueryServicesOptions { public static final int DEFAULT_CONNECTION_ACTIVITY_LOGGING_INTERVAL_IN_MINS = 15; public static final boolean DEFAULT_STATS_COLLECTION_ENABLED = true; public static final boolean DEFAULT_USE_STATS_FOR_PARALLELIZATION = true; - + //Security defaults public static final boolean DEFAULT_PHOENIX_ACLS_ENABLED = false; @@ -472,6 +473,9 @@ public class QueryServicesOptions { public static final Boolean DEFAULT_SYNCHRONOUS_REPLICATION_ENABLED = false; + // Default HA Group Store sync job interval in seconds (15 minutes = 900 seconds) + public static final int DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS = 900; + private final Configuration config; private QueryServicesOptions(Configuration config) { @@ -586,7 +590,9 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(CQSI_THREAD_POOL_MAX_QUEUE, DEFAULT_CQSI_THREAD_POOL_MAX_QUEUE) .setIfUnset(CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT, DEFAULT_CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT) - .setIfUnset(CQSI_THREAD_POOL_METRICS_ENABLED, DEFAULT_CQSI_THREAD_POOL_METRICS_ENABLED); + .setIfUnset(CQSI_THREAD_POOL_METRICS_ENABLED, DEFAULT_CQSI_THREAD_POOL_METRICS_ENABLED) + .setIfUnset(HA_GROUP_STORE_SYNC_INTERVAL_SECONDS, + DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set diff --git a/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto b/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto index 3df27aebaff..7da42ca5c82 100644 --- a/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto +++ b/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto @@ -52,7 +52,6 @@ message InvalidateServerMetadataCacheRequest { message InvalidateHAGroupStoreClientRequest { required bytes haGroupName = 1; - required bool broadcastUpdate = 2; } message InvalidateHAGroupStoreClientResponse { diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java index ce314020eb1..e5c6fb813ba 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java @@ -130,8 +130,7 @@ public void invalidateHAGroupStoreClient(RpcController controller, = HAGroupStoreManager.getInstance(conf); if (haGroupStoreManager != null) { haGroupStoreManager - .invalidateHAGroupStoreClient(request.getHaGroupName().toStringUtf8(), - request.getBroadcastUpdate()); + .invalidateHAGroupStoreClient(request.getHaGroupName().toStringUtf8()); } else { throw new IOException("HAGroupStoreManager is null for " + "current cluster, check configuration"); diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java index 1aeb7787ceb..b4b9eac3592 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java @@ -155,6 +155,7 @@ import static org.apache.phoenix.hbase.index.util.IndexManagementUtil.rethrowIndexingException; import static org.apache.phoenix.index.PhoenixIndexBuilderHelper.ATOMIC_OP_ATTRIB; import static org.apache.phoenix.index.PhoenixIndexBuilderHelper.RETURN_RESULT; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME; import static org.apache.phoenix.query.QueryServices.SYNCHRONOUS_REPLICATION_ENABLED; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SYNCHRONOUS_REPLICATION_ENABLED; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; @@ -630,15 +631,20 @@ public void preBatchMutate(ObserverContext c, throw new IOException("HAGroupStoreManager is null " + "for current cluster, check configuration"); } - // Extract HAGroupName from the mutations - final Set haGroupNames = extractHAGroupNameAttribute(miniBatchOp); - // Check if mutation is blocked for any of the HAGroupNames - for (String haGroupName : haGroupNames) { - if (StringUtils.isNotBlank(haGroupName) - && haGroupStoreManager.isMutationBlocked(haGroupName)) { - throw new MutationBlockedIOException("Blocking Mutation as Some CRRs are in " - + "ACTIVE_TO_STANDBY state and " - + "CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true"); + String tableName + = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString(); + // We don't want to check for mutation blocking for the system ha group table + if (!tableName.equals(SYSTEM_HA_GROUP_NAME)) { + // Extract HAGroupName from the mutations + final Set haGroupNames = extractHAGroupNameAttribute(miniBatchOp); + // Check if mutation is blocked for any of the HAGroupNames + for (String haGroupName : haGroupNames) { + if (StringUtils.isNotBlank(haGroupName) + && haGroupStoreManager.isMutationBlocked(haGroupName)) { + throw new MutationBlockedIOException("Blocking Mutation as Some CRRs are in " + + "ACTIVE_TO_STANDBY state and " + + "CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true"); + } } } preBatchMutateWithExceptions(c, miniBatchOp); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java similarity index 85% rename from phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java rename to phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java index e66da0829b4..e733d993b4e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java @@ -43,14 +43,16 @@ import java.util.Map; -import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; @Category({NeedsOwnMiniClusterTest.class }) -public class PhoenixRegionServerEndpointITWithConsistentFailover extends BaseTest { +public class PhoenixRegionServerEndpointWithConsistentFailoverIT extends BaseTest { private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 5000L; private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair(); @@ -83,30 +85,43 @@ public void testGetClusterRoleRecordAndInvalidate() throws Exception { assertNotNull(coprocessor); ServerRpcController controller = new ServerRpcController(); - try (PhoenixHAAdmin peerHAAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE)) { - HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupState.STANDBY); + try (PhoenixHAAdmin peerHAAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE)) { + HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, + HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + CLUSTERS.getZkUrl2(), CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L); peerHAAdmin.createHAGroupStoreRecordInZooKeeper(peerHAGroupStoreRecord); } Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // First getClusterRoleRecord to check if HAGroupStoreClient is working as expected ClusterRoleRecord expectedRecord = buildExpectedClusterRoleRecord(haGroupName, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY); - executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord); + executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord, false); - // Change the role of local cluster to ACTIVE_TO_STANDBY in System Table + // Delete the HAGroupStoreRecord from ZK + try (PhoenixHAAdmin haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE)) { + haAdmin.deleteHAGroupStoreRecordInZooKeeper(haGroupName); + } + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + // Delete the row from System Table + HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl); + + // Expect exception when getting ClusterRoleRecord because the HAGroupStoreRecord is not found in ZK + controller = new ServerRpcController(); + executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord, true); + + // Update the row HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY, null); - // Cluster Role will still be same as before as cache is not invalidated yet - executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord); - // Now Invalidate the Cache + controller = new ServerRpcController(); coprocessor.invalidateHAGroupStoreClient(controller, getInvalidateHAGroupStoreClientRequest(haGroupName), null); assertFalse(controller.failed()); // Local Cluster Role will be updated to ACTIVE_TO_STANDBY as cache is invalidated + controller = new ServerRpcController(); ClusterRoleRecord expectedRecordAfterInvalidation = buildExpectedClusterRoleRecord(haGroupName, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY); - executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecordAfterInvalidation); + executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecordAfterInvalidation, false); } private ClusterRoleRecord buildExpectedClusterRoleRecord(String haGroupName, ClusterRoleRecord.ClusterRole localRole, ClusterRoleRecord.ClusterRole peerRole) { @@ -114,10 +129,10 @@ private ClusterRoleRecord buildExpectedClusterRoleRecord(String haGroupName, Clu } private void executeGetClusterRoleRecordAndVerify(PhoenixRegionServerEndpoint coprocessor, ServerRpcController controller, - String haGroupName, ClusterRoleRecord expectedRecord) { + String haGroupName, ClusterRoleRecord expectedRecord, boolean expectControllerFail) { RpcCallback rpcCallback = createValidationCallback(haGroupName, expectedRecord); coprocessor.getClusterRoleRecord(controller, getClusterRoleRecordRequest(haGroupName), rpcCallback); - assertFalse(controller.failed()); + assertEquals(expectControllerFail, controller.failed()); } private RpcCallback createValidationCallback(String haGroupName, ClusterRoleRecord expectedRecord) { @@ -151,7 +166,6 @@ private RegionServerEndpointProtos.GetClusterRoleRecordRequest getClusterRoleRec private RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest getInvalidateHAGroupStoreClientRequest(String haGroupName) { RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.Builder requestBuilder = RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.newBuilder(); - requestBuilder.setBroadcastUpdate(false); requestBuilder.setHaGroupName(ByteStringer.wrap(Bytes.toBytes(haGroupName))); return requestBuilder.build(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java index 377bdf69d75..93faee5f2b2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java @@ -17,14 +17,17 @@ */ package org.apache.phoenix.end2end.index; -import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME; import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.Map; import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; @@ -33,6 +36,7 @@ import org.apache.phoenix.execute.CommitException; import org.apache.phoenix.jdbc.ClusterRoleRecord; import org.apache.phoenix.jdbc.HAGroupStoreRecord; +import org.apache.phoenix.jdbc.HighAvailabilityPolicy; import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixHAAdmin; @@ -59,6 +63,8 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest { private PhoenixHAAdmin haAdmin; private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair(); + private String zkUrl; + private String peerZkUrl; @Rule public TestName testName = new TestName(); @@ -75,10 +81,10 @@ public static synchronized void doSetup() throws Exception { @Before public void setUp() throws Exception { - haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - String zkUrl = getLocalZkUrl(config); - String peerZkUrl = CLUSTERS.getZkUrl2(); + zkUrl = getLocalZkUrl(config); + peerZkUrl = CLUSTERS.getZkUrl2(); HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); } @@ -105,7 +111,7 @@ public void testMutationBlockedOnDataTableWithIndex() throws Exception { // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state) HAGroupStoreRecord haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, - haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZkUrl, this.zkUrl, this.peerZkUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); // Wait for the event to propagate @@ -176,7 +182,9 @@ public void testMutationBlockingTransition() throws Exception { // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state) HAGroupStoreRecord haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, - haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZkUrl, this.zkUrl, this.peerZkUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -195,7 +203,9 @@ public void testMutationBlockingTransition() throws Exception { // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state) haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, - haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZkUrl, this.zkUrl, this.peerZkUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -214,4 +224,80 @@ private boolean containsMutationBlockedException(CommitException e) { } return false; } + + @Test + public void testSystemHAGroupTableMutationsAllowedDuringActiveToStandby() throws Exception { + String dataTableName = generateUniqueName(); + String indexName = generateUniqueName(); + String haGroupName = testName.getMethodName(); + + try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) { + conn.setHAGroupName(haGroupName); + + // Create data table and index for testing regular table blocking + conn.createStatement().execute("CREATE TABLE " + dataTableName + + " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)"); + conn.createStatement().execute("CREATE INDEX " + indexName + + " ON " + dataTableName + "(name)"); + + // Initially, mutations should work on both tables - verify baseline + conn.createStatement().execute("UPSERT INTO " + dataTableName + + " VALUES ('1', 'John', 25)"); + + // Update system HA group table (should always work) + conn.createStatement().execute("UPSERT INTO " + SYSTEM_HA_GROUP_NAME + + " (HA_GROUP_NAME, POLICY, VERSION) VALUES ('" + haGroupName + "_test', 'FAILOVER', 1)"); + conn.commit(); + + // Set up HAGroupStoreRecord in ACTIVE_TO_STANDBY state (should block regular tables) + HAGroupStoreRecord haGroupStoreRecord = new HAGroupStoreRecord( + HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, + haGroupName, + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, + null, + HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZkUrl, + this.zkUrl, + this.peerZkUrl, + 0L); + haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); + + // Wait for the event to propagate + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // Test 1: Regular data table mutations should be BLOCKED + try { + conn.createStatement().execute("UPSERT INTO " + dataTableName + + " VALUES ('2', 'Jane', 30)"); + conn.commit(); + fail("Expected MutationBlockedIOException for regular table during ACTIVE_TO_STANDBY"); + } catch (CommitException e) { + assertTrue("Expected MutationBlockedIOException for regular table", + containsMutationBlockedException(e)); + } + + // Test 2: System HA Group table mutations should be ALLOWED + try { + conn.createStatement().execute("UPSERT INTO " + SYSTEM_HA_GROUP_NAME + + " (HA_GROUP_NAME, POLICY, VERSION) VALUES ('" + haGroupName + "_test2', 'FAILOVER', 2)"); + conn.commit(); + + // Verify the system table mutation succeeded + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + SYSTEM_HA_GROUP_NAME + + " WHERE HA_GROUP_NAME LIKE '" + haGroupName + "_test%'")) { + assertTrue("Should have results", rs.next()); + assertEquals("Should have 2 test records in system table", 2, rs.getInt(1)); + } + + } catch (Exception e) { + fail("System HA Group table mutations should NOT be blocked during ACTIVE_TO_STANDBY: " + e.getMessage()); + } + + // Clean up test records from system table + conn.createStatement().execute("DELETE FROM " + SYSTEM_HA_GROUP_NAME + + " WHERE HA_GROUP_NAME LIKE '" + haGroupName + "_test%'"); + conn.commit(); + } + } } \ No newline at end of file diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java index 32239e38873..e3e0b45be29 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java @@ -46,7 +46,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath; import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED; @@ -84,10 +85,10 @@ public static synchronized void doSetup() throws Exception { @Before public void before() throws Exception { - haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); zkUrl = getLocalZkUrl(config); this.peerZKUrl = CLUSTERS.getZkUrl2(); - peerHaAdmin = new PhoenixHAAdmin(peerZKUrl, config, ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + peerHaAdmin = new PhoenixHAAdmin(peerZKUrl, config, ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); // Clean up existing HAGroupStoreRecords try { @@ -122,33 +123,35 @@ public void testDifferentTargetStatesPerCluster() throws Exception { AtomicReference lastPeerClusterType = new AtomicReference<>(); // Create listeners for different target states - HAGroupStateListener localListener = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.STANDBY_TO_ACTIVE && clusterType == ClusterType.LOCAL) { + HAGroupStateListener localListener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + LOGGER.info("Local target state listener called: {} -> {} on {}", fromState, toState, clusterType); + if (fromState == ACTIVE_NOT_IN_SYNC && toState == HAGroupState.STANDBY_TO_ACTIVE && clusterType == ClusterType.LOCAL) { localNotifications.incrementAndGet(); lastLocalClusterType.set(clusterType); - LOGGER.info("Local target state listener called: {} on {}", toState, clusterType); + LOGGER.info("Local target state listener called: {} -> {} on {}", fromState, toState, clusterType); } }; - HAGroupStateListener peerListener = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.ACTIVE_NOT_IN_SYNC && clusterType == ClusterType.PEER) { + HAGroupStateListener peerListener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + LOGGER.info("Peer target state listener called: {} -> {} on {}", fromState, toState, clusterType); + if (fromState == null && toState == ACTIVE_NOT_IN_SYNC && clusterType == ClusterType.PEER) { peerNotifications.incrementAndGet(); lastPeerClusterType.set(clusterType); - LOGGER.info("Peer target state listener called: {} on {}", toState, clusterType); + LOGGER.info("Peer target state listener called: {} -> {} on {}", fromState, toState, clusterType); } }; // Subscribe to different target states on different clusters manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY_TO_ACTIVE, ClusterType.LOCAL, localListener); - manager.subscribeToTargetState(haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC, ClusterType.PEER, peerListener); + manager.subscribeToTargetState(haGroupName, ACTIVE_NOT_IN_SYNC, ClusterType.PEER, peerListener); // Trigger transition to STANDBY_TO_ACTIVE on LOCAL cluster - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE); + HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Trigger transition to STANDBY on PEER cluster - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC); + HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -169,35 +172,42 @@ public void testUnsubscribeSpecificCluster() throws Exception { AtomicInteger totalNotifications = new AtomicInteger(0); AtomicReference lastClusterType = new AtomicReference<>(); - HAGroupStateListener listener = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.STANDBY) { + AtomicReference lastFromState = new AtomicReference<>(); + + HAGroupStateListener listener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + // Check for specific transition: ACTIVE -> STANDBY on PEER cluster + if (fromState == HAGroupState.ACTIVE_IN_SYNC && toState == HAGroupState.STANDBY && clusterType == ClusterType.PEER) { totalNotifications.incrementAndGet(); lastClusterType.set(clusterType); - LOGGER.info("Listener called: {} on {}", toState, clusterType); + lastFromState.set(fromState); + LOGGER.info("Listener called for specific transition: {} -> {} on {}", fromState, toState, clusterType); } }; - // Subscribe to same target state on both clusters + // Subscribe to STANDBY target state on both clusters manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.LOCAL, listener); manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.PEER, listener); // Unsubscribe from LOCAL cluster only manager.unsubscribeFromTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.LOCAL, listener); - // Trigger transition to STANDBY on LOCAL → should NOT call listener - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY); - haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); + // First, establish ACTIVE_IN_SYNC state on PEER cluster + HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerActiveRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - assertEquals("Should receive no notifications from LOCAL cluster", 0, totalNotifications.get()); + // Should receive no notifications yet (we're looking for ACTIVE_IN_SYNC -> STANDBY transition) + assertEquals("Should receive no notifications for ACTIVE_IN_SYNC state", 0, totalNotifications.get()); - // Trigger transition to STANDBY on PEER → should call listener - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY); - peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); + // Now trigger transition from ACTIVE_IN_SYNC to STANDBY on PEER → should call listener + HAGroupStoreRecord peerStandbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerStandbyRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - assertEquals("Should receive notification only from PEER cluster", 1, totalNotifications.get()); + // Verify the specific transition was detected + assertEquals("Should receive notification for ACTIVE_IN_SYNC -> STANDBY transition", 1, totalNotifications.get()); assertEquals("Notification should be from PEER cluster", ClusterType.PEER, lastClusterType.get()); + assertEquals("FromState should be ACTIVE_IN_SYNC", HAGroupState.ACTIVE_IN_SYNC, lastFromState.get()); } @@ -214,25 +224,25 @@ public void testMultipleListenersMultipleClusters() throws Exception { AtomicInteger listener1PeerNotifications = new AtomicInteger(0); AtomicInteger listener2PeerNotifications = new AtomicInteger(0); - HAGroupStateListener listener1 = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener1 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.DEGRADED_STANDBY) { if (clusterType == ClusterType.LOCAL) { listener1LocalNotifications.incrementAndGet(); } else { listener1PeerNotifications.incrementAndGet(); } - LOGGER.info("Listener1 called: {} on {}", toState, clusterType); + LOGGER.info("Listener1 called: {} -> {} on {}", fromState, toState, clusterType); } }; - HAGroupStateListener listener2 = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener2 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.DEGRADED_STANDBY) { if (clusterType == ClusterType.LOCAL) { listener2LocalNotifications.incrementAndGet(); } else { listener2PeerNotifications.incrementAndGet(); } - LOGGER.info("Listener2 called: {} on {}", toState, clusterType); + LOGGER.info("Listener2 called: {} -> {} on {}", fromState, toState, clusterType); } }; @@ -243,12 +253,12 @@ public void testMultipleListenersMultipleClusters() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.DEGRADED_STANDBY, ClusterType.PEER, listener2); // Trigger transition to DEGRADED_STANDBY on LOCAL - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY); + HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Trigger transition to DEGRADED_STANDBY on PEER - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY); + HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -271,15 +281,15 @@ public void testSameListenerDifferentTargetStates() throws Exception { AtomicReference lastStateAClusterType = new AtomicReference<>(); AtomicReference lastStateBClusterType = new AtomicReference<>(); - HAGroupStateListener sharedListener = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY && clusterType == ClusterType.LOCAL) { + HAGroupStateListener sharedListener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + if (fromState == ACTIVE_NOT_IN_SYNC && toState == HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY && clusterType == ClusterType.LOCAL) { stateANotifications.incrementAndGet(); lastStateAClusterType.set(clusterType); - LOGGER.info("Shared listener - Target State A: {} on {}", toState, clusterType); - } else if (toState == HAGroupState.ACTIVE_IN_SYNC && clusterType == ClusterType.PEER) { + LOGGER.info("Shared listener - Target State A: {} -> {} on {}", fromState, toState, clusterType); + } else if (fromState == null && toState == HAGroupState.ACTIVE_IN_SYNC && clusterType == ClusterType.PEER) { stateBNotifications.incrementAndGet(); lastStateBClusterType.set(clusterType); - LOGGER.info("Shared listener - Target State B: {} on {}", toState, clusterType); + LOGGER.info("Shared listener - Target State B: {} -> {} on {}", fromState, toState, clusterType); } }; @@ -288,12 +298,12 @@ public void testSameListenerDifferentTargetStates() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.ACTIVE_IN_SYNC, ClusterType.PEER, sharedListener); // Trigger target state A on LOCAL - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Trigger target state B on PEER - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -302,7 +312,6 @@ public void testSameListenerDifferentTargetStates() throws Exception { assertEquals("Should receive target state B notification", 1, stateBNotifications.get()); assertEquals("Target state A should be from LOCAL cluster", ClusterType.LOCAL, lastStateAClusterType.get()); assertEquals("Target state B should be from PEER cluster", ClusterType.PEER, lastStateBClusterType.get()); - } // ========== Edge Cases & Error Handling ========== @@ -312,7 +321,7 @@ public void testSubscriptionToNonExistentHAGroup() throws Exception { String nonExistentHAGroup = "nonExistentGroup_" + testName.getMethodName(); HAGroupStoreManager manager = HAGroupStoreManager.getInstance(config); - HAGroupStateListener listener = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { // Should not be called }; @@ -336,25 +345,25 @@ public void testListenerExceptionIsolation() throws Exception { AtomicInteger goodListener2Notifications = new AtomicInteger(0); AtomicInteger badListenerCalls = new AtomicInteger(0); - HAGroupStateListener goodListener1 = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.ACTIVE_IN_SYNC) { + HAGroupStateListener goodListener1 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + if (fromState == ACTIVE_NOT_IN_SYNC && toState == HAGroupState.ACTIVE_IN_SYNC) { goodListener1Notifications.incrementAndGet(); - LOGGER.info("Good listener 1 called: {} on {}", toState, clusterType); + LOGGER.info("Good listener 1 called: {} -> {} on {}", fromState, toState, clusterType); } }; - HAGroupStateListener badListener = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.ACTIVE_IN_SYNC) { + HAGroupStateListener badListener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + if (fromState == ACTIVE_NOT_IN_SYNC && toState == HAGroupState.ACTIVE_IN_SYNC) { badListenerCalls.incrementAndGet(); - LOGGER.info("Bad listener called, about to throw exception"); + LOGGER.info("Bad listener called: {} -> {}, about to throw exception", fromState, toState); throw new RuntimeException("Test exception from bad listener"); } }; - HAGroupStateListener goodListener2 = (groupName, toState, modifiedTime, clusterType) -> { - if (toState == HAGroupState.ACTIVE_IN_SYNC) { + HAGroupStateListener goodListener2 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + if (fromState == ACTIVE_NOT_IN_SYNC && toState == HAGroupState.ACTIVE_IN_SYNC) { goodListener2Notifications.incrementAndGet(); - LOGGER.info("Good listener 2 called: {} on {}", toState, clusterType); + LOGGER.info("Good listener 2 called: {} -> {} on {}", fromState, toState, clusterType); } }; @@ -364,7 +373,7 @@ public void testListenerExceptionIsolation() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.ACTIVE_IN_SYNC, ClusterType.LOCAL, goodListener2); // Trigger transition to target state - HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -395,8 +404,8 @@ public void testConcurrentMultiClusterSubscriptions() throws Exception { try { startLatch.await(); // Wait for all threads to be ready - HAGroupStateListener listener = (groupName, toState, modifiedTime, clusterType) -> { - LOGGER.debug("Thread {} listener called: {} on {}", threadIndex, toState, clusterType); + HAGroupStateListener listener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { + LOGGER.debug("Thread {} listener called: {} -> {} on {}", threadIndex, fromState, toState, clusterType); }; // Half subscribe to LOCAL, half to PEER @@ -435,13 +444,13 @@ public void testHighFrequencyMultiClusterChanges() throws Exception { AtomicInteger localNotifications = new AtomicInteger(0); AtomicInteger peerNotifications = new AtomicInteger(0); - HAGroupStateListener listener = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (clusterType == ClusterType.LOCAL) { localNotifications.incrementAndGet(); } else { peerNotifications.incrementAndGet(); } - LOGGER.debug("High frequency listener: {} on {}", toState, clusterType); + LOGGER.debug("High frequency listener: {} -> {} on {}", fromState, toState, clusterType); }; // Subscribe to target state on both clusters @@ -450,18 +459,18 @@ public void testHighFrequencyMultiClusterChanges() throws Exception { // Rapidly alternate state changes on both clusters final int changeCount = 5; - HAGroupStoreRecord initialPeerRecord = new HAGroupStoreRecord("1.0", haGroupName,HAGroupState.DEGRADED_STANDBY_FOR_WRITER); + HAGroupStoreRecord initialPeerRecord = new HAGroupStoreRecord("1.0", haGroupName,HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(initialPeerRecord); for (int i = 0; i < changeCount; i++) { // Change local cluster HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, - (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.DEGRADED_STANDBY_FOR_READER); + (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, -1); // Change peer cluster HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, - (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.DEGRADED_STANDBY_FOR_WRITER); + (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerRecord, -1); // Small delay between changes @@ -494,37 +503,37 @@ public void testSubscriptionCleanupPerCluster() throws Exception { AtomicInteger peerStandbyNotifications = new AtomicInteger(0); // Create listeners that track which ones are called - HAGroupStateListener listener1 = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener1 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.ACTIVE_IN_SYNC && clusterType == ClusterType.LOCAL) { localActiveNotifications.incrementAndGet(); - LOGGER.info("Listener1 LOCAL ACTIVE_IN_SYNC: {}", toState); + LOGGER.info("Listener1 LOCAL ACTIVE_IN_SYNC: {} -> {}", fromState, toState); } }; - HAGroupStateListener listener2 = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener2 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.ACTIVE_IN_SYNC && clusterType == ClusterType.LOCAL) { localActiveNotifications.incrementAndGet(); - LOGGER.info("Listener2 LOCAL ACTIVE_IN_SYNC: {}", toState); + LOGGER.info("Listener2 LOCAL ACTIVE_IN_SYNC: {} -> {}", fromState, toState); } else if (toState == HAGroupState.STANDBY_TO_ACTIVE && clusterType == ClusterType.PEER) { peerActiveNotifications.incrementAndGet(); - LOGGER.info("Listener2 PEER STANDBY_TO_ACTIVE: {}", toState); + LOGGER.info("Listener2 PEER STANDBY_TO_ACTIVE: {} -> {}", fromState, toState); } }; - HAGroupStateListener listener3 = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener3 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.STANDBY_TO_ACTIVE && clusterType == ClusterType.PEER) { peerActiveNotifications.incrementAndGet(); - LOGGER.info("Listener3 PEER STANDBY_TO_ACTIVE: {}", toState); + LOGGER.info("Listener3 PEER STANDBY_TO_ACTIVE: {} -> {}", fromState, toState); } }; - HAGroupStateListener listener4 = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener listener4 = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.ACTIVE_IN_SYNC && clusterType == ClusterType.LOCAL) { localStandbyNotifications.incrementAndGet(); - LOGGER.info("Listener4 LOCAL ACTIVE_IN_SYNC: {}", toState); + LOGGER.info("Listener4 LOCAL ACTIVE_IN_SYNC: {} -> {}", fromState, toState); } else if (toState == HAGroupState.STANDBY_TO_ACTIVE && clusterType == ClusterType.PEER) { peerStandbyNotifications.incrementAndGet(); - LOGGER.info("Listener4 PEER STANDBY_TO_ACTIVE: {}", toState); + LOGGER.info("Listener4 PEER STANDBY_TO_ACTIVE: {} -> {}", fromState, toState); } }; @@ -537,7 +546,7 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY_TO_ACTIVE, ClusterType.PEER, listener4); // Test initial functionality - trigger ACTIVE_IN_SYNC on LOCAL - HAGroupStoreRecord localActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord localActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -545,7 +554,7 @@ public void testSubscriptionCleanupPerCluster() throws Exception { assertEquals("Should have 2 LOCAL ACTIVE_IN_SYNC notifications initially", 2, localActiveNotifications.get()); // Test initial functionality - trigger STANDBY_TO_ACTIVE on PEER - HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE); + HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerActiveRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -562,11 +571,11 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.unsubscribeFromTargetState(haGroupName, HAGroupState.ACTIVE_IN_SYNC, ClusterType.LOCAL, listener4); // Test after partial unsubscribe - trigger ACTIVE_IN_SYNC on LOCAL again by first changing to some other state. - HAGroupStoreRecord localActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC); + HAGroupStoreRecord localActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord2, 1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - HAGroupStoreRecord localActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord localActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord3, 2); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -574,11 +583,11 @@ public void testSubscriptionCleanupPerCluster() throws Exception { assertEquals("Should have 1 LOCAL ACTIVE_IN_SYNC notification after partial unsubscribe", 1, localActiveNotifications.get()); // Test after partial unsubscribe - trigger STANDBY_TO_ACTIVE on PEER again - HAGroupStoreRecord peerActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY); + HAGroupStoreRecord peerActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerActiveRecord2, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - HAGroupStoreRecord peerActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE); + HAGroupStoreRecord peerActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerActiveRecord3, 1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -590,16 +599,16 @@ public void testSubscriptionCleanupPerCluster() throws Exception { peerActiveNotifications.set(0); // Unsubscribe all remaining listeners - manager.unsubscribeFromTargetState(haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC, ClusterType.LOCAL, listener2); - manager.unsubscribeFromTargetState(haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC, ClusterType.PEER, listener3); + manager.unsubscribeFromTargetState(haGroupName, ACTIVE_NOT_IN_SYNC, ClusterType.LOCAL, listener2); + manager.unsubscribeFromTargetState(haGroupName, ACTIVE_NOT_IN_SYNC, ClusterType.PEER, listener3); manager.unsubscribeFromTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.PEER, listener4); // Test after complete unsubscribe - trigger ACTIVE_NOT_IN_SYNC on both clusters - HAGroupStoreRecord localActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC); + HAGroupStoreRecord localActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord4, 3); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - HAGroupStoreRecord peerActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_NOT_IN_SYNC); + HAGroupStoreRecord peerActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerActiveRecord4, 2); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -609,10 +618,10 @@ public void testSubscriptionCleanupPerCluster() throws Exception { // Test that new subscriptions still work properly AtomicInteger newSubscriptionNotifications = new AtomicInteger(0); - HAGroupStateListener newTestListener = (groupName, toState, modifiedTime, clusterType) -> { + HAGroupStateListener newTestListener = (groupName, fromState, toState, modifiedTime, clusterType, lastSyncStateTimeInMs) -> { if (toState == HAGroupState.STANDBY && clusterType == ClusterType.LOCAL) { newSubscriptionNotifications.incrementAndGet(); - LOGGER.info("New subscription triggered: {} on {} at {}", toState, clusterType, modifiedTime); + LOGGER.info("New subscription triggered: {} -> {} on {} at {}", fromState, toState, clusterType, modifiedTime); } }; @@ -620,7 +629,7 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.LOCAL, newTestListener); // Trigger STANDBY state and verify new subscription works - HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY); + HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, standbyRecord, 4); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java index 9e8087a57f7..63fe00fc608 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java @@ -25,6 +25,7 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; import org.apache.phoenix.util.HAGroupStoreTestUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.hadoop.conf.Configuration; import org.apache.zookeeper.data.Stat; import org.junit.After; import org.junit.Before; @@ -37,10 +38,10 @@ import java.io.IOException; import java.lang.reflect.Field; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -48,12 +49,13 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT; -import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath; @@ -92,8 +94,8 @@ public static synchronized void doSetup() throws Exception { @Before public void before() throws Exception { - haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); - peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); + peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName())); peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName())); zkUrl = getLocalZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration()); @@ -114,8 +116,7 @@ public void after() throws Exception { peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName())); haAdmin.close(); peerHaAdmin.close(); - HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl); - + //HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl); } @Test @@ -132,10 +133,10 @@ public void testHAGroupStoreClientWithBothNullZKUrl() throws Exception { public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidUrl() throws Exception { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); - HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC; @@ -144,43 +145,47 @@ public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidU peerPathChildrenCache.setAccessible(true); assertNotNull(peerPathChildrenCache.get(haGroupStoreClient)); - // Clean existing record - HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl); // Now update peerZKUrl to null and rebuild - HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); - try { - haGroupStoreClient.rebuild(); - fail("Should have thrown NullPointerException"); - } catch (NullPointerException npe) { - // This exception is expected here. - } + record = new HAGroupStoreRecord("v1.0", haGroupName, + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + null, this.zkUrl, null, 0L); + createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + assertNull(peerPathChildrenCache.get(haGroupStoreClient)); // Now update System table to contain valid peer ZK URL and also change local cluster role to STANDBY - HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null); - haGroupStoreClient.rebuild(); + record = new HAGroupStoreRecord("v1.0", haGroupName, + HAGroupStoreRecord.HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); assertNotNull(currentRecord); - assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, currentRecord.getHAGroupState()); + assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState()); // Check that peerPathChildrenCache is not null now in HAGroupStoreClient via reflection assertNotNull(peerPathChildrenCache.get(haGroupStoreClient)); // Now update local HAGroupStoreRecord to STANDBY to verify that HAGroupStoreClient is working as normal record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY); + HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); assertNotNull(currentRecord); - assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState()); + assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, currentRecord.getHAGroupState()); // Now update peerZKUrl to invalid but non-null url and rebuild // This URL can also be considered unreachable url due to a connectivity issue. String invalidUrl = "invalidURL"; - HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, invalidUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null); - haGroupStoreClient.rebuild(); + record = new HAGroupStoreRecord("v1.0", haGroupName, + HAGroupStoreRecord.HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + invalidUrl, this.zkUrl, invalidUrl, 0L); + createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); assertNotNull(currentRecord); assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState()); @@ -194,24 +199,18 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, ClusterRoleRecord.ClusterRole.STANDBY, invalidUrl, ClusterRoleRecord.ClusterRole.UNKNOWN, - 1); + 0); assertEquals(expected, clusterRoleRecord); // Check that peerPathChildrenCache is null now in HAGroupStoreClient via reflection assertNull(peerPathChildrenCache.get(haGroupStoreClient)); - - } @Test - public void testHAGroupStoreClientWithoutPeerZK() throws Exception { + public void testHAGroupStoreClientWithoutSystemTableRecord() throws Exception { String haGroupName = testName.getMethodName(); // Clean existing record HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, this.zkUrl); - HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); - HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); - createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); assertNull(HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl)); } @@ -222,7 +221,8 @@ public void testHAGroupStoreClientWithSingleHAGroupStoreRecord() throws Exceptio // Create and store HAGroupStoreRecord with ACTIVE state HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); @@ -230,7 +230,8 @@ public void testHAGroupStoreClientWithSingleHAGroupStoreRecord() throws Exceptio // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -240,7 +241,8 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -249,7 +251,8 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Change it again to ACTIVE_TO_STANDBY so that we can validate watcher works repeatedly record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -258,7 +261,8 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Change it back to ACTIVE to verify transition works record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -293,9 +297,11 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Setup initial HAGroupStoreRecords HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); HAGroupStoreRecord record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2); @@ -311,9 +317,11 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY for only 1 record record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2); @@ -327,9 +335,11 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2); @@ -342,9 +352,11 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Change other record to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2); @@ -362,7 +374,8 @@ public void testMultiThreadedAccessToHACache() throws Exception { // Setup initial HAGroupStoreRecord HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -387,7 +400,8 @@ public void testMultiThreadedAccessToHACache() throws Exception { // Update HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -414,7 +428,8 @@ public void testHAGroupStoreClientWithRootPathDeletion() throws Exception { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -434,7 +449,8 @@ public void testHAGroupStoreClientWithRootPathDeletion() throws Exception { assertNotNull(currentRecord.getLastSyncStateTimeInMs()); record1 = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY); + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -449,7 +465,8 @@ public void testThrowsExceptionWithZKDisconnectionAndThenConnection() throws Exc // Setup initial HAGroupStoreRecord HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -518,7 +535,8 @@ public void testSetHAGroupStatusIfNeededUpdateExistingRecord() throws Exception // Create initial record HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -543,7 +561,8 @@ public void testSetHAGroupStatusIfNeededNoUpdateWhenNotNeeded() throws Exception // Create initial record with current timestamp HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); Stat initialRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight(); int initialRecordVersion = initialRecordInZKStat.getVersion(); @@ -572,7 +591,8 @@ public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception { String haGroupName = testName.getMethodName(); // Create initial record HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -592,7 +612,8 @@ public void testSetHAGroupStatusIfNeededWithInvalidTransition() throws Exception // Create initial record with ACTIVE state HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -637,7 +658,8 @@ public void testSetHAGroupStatusIfNeededMultipleTransitions() throws Exception { // Create initial record with old timestamp HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -732,12 +754,14 @@ public void testGetClusterRoleRecordNormalCase() throws Exception { // Create HAGroupStoreRecord for local cluster HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord); // Create HAGroupStoreRecord for peer cluster HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY); + HAGroupStoreRecord.HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -750,7 +774,7 @@ public void testGetClusterRoleRecordNormalCase() throws Exception { assertNotNull(clusterRoleRecord); ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName, HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, peerZKUrl, - ClusterRoleRecord.ClusterRole.STANDBY, 1); + ClusterRoleRecord.ClusterRole.STANDBY, 0); assertEquals(expectedClusterRoleRecord, clusterRoleRecord); } @@ -763,7 +787,8 @@ public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws E // Create HAGroupStoreRecord for local cluster only (no peer record) HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord); // Explicitly ensure no peer record exists @@ -782,13 +807,14 @@ public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws E assertNotNull(clusterRoleRecord); ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName, HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, this.peerZKUrl, - ClusterRoleRecord.ClusterRole.UNKNOWN, 1); + ClusterRoleRecord.ClusterRole.UNKNOWN, 0); assertEquals(expectedClusterRoleRecord, clusterRoleRecord); } private HAGroupStoreRecord createHAGroupStoreRecord(String haGroupName) { return new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); } // Tests for getHAGroupNames static method @@ -920,5 +946,76 @@ public void testGetHAGroupNamesWithNullZkUrl() throws Exception { } } + @Test + public void testPeriodicSyncJobExecutorStartsAndSyncsData() throws Exception { + String haGroupName = testName.getMethodName(); + + // 1. Setup: Create initial system table record with default values + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); + + // 2. Create ZK record with DIFFERENT values for testable fields (skip zkUrl changes) + String updatedClusterUrl = this.zkUrl + ":updated"; + String updatedPeerClusterUrl = this.peerZKUrl + ":updated"; + HAGroupStoreRecord zkRecord = new HAGroupStoreRecord("v2.0", haGroupName, + HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, System.currentTimeMillis(), // Different state and sync time + HighAvailabilityPolicy.FAILOVER.toString(), + this.peerZKUrl, // Keep original peer ZK URL + updatedClusterUrl, // Different cluster URL + updatedPeerClusterUrl, // Different peer cluster URL + 5L); // Different version + createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, zkRecord); + + // Also create a peer ZK record with STANDBY_TO_ACTIVE role to test peer role sync + HAGroupStoreRecord peerZkRecord = new HAGroupStoreRecord("v2.0", haGroupName, + HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, null, + HighAvailabilityPolicy.FAILOVER.toString(), + updatedClusterUrl, this.peerZKUrl, updatedClusterUrl, 5L); + createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerZkRecord); + + // 3. Create HAGroupStoreClient with short sync interval for testing + Configuration testConf = new Configuration(CLUSTERS.getHBaseCluster1().getConfiguration()); + testConf.setLong("phoenix.ha.group.store.sync.interval.seconds", 15); // 15 seconds for faster testing + + try (HAGroupStoreClient haGroupStoreClient = new HAGroupStoreClient(testConf, null, null, haGroupName, zkUrl)) { + + // 3. Verify sync executor is running by checking private field via reflection + Field syncExecutorField = HAGroupStoreClient.class.getDeclaredField("syncExecutor"); + syncExecutorField.setAccessible(true); + ScheduledExecutorService syncExecutor = (ScheduledExecutorService) syncExecutorField.get(haGroupStoreClient); + assertNotNull("Sync executor should be initialized", syncExecutor); + assertFalse("Sync executor should not be shutdown", syncExecutor.isShutdown()); + + // 4. Wait for at least one sync cycle (with jitter buffer) + Thread.sleep(25000); // Wait 25 seconds (15s + 10s buffer for jitter) + + // 5. Verify that system table was updated with ZK data (ZK is source of truth) + // Check system table directly to see if all fields were synced + try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection( + JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT * FROM " + SYSTEM_HA_GROUP_NAME + + " WHERE HA_GROUP_NAME = '" + haGroupName + "'")) { + assertTrue("System table should have record", rs.next()); + + // Verify all fields were synced from ZK with the UPDATED values (except zkUrls which remain unchanged) + assertEquals("HA_GROUP_NAME should match", haGroupName, rs.getString("HA_GROUP_NAME")); + assertEquals("POLICY should be synced from ZK", "FAILOVER", rs.getString("POLICY")); + assertEquals("VERSION should be synced from ZK", 5L, rs.getLong("VERSION")); + assertEquals("ZK_URL_1 should remain unchanged", this.zkUrl, rs.getString("ZK_URL_1")); + assertEquals("ZK_URL_2 should remain unchanged", this.peerZKUrl, rs.getString("ZK_URL_2")); + assertEquals("CLUSTER_ROLE_1 should be synced", "STANDBY", rs.getString("CLUSTER_ROLE_1")); // DEGRADED_STANDBY maps to STANDBY role + assertEquals("CLUSTER_ROLE_2 should be synced", "STANDBY_TO_ACTIVE", rs.getString("CLUSTER_ROLE_2")); // Peer role from peer ZK + assertEquals("CLUSTER_URL_1 should be synced", updatedClusterUrl, rs.getString("CLUSTER_URL_1")); + assertEquals("CLUSTER_URL_2 should be synced", updatedPeerClusterUrl, rs.getString("CLUSTER_URL_2")); + + // All fields successfully verified - sync job is working correctly + } + // 6. Test cleanup - verify executor shuts down properly when we exit try-with-resources + // The close() will be called automatically, and we can verify shutdown in a separate assertion + haGroupStoreClient.close(); // Explicit close to test shutdown + assertTrue("Sync executor should be shutdown after close", syncExecutor.isShutdown()); + } + } } \ No newline at end of file diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java index c9e17294c52..2cf4946f60b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java @@ -42,7 +42,7 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT; import static org.apache.hadoop.hbase.HConstants.ZK_SESSION_TIMEOUT; -import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE; import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_SESSION_TIMEOUT_MULTIPLIER; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath; @@ -80,7 +80,7 @@ public static synchronized void doSetup() throws Exception { @Before public void before() throws Exception { - haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); zkUrl = getLocalZkUrl(config); this.peerZKUrl = CLUSTERS.getZkUrl2(); @@ -107,7 +107,9 @@ public void testMutationBlockingWithSingleHAGroup() throws Exception { // Update to ACTIVE_TO_STANDBY role (should block mutations) HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -126,7 +128,9 @@ public void testMutationBlockingWithMultipleHAGroups() throws Exception { HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null); HAGroupStoreRecord activeRecord1 = new HAGroupStoreRecord( - "1.0", haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + "1.0", haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(activeRecord1); HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, zkUrl, @@ -138,7 +142,9 @@ public void testMutationBlockingWithMultipleHAGroups() throws Exception { // Update only second group to ACTIVE_NOT_IN_SYNC_TO_STANDBY HAGroupStoreRecord transitionRecord2 = new HAGroupStoreRecord( - "1.0", haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY); + "1.0", haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName2, transitionRecord2, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -200,12 +206,14 @@ public void testGetPeerHAGroupStoreRecord() throws Exception { // Create a peer HAAdmin to create records in peer cluster PhoenixHAAdmin peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), - ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); try { // Create a HAGroupStoreRecord in the peer cluster HAGroupStoreRecord peerRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -230,7 +238,9 @@ public void testGetPeerHAGroupStoreRecord() throws Exception { // Create peer record again with different state HAGroupStoreRecord newPeerRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(newPeerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -238,7 +248,7 @@ public void testGetPeerHAGroupStoreRecord() throws Exception { // Verify the updated peer record peerRecordOpt = haGroupStoreManager.getPeerHAGroupStoreRecord(haGroupName); assertTrue(peerRecordOpt.isPresent()); - assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER, + assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, peerRecordOpt.get().getHAGroupState()); } finally { @@ -269,7 +279,9 @@ public void testInvalidateHAGroupStoreClient() throws Exception { // Create a HAGroupStoreRecord first HAGroupStoreRecord record = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -279,14 +291,14 @@ public void testInvalidateHAGroupStoreClient() throws Exception { assertTrue(recordOpt.isPresent()); // Invalidate the specific HA group client - haGroupStoreManager.invalidateHAGroupStoreClient(haGroupName, false); + haGroupStoreManager.invalidateHAGroupStoreClient(haGroupName); // Should still be able to get the record after invalidation recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); assertTrue(recordOpt.isPresent()); // Test global invalidation - haGroupStoreManager.invalidateHAGroupStoreClient(false); + haGroupStoreManager.invalidateHAGroupStoreClient(); // Should still be able to get the record after global invalidation recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); @@ -307,10 +319,12 @@ public void testMutationBlockDisabled() throws Exception { field.setAccessible(true); field.set(null, null); - HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config); + HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(conf); // Create HAGroupStoreRecord with ACTIVE_IN_SYNC_TO_STANDBY role HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(transitionRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -329,7 +343,9 @@ public void testSetHAGroupStatusToStoreAndForward() throws Exception { // Create an initial HAGroupStoreRecord with ACTIVE status HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -466,7 +482,9 @@ public void testSetReaderToDegraded() throws Exception { // Update the auto-created record to STANDBY state for testing HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); // Get the record to initialize ZNode from HAGroup so that we can artificially update it via HAAdmin Optional currentRecord = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); @@ -480,27 +498,10 @@ public void testSetReaderToDegraded() throws Exception { haGroupStoreManager.setReaderToDegraded(haGroupName); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - // Verify the status was updated to DEGRADED_STANDBY_FOR_READER + // Verify the status was updated to DEGRADED_STANDBY Optional updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); assertTrue(updatedRecordOpt.isPresent()); HAGroupStoreRecord updatedRecord = updatedRecordOpt.get(); - assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER, updatedRecord.getHAGroupState()); - - // Test transition from DEGRADED_STANDBY_FOR_WRITER to DEGRADED_STANDBY - HAGroupStoreRecord degradedWriterRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER); - - haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, degradedWriterRecord, 2); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - - // Call setReaderToDegraded again - haGroupStoreManager.setReaderToDegraded(haGroupName); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - - // Verify the status was updated to DEGRADED_STANDBY - updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); - assertTrue(updatedRecordOpt.isPresent()); - updatedRecord = updatedRecordOpt.get(); assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, updatedRecord.getHAGroupState()); } @@ -513,9 +514,11 @@ public void testSetReaderToHealthy() throws Exception { Optional currentRecord = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); assertTrue(currentRecord.isPresent()); - // Update the auto-created record to DEGRADED_STANDBY_FOR_READER state for testing + // Update the auto-created record to DEGRADED_STANDBY state for testing HAGroupStoreRecord degradedReaderRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, degradedReaderRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -529,23 +532,6 @@ public void testSetReaderToHealthy() throws Exception { assertTrue(updatedRecordOpt.isPresent()); HAGroupStoreRecord updatedRecord = updatedRecordOpt.get(); assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, updatedRecord.getHAGroupState()); - - // Test transition from DEGRADED_STANDBY to DEGRADED_STANDBY_FOR_WRITER - HAGroupStoreRecord degradedRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY); - - haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, degradedRecord, 2); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - - // Call setReaderToHealthy again - haGroupStoreManager.setReaderToHealthy(haGroupName); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - - // Verify the status was updated to DEGRADED_STANDBY_FOR_WRITER - updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName); - assertTrue(updatedRecordOpt.isPresent()); - updatedRecord = updatedRecordOpt.get(); - assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER, updatedRecord.getHAGroupState()); } @Test @@ -559,7 +545,9 @@ public void testReaderStateTransitionInvalidStates() throws Exception { // Update the auto-created record to ACTIVE_IN_SYNC state (invalid for both operations) HAGroupStoreRecord activeRecord = new HAGroupStoreRecord( - "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, activeRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -571,7 +559,7 @@ public void testReaderStateTransitionInvalidStates() throws Exception { } catch (InvalidClusterRoleTransitionException e) { // Expected behavior assertTrue("Exception should mention the invalid transition", - e.getMessage().contains("ACTIVE_IN_SYNC") && e.getMessage().contains("DEGRADED_STANDBY_FOR_READER")); + e.getMessage().contains("ACTIVE_IN_SYNC") && e.getMessage().contains("DEGRADED_STANDBY")); } // Test setReaderToHealthy with invalid state @@ -585,4 +573,307 @@ public void testReaderStateTransitionInvalidStates() throws Exception { } } + @Test + public void testE2EFailoverWithAutomaticStateTransitions() throws Exception { + String haGroupName = testName.getMethodName(); + + String zkUrl1 = getLocalZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration()); + String zkUrl2 = getLocalZkUrl(CLUSTERS.getHBaseCluster2().getConfiguration()); + + CLUSTERS.getHBaseCluster1().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); + CLUSTERS.getHBaseCluster2().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); + + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl2); + + // Create separate HAAdmin instances for both clusters using try-with-resources + // Create HAGroupStoreManager instances for both clusters using constructor + // This will automatically setup failover management and create ZNodes from system table + // Cluster1 will be initialized as ACTIVE_NOT_IN_SYNC, Cluster2 as STANDBY + HAGroupStoreManager cluster1HAManager = new HAGroupStoreManager(CLUSTERS.getHBaseCluster1().getConfiguration()); + HAGroupStoreManager cluster2HAManager = new HAGroupStoreManager(CLUSTERS.getHBaseCluster2().getConfiguration()); + + // Initialize HAGroupStoreClient. + cluster1HAManager.getHAGroupStoreRecord(haGroupName); + // Move cluster1 from ACTIVE_NOT_IN_SYNC to ACTIVE_IN_SYNC, + // we can move after DEFAULT_ZK_SESSION_TIMEOUT * ZK_SESSION_TIMEOUT_MULTIPLIER + Thread.sleep(20 * 1000); + cluster1HAManager.setHAGroupStatusToSync(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + cluster2HAManager.setReaderToHealthy(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // === INITIAL STATE VERIFICATION === + Optional cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + Optional cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be in ACTIVE_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster1Record.get().getHAGroupState()); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should be in STANDBY state", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); + + + // === STEP 1: Operator initiates failover on cluster1 (active) === + cluster1HAManager.setHAGroupStatusToActiveInSyncToStandby(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // Verify cluster1 is now in transition state + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be in ACTIVE_IN_SYNC_TO_STANDBY state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, cluster1Record.get().getHAGroupState()); + + + // === STEP 2: Verify automatic peer reaction === + // Cluster2 (standby) should automatically move to STANDBY_TO_ACTIVE + // Allow extra time for failover management to react + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); + + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should automatically transition to STANDBY_TO_ACTIVE", + HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, cluster2Record.get().getHAGroupState()); + + // === STEP 3: Complete cluster2 transition to active === + // Explicitly call setHAGroupStatusToSync on cluster2 + cluster2HAManager.setHAGroupStatusToSync(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // Verify cluster2 is now ACTIVE_IN_SYNC + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should be in ACTIVE_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster2Record.get().getHAGroupState()); + + // === STEP 4: Verify automatic cluster1-to-standby completion === + // Cluster1 (original active) should automatically move to STANDBY + // Allow extra time for failover management to react to peer ACTIVE_IN_SYNC + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); + + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should automatically transition to STANDBY", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster1Record.get().getHAGroupState()); + + // === FINAL VERIFICATION === + // Verify complete role swap has occurred + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + + assertEquals("Cluster1 should now be STANDBY", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster1Record.get().getHAGroupState()); + assertEquals("Cluster2 should now be ACTIVE_IN_SYNC", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster2Record.get().getHAGroupState()); + + + // Verify cluster role records + ClusterRoleRecord cluster1Role = cluster1HAManager.getClusterRoleRecord(haGroupName); + ClusterRoleRecord cluster2Role = cluster2HAManager.getClusterRoleRecord(haGroupName); + assertEquals("Cluster1 should now have STANDBY role", + ClusterRoleRecord.ClusterRole.STANDBY, cluster1Role.getRole(CLUSTERS.getZkUrl1())); + assertEquals("Cluster2 should now have ACTIVE role", + ClusterRoleRecord.ClusterRole.ACTIVE, cluster2Role.getRole(CLUSTERS.getZkUrl2())); + } + + @Test + public void testE2EFailoverAbortWithAutomaticStateTransitions() throws Exception { + String haGroupName = testName.getMethodName(); + + String zkUrl1 = getLocalZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration()); + String zkUrl2 = getLocalZkUrl(CLUSTERS.getHBaseCluster2().getConfiguration()); + + CLUSTERS.getHBaseCluster1().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); + CLUSTERS.getHBaseCluster2().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); + + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl2); + + // Create HAGroupStoreManager instances for both clusters using constructor + // This will automatically setup failover management and create ZNodes from system table + // Cluster1 will be initialized as ACTIVE_NOT_IN_SYNC, Cluster2 as STANDBY + HAGroupStoreManager cluster1HAManager = new HAGroupStoreManager(CLUSTERS.getHBaseCluster1().getConfiguration()); + HAGroupStoreManager cluster2HAManager = new HAGroupStoreManager(CLUSTERS.getHBaseCluster2().getConfiguration()); + + // Initialize HAGroupStoreClient. + cluster1HAManager.getHAGroupStoreRecord(haGroupName); + // Move cluster1 from ACTIVE_NOT_IN_SYNC to ACTIVE_IN_SYNC, + // we can move after DEFAULT_ZK_SESSION_TIMEOUT * ZK_SESSION_TIMEOUT_MULTIPLIER + Thread.sleep(20 * 1000); + cluster1HAManager.setHAGroupStatusToSync(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + cluster2HAManager.setReaderToHealthy(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // === INITIAL STATE VERIFICATION === + Optional cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + Optional cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be in ACTIVE_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster1Record.get().getHAGroupState()); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should be in STANDBY state", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); + + // === STEP 1: Operator initiates failover on cluster1 (active) === + cluster1HAManager.setHAGroupStatusToActiveInSyncToStandby(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // Verify cluster1 is now in transition state + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be in ACTIVE_IN_SYNC_TO_STANDBY state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, cluster1Record.get().getHAGroupState()); + + // === STEP 2: Verify automatic peer reaction === + // Cluster2 (standby) should automatically move to STANDBY_TO_ACTIVE + // Allow extra time for failover management to react + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); + + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should automatically transition to STANDBY_TO_ACTIVE", + HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, cluster2Record.get().getHAGroupState()); + + // === STEP 3: Operator decides to abort failover === + // Set cluster2 (which is in STANDBY_TO_ACTIVE) to ABORT_TO_STANDBY + cluster2HAManager.setHAGroupStatusToAbortToStandby(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // === STEP 4: Verify automatic cluster1 abort reaction === + // Cluster1 should automatically move from ACTIVE_IN_SYNC_TO_STANDBY back to ACTIVE_IN_SYNC + // Allow extra time for failover management to react to peer ABORT_TO_STANDBY + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); + + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should automatically transition back to ACTIVE_IN_SYNC", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster1Record.get().getHAGroupState()); + + // === STEP 5: Complete abort process === + // Cluster2 should automatically transition from ABORT_TO_STANDBY to STANDBY + // This should happen automatically via local failover management + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should automatically transition back to STANDBY", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); + + // === FINAL VERIFICATION === + // Verify we're back to the original state + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + + assertEquals("Cluster1 should be back to ACTIVE_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster1Record.get().getHAGroupState()); + assertEquals("Cluster2 should be back to STANDBY state", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); + + // Verify cluster role records are back to original + ClusterRoleRecord cluster1Role = cluster1HAManager.getClusterRoleRecord(haGroupName); + ClusterRoleRecord cluster2Role = cluster2HAManager.getClusterRoleRecord(haGroupName); + assertEquals("Cluster1 should have ACTIVE role", + ClusterRoleRecord.ClusterRole.ACTIVE, cluster1Role.getRole(CLUSTERS.getZkUrl1())); + assertEquals("Cluster2 should have STANDBY role", + ClusterRoleRecord.ClusterRole.STANDBY, cluster2Role.getRole(CLUSTERS.getZkUrl2())); + } + + @Test + public void testE2EStoreAndForwardWithAutomaticStateTransitions() throws Exception { + String haGroupName = testName.getMethodName(); + + String zkUrl1 = getLocalZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration()); + String zkUrl2 = getLocalZkUrl(CLUSTERS.getHBaseCluster2().getConfiguration()); + + CLUSTERS.getHBaseCluster1().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); + CLUSTERS.getHBaseCluster2().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); + + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl2); + + // Create HAGroupStoreManager instances for both clusters using constructor + // This will automatically setup failover management and create ZNodes from system table + // Cluster1 will be initialized as ACTIVE_NOT_IN_SYNC, Cluster2 as STANDBY + HAGroupStoreManager cluster1HAManager = new HAGroupStoreManager(CLUSTERS.getHBaseCluster1().getConfiguration()); + HAGroupStoreManager cluster2HAManager = new HAGroupStoreManager(CLUSTERS.getHBaseCluster2().getConfiguration()); + + // Initialize HAGroupStoreClient. + cluster1HAManager.getHAGroupStoreRecord(haGroupName); + cluster2HAManager.getHAGroupStoreRecord(haGroupName); + // Move cluster1 from ACTIVE_NOT_IN_SYNC to ACTIVE_IN_SYNC, + // we can move after DEFAULT_ZK_SESSION_TIMEOUT * ZK_SESSION_TIMEOUT_MULTIPLIER + Thread.sleep(20 * 1000); + cluster1HAManager.setHAGroupStatusToSync(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // === INITIAL STATE VERIFICATION === + Optional cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + Optional cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be in ACTIVE_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster1Record.get().getHAGroupState()); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should be in STANDBY state", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); + + // === STEP 1: Transition to store-and-forward mode === + // Move cluster1 from ACTIVE_IN_SYNC to ACTIVE_NOT_IN_SYNC (store-and-forward mode) + cluster1HAManager.setHAGroupStatusToStoreAndForward(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // Verify cluster1 is now in store-and-forward state + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be in ACTIVE_NOT_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, cluster1Record.get().getHAGroupState()); + + // === STEP 2: Verify automatic peer reaction to store-and-forward === + // Cluster2 (standby) should automatically move from STANDBY to DEGRADED_STANDBY + // Allow extra time for failover management to react + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); + + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should automatically transition to DEGRADED_STANDBY", + HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, cluster2Record.get().getHAGroupState()); + + // === STEP 3: Return to sync mode === + // Move cluster1 back from ACTIVE_NOT_IN_SYNC to ACTIVE_IN_SYNC + // Wait for the required time before transitioning back to sync + Thread.sleep(20 * 1000); + cluster1HAManager.setHAGroupStatusToSync(haGroupName); + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + + // Verify cluster1 is back in sync state + cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); + assertEquals("Cluster1 should be back in ACTIVE_IN_SYNC state", + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster1Record.get().getHAGroupState()); + + // === STEP 4: Verify automatic peer recovery === + // Cluster2 should automatically move from DEGRADED_STANDBY back to STANDBY + // Allow extra time for failover management to react to peer ACTIVE_IN_SYNC + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); + + cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); + assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); + assertEquals("Cluster2 should automatically transition back to STANDBY", + HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); + } } \ No newline at end of file diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java index 044a8a49257..113243a1e40 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java @@ -41,7 +41,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE; +import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE; import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -71,8 +71,8 @@ public static synchronized void doSetup() throws Exception { @Before public void before() throws Exception { - haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); - peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE); + haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); + peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE); cleanupTestZnodes(); } @@ -100,7 +100,9 @@ public void testCreateHAGroupStoreRecordInZooKeeper() throws Exception { HAGroupStoreRecord record = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); // Create the record in ZooKeeper @@ -121,7 +123,9 @@ public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exc HAGroupStoreRecord record1 = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); // Create the first record @@ -131,7 +135,9 @@ public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exc HAGroupStoreRecord record2 = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY + HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); // This should throw an exception due to NodeExistsException handling @@ -162,7 +168,9 @@ public void testUpdateHAGroupStoreRecordInZooKeeper() throws Exception { HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -176,7 +184,9 @@ public void testUpdateHAGroupStoreRecordInZooKeeper() throws Exception { HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY + HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion); @@ -200,7 +210,9 @@ public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exc HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -214,7 +226,9 @@ public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exc HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY + HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion); @@ -223,7 +237,9 @@ public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exc HAGroupStoreRecord anotherUpdate = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); try { @@ -244,7 +260,9 @@ public void testGetHAGroupStoreRecordInZooKeeper() throws Exception { HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -285,7 +303,9 @@ public void testCompleteWorkflowCreateUpdateGet() throws Exception { HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -304,7 +324,9 @@ public void testCompleteWorkflowCreateUpdateGet() throws Exception { HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY + HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, stat.getVersion()); @@ -331,7 +353,9 @@ public void testMultiThreadedUpdatesConcurrentVersionConflict() throws Exception HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -362,7 +386,9 @@ public void testMultiThreadedUpdatesConcurrentVersionConflict() throws Exception HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY + HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); // All threads use the same currentVersion, causing conflicts @@ -416,7 +442,9 @@ public void testMultiThreadedUpdatesWithDifferentVersions() throws Exception { HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, + null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -443,7 +471,9 @@ public void testMultiThreadedUpdatesWithDifferentVersions() throws Exception { HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord( "v1.0", haGroupName, - threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY + threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2(), 0L ); // Update with the current version diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java index 8661283c0fb..3bdd34527d7 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java @@ -61,7 +61,8 @@ public static String createJsonFileWithRecords(HAGroupStoreRecord record) @Test public void testReadWriteJsonToFile() throws IOException { HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(), - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); String fileName = createJsonFileWithRecords(record); String fileContent = FileUtils.readFileToString(new File(fileName), "UTF-8"); assertTrue(fileContent.contains(record.getHaGroupName())); @@ -77,7 +78,8 @@ public void testReadWriteJsonToFile() throws IOException { @Test public void testToAndFromJson() throws IOException { HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(), - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); byte[] bytes = HAGroupStoreRecord.toJson(record); Optional record2 = HAGroupStoreRecord.fromJson(bytes); assertTrue(record2.isPresent()); @@ -101,9 +103,11 @@ public void testFromJsonWithInvalidJson() { public void testHasSameInfo() { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); assertTrue(record1.hasSameInfo(record2)); // Same core info despite different state assertTrue(record1.hasSameInfo(record1)); // reflexive @@ -114,24 +118,28 @@ public void testHasSameInfo() { public void testHasSameInfoNegative() { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); // Different protocol version HAGroupStoreRecord recordDifferentProtocol = getHAGroupStoreRecord(haGroupName, - "2.0", HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + "2.0", HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); assertFalse(record.hasSameInfo(recordDifferentProtocol)); assertFalse(recordDifferentProtocol.hasSameInfo(record)); // Different HA group name String haGroupName2 = haGroupName + RandomStringUtils.randomAlphabetic(2); HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName2, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); assertFalse(record.hasSameInfo(record2)); assertFalse(record2.hasSameInfo(record)); // Different HA group state HAGroupStoreRecord recordDifferentState = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); assertFalse(record.hasSameInfo(recordDifferentState)); assertFalse(recordDifferentState.hasSameInfo(record)); } @@ -142,7 +150,8 @@ public void testGetters() { String protocolVersion = "1.5"; HAGroupStoreRecord.HAGroupState haGroupState = HAGroupStoreRecord.HAGroupState.STANDBY; - HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, protocolVersion, haGroupState); + HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, protocolVersion, haGroupState, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); assertEquals(haGroupName, record.getHaGroupName()); assertEquals(protocolVersion, record.getProtocolVersion()); @@ -154,11 +163,14 @@ public void testGetters() { public void testEqualsAndHashCode() { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); HAGroupStoreRecord record3 = getHAGroupStoreRecord(haGroupName, - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY); // Different state + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); // Different state // Test equals assertEquals(record1, record2); // symmetric @@ -175,7 +187,8 @@ public void testEqualsAndHashCode() { @Test public void testToString() { HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(), - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); String toString = record.toString(); // Verify all fields are present in toString @@ -187,7 +200,8 @@ public void testToString() { @Test public void testToPrettyString() { HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(), - PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); LOG.info("toString(): {}", record.toString()); LOG.info("toPrettyString:\n{}", record.toPrettyString()); assertNotEquals(record.toString(), record.toPrettyString()); @@ -196,12 +210,15 @@ public void testToPrettyString() { @Test(expected = NullPointerException.class) public void testConstructorWithNullHaGroupName() { - getHAGroupStoreRecord(null, PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + getHAGroupStoreRecord(null, PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); } @Test(expected = NullPointerException.class) public void testConstructorWithNullHAGroupState() { - getHAGroupStoreRecord(testName.getMethodName(), PROTOCOL_VERSION, null); + getHAGroupStoreRecord(testName.getMethodName(), PROTOCOL_VERSION, null, + HighAvailabilityPolicy.FAILOVER.toString(), + "peerZKUrl", "clusterUrl", "peerClusterUrl", 0L); } // Tests for HAGroupState enum @@ -226,10 +243,6 @@ public void testHAGroupStateGetClusterRole() { HAGroupStoreRecord.HAGroupState.ACTIVE_WITH_OFFLINE_PEER.getClusterRole()); assertEquals(ClusterRoleRecord.ClusterRole.STANDBY, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY.getClusterRole()); - assertEquals(ClusterRoleRecord.ClusterRole.STANDBY, - HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER.getClusterRole()); - assertEquals(ClusterRoleRecord.ClusterRole.STANDBY, - HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER.getClusterRole()); assertEquals(ClusterRoleRecord.ClusterRole.OFFLINE, HAGroupStoreRecord.HAGroupState.OFFLINE.getClusterRole()); assertEquals(ClusterRoleRecord.ClusterRole.STANDBY, @@ -263,10 +276,6 @@ public void testHAGroupStateValidTransitions() { // Test valid transitions for STANDBY assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE)); - assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY - .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER)); - assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY - .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER)); // Test valid transitions for ACTIVE_TO_STANDBY assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY @@ -328,9 +337,6 @@ public void testHAGroupStateFromBytesValidValues() { assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, HAGroupStoreRecord.HAGroupState.from("ACTIVE_NOT_IN_SYNC".getBytes())); - - assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER, - HAGroupStoreRecord.HAGroupState.from("DEGRADED_STANDBY_FOR_READER".getBytes())); } @Test @@ -353,7 +359,10 @@ public void testHAGroupStateFromBytesInvalidValues() { // Private Helper Methods private HAGroupStoreRecord getHAGroupStoreRecord(String haGroupName, String protocolVersion, - HAGroupStoreRecord.HAGroupState haGroupState) { - return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState); + HAGroupStoreRecord.HAGroupState haGroupState, + String policy, String peerZKUrl, String clusterUrl, + String peerClusterUrl, long adminCRRVersion) { + return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState, + null, policy, peerZKUrl, clusterUrl, peerClusterUrl, adminCRRVersion); } } \ No newline at end of file From 4fadd023cef7592358601f0f7f93b5bbe0f58d08 Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Wed, 15 Oct 2025 18:02:01 -0700 Subject: [PATCH 2/7] PHOENIX-7566 ZK to SystemTable Sync and Event Reactor for Failover --- .../phoenix/jdbc/HAGroupStoreManager.java | 46 ++++++++++++++++--- .../phoenix/jdbc/HAGroupStoreManagerIT.java | 4 +- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java index f829354c650..9e7cdc918d6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java @@ -287,13 +287,25 @@ public void setHAGroupStatusToSync(final String haGroupName) InvalidClusterRoleTransitionException, SQLException { HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); - haGroupStoreClient.setHAGroupStatusIfNeeded( - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); + HAGroupStoreRecord haGroupStoreRecord = haGroupStoreClient.getHAGroupStoreRecord(); + if (haGroupStoreRecord != null) { + HAGroupState targetHAGroupState = haGroupStoreRecord.getHAGroupState() + == HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY + ? ACTIVE_IN_SYNC_TO_STANDBY + : ACTIVE_IN_SYNC; + haGroupStoreClient.setHAGroupStatusIfNeeded(targetHAGroupState); + } else { + throw new IOException("Current HAGroupStoreRecord is null for HA group: " + + haGroupName); + } + } /** - * Sets the HAGroupStoreRecord to transition from ACTIVE_IN_SYNC to STANDBY in local cluster. - * This initiates the failover process by moving the active cluster to a transitional state. + * Initiates failover on the active cluster by transitioning to the appropriate TO_STANDBY state. + * Checks current state and transitions to: + * - ACTIVE_IN_SYNC_TO_STANDBY if currently ACTIVE_IN_SYNC + * - ACTIVE_NOT_IN_SYNC_TO_STANDBY if currently ACTIVE_NOT_IN_SYNC * * @param haGroupName name of the HA group * @throws IOException when HAGroupStoreClient is not healthy. @@ -301,13 +313,33 @@ public void setHAGroupStatusToSync(final String haGroupName) * @throws InvalidClusterRoleTransitionException when the transition is not valid * @throws SQLException when there is an error with the database operation */ - public void setHAGroupStatusToActiveInSyncToStandby(final String haGroupName) + public void initiateFailoverOnActiveCluster(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, InvalidClusterRoleTransitionException, SQLException { HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClientAndSetupFailoverManagement(haGroupName); - haGroupStoreClient.setHAGroupStatusIfNeeded( - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); + + // Get current state + HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); + if (currentRecord == null) { + throw new IOException("Current HAGroupStoreRecord is null for HA group: " + haGroupName); + } + + HAGroupStoreRecord.HAGroupState currentState = currentRecord.getHAGroupState(); + HAGroupStoreRecord.HAGroupState targetState; + + // Determine target state based on current state + if (currentState == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC) { + targetState = HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY; + } else if (currentState == HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC) { + targetState = HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY; + } else { + throw new InvalidClusterRoleTransitionException( + "Cannot initiate failover from state: " + currentState + + ". Cluster must be in ACTIVE_IN_SYNC or ACTIVE_NOT_IN_SYNC state."); + } + + haGroupStoreClient.setHAGroupStatusIfNeeded(targetState); } /** diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java index 2cf4946f60b..b2dd68650b0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java @@ -619,7 +619,7 @@ public void testE2EFailoverWithAutomaticStateTransitions() throws Exception { // === STEP 1: Operator initiates failover on cluster1 (active) === - cluster1HAManager.setHAGroupStatusToActiveInSyncToStandby(haGroupName); + cluster1HAManager.initiateFailoverOnActiveCluster(haGroupName); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Verify cluster1 is now in transition state @@ -727,7 +727,7 @@ public void testE2EFailoverAbortWithAutomaticStateTransitions() throws Exception HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); // === STEP 1: Operator initiates failover on cluster1 (active) === - cluster1HAManager.setHAGroupStatusToActiveInSyncToStandby(haGroupName); + cluster1HAManager.initiateFailoverOnActiveCluster(haGroupName); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Verify cluster1 is now in transition state From bf3f684976be06c78d357e90cc57e6b56fce7894 Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Thu, 16 Oct 2025 23:12:03 -0700 Subject: [PATCH 3/7] PHOENIX-7566 ZK Removing retry logic, updating e2e failover test and checking system table record before writing --- .../phoenix/jdbc/HAGroupStoreClient.java | 94 ++++++------ .../phoenix/jdbc/HAGroupStoreManager.java | 140 +++++++++++++----- .../phoenix/jdbc/HAGroupStoreClientIT.java | 34 ++++- .../phoenix/jdbc/HAGroupStoreManagerIT.java | 62 +++----- 4 files changed, 193 insertions(+), 137 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java index eeae13f311c..c94cb14452e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java @@ -339,52 +339,8 @@ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupStat currentHAGroupStoreRecord.getPeerClusterUrl(), currentHAGroupStoreRecord.getAdminCRRVersion() ); - try { - phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, + phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, newHAGroupStoreRecord, currentHAGroupStoreRecordStat.getVersion()); - } catch (StaleHAGroupStoreRecordVersionException e) { - LOGGER.debug("Failed to update HAGroupStoreRecord for HA group " - + haGroupName + " with cached stat version " - + currentHAGroupStoreRecordStat.getVersion() - + " checking if state is already updated in local cache", e); - // Check against current cached record, - // hoping that record is updated in local cache. - Pair cachedRecord - = fetchCacheRecordAndPopulateZKIfNeeded(pathChildrenCache, ClusterType.LOCAL); - currentHAGroupStoreRecord = cachedRecord.getLeft(); - Stat previousHAGroupStoreRecordStat = currentHAGroupStoreRecordStat; - currentHAGroupStoreRecordStat = cachedRecord.getRight(); - if (currentHAGroupStoreRecord != null - && currentHAGroupStoreRecord.getHAGroupState() - == haGroupState) { - LOGGER.debug("HAGroupStoreRecord for HA group {} is already updated" - + "with state {}, no need to update", - haGroupName, haGroupState); - return; - // Check if the cached version is not updated, only then check with ZK. - } else if (currentHAGroupStoreRecordStat != null - && currentHAGroupStoreRecordStat.getVersion() - == previousHAGroupStoreRecordStat.getVersion()) { - try { - // Check against record in ZK, if it is still what we don't want, - // throw an exception. - currentHAGroupStoreRecord - = phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName) - .getLeft(); - if (currentHAGroupStoreRecord != null - && currentHAGroupStoreRecord.getHAGroupState() - == haGroupState) { - LOGGER.debug("HAGroupStoreRecord for HA group {} is already " - + "updated with state {}, no need to update", - haGroupName, haGroupState); - return; - } - throw e; - } catch (StaleHAGroupStoreRecordVersionException e2) { - throw e2; - } - } - } // If cluster role is changing, if so, we update, // the system table on best effort basis. // We also have a periodic job which syncs the ZK @@ -544,6 +500,31 @@ public String getPeerZKUrl() { public long getAdminCRRVersion() { return adminCRRVersion; } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SystemTableHAGroupRecord other = (SystemTableHAGroupRecord) obj; + return Objects.equals(policy, other.policy) + && Objects.equals(clusterRole, other.clusterRole) + && Objects.equals(peerClusterRole, other.peerClusterRole) + && Objects.equals(clusterUrl, other.clusterUrl) + && Objects.equals(peerClusterUrl, other.peerClusterUrl) + && Objects.equals(zkUrl, other.zkUrl) + && Objects.equals(peerZKUrl, other.peerZKUrl) + && adminCRRVersion == other.adminCRRVersion; + } + + @Override + public int hashCode() { + return Objects.hash(policy, clusterRole, peerClusterRole, clusterUrl, + peerClusterUrl, zkUrl, peerZKUrl, adminCRRVersion); + } } private SystemTableHAGroupRecord getSystemTableHAGroupRecord(String haGroupName) @@ -763,7 +744,7 @@ private void syncZKToSystemTable() { ? peerZkRecord.getClusterRole() : ClusterRoleRecord.ClusterRole.UNKNOWN; // Create SystemTableHAGroupRecord from ZK data - SystemTableHAGroupRecord systemTableRecord = new SystemTableHAGroupRecord( + SystemTableHAGroupRecord newSystemTableRecord = new SystemTableHAGroupRecord( HighAvailabilityPolicy.valueOf(zkRecord.getPolicy()), zkRecord.getClusterRole(), peerClusterRole, @@ -773,8 +754,17 @@ private void syncZKToSystemTable() { zkRecord.getPeerZKUrl(), zkRecord.getAdminCRRVersion() ); + + // Read existing record from system table to check if update is needed + SystemTableHAGroupRecord existingSystemTableRecord = getSystemTableHAGroupRecord(haGroupName); + if (newSystemTableRecord.equals(existingSystemTableRecord)) { + LOGGER.debug("System table record is already up-to-date for HA group {}, skipping update", + haGroupName); + return; + } + // Update system table with ZK data - updateSystemTableHAGroupRecordSilently(haGroupName, systemTableRecord); + updateSystemTableHAGroupRecordSilently(haGroupName, newSystemTableRecord); LOGGER.info("Successfully synced ZK data to system table for HA group {}", haGroupName); } catch (IOException | SQLException e) { long syncIntervalSeconds = conf.getLong(HA_GROUP_STORE_SYNC_INTERVAL_SECONDS, @@ -907,8 +897,8 @@ private PathChildrenCacheListener createCacheListener(CountDownLatch latch, } - private Pair fetchCacheRecordAndPopulateZKIfNeeded(PathChildrenCache cache, - ClusterType cacheType) { + private Pair fetchCacheRecordAndPopulateZKIfNeeded( + PathChildrenCache cache, ClusterType cacheType) { if (cache == null) { LOGGER.warn("{} HAGroupStoreClient cache is null, returning null", cacheType); return Pair.of(null, null); @@ -945,7 +935,7 @@ private Pair extractRecordAndStat(PathChildrenCache ca if (childData != null) { Pair recordAndStat = extractHAGroupStoreRecordOrNull(childData); - LOGGER.info("Built {} cluster record: {}", cacheType, recordAndStat.getLeft()); + LOGGER.debug("Built {} cluster record: {}", cacheType, recordAndStat.getLeft()); return recordAndStat; } return Pair.of(null, null); @@ -1034,7 +1024,9 @@ private boolean isUpdateNeeded(HAGroupStoreRecord.HAGroupState currentHAGroupSta long waitTime = 0L; if (currentHAGroupState.isTransitionAllowed(newHAGroupState)) { if (currentHAGroupState == HAGroupState.ACTIVE_NOT_IN_SYNC - && newHAGroupState == HAGroupState.ACTIVE_IN_SYNC) { + && newHAGroupState == HAGroupState.ACTIVE_IN_SYNC + || (currentHAGroupState == HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY + && newHAGroupState == HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY)) { waitTime = waitTimeForSyncModeInMs; } } else { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java index 9e7cdc918d6..7a022f9db2f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java @@ -265,7 +265,12 @@ public Optional getPeerHAGroupStoreRecord(final String haGro * Sets the HAGroupStoreRecord to StoreAndForward mode in local cluster. * * @param haGroupName name of the HA group - * @throws IOException when HAGroupStoreClient is not healthy. + * @throws StaleHAGroupStoreRecordVersionException if the cached version is invalid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. + * @throws InvalidClusterRoleTransitionException if the transition is invalid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. */ public void setHAGroupStatusToStoreAndForward(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, @@ -281,6 +286,12 @@ public void setHAGroupStatusToStoreAndForward(final String haGroupName) * * @param haGroupName name of the HA group * @throws IOException when HAGroupStoreClient is not healthy. + * @throws StaleHAGroupStoreRecordVersionException if the cached version is invalid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. + * @throws InvalidClusterRoleTransitionException if the transition is invalid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. */ public void setHAGroupStatusToSync(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, @@ -293,12 +304,11 @@ public void setHAGroupStatusToSync(final String haGroupName) == HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY ? ACTIVE_IN_SYNC_TO_STANDBY : ACTIVE_IN_SYNC; - haGroupStoreClient.setHAGroupStatusIfNeeded(targetHAGroupState); + haGroupStoreClient.setHAGroupStatusIfNeeded(targetHAGroupState); } else { throw new IOException("Current HAGroupStoreRecord is null for HA group: " + haGroupName); } - } /** @@ -309,8 +319,12 @@ public void setHAGroupStatusToSync(final String haGroupName) * * @param haGroupName name of the HA group * @throws IOException when HAGroupStoreClient is not healthy. - * @throws StaleHAGroupStoreRecordVersionException when the version is stale - * @throws InvalidClusterRoleTransitionException when the transition is not valid + * @throws StaleHAGroupStoreRecordVersionException when the version is stale, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. + * @throws InvalidClusterRoleTransitionException when the transition is not valid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. * @throws SQLException when there is an error with the database operation */ public void initiateFailoverOnActiveCluster(final String haGroupName) @@ -348,8 +362,12 @@ public void initiateFailoverOnActiveCluster(final String haGroupName) * * @param haGroupName name of the HA group * @throws IOException when HAGroupStoreClient is not healthy. - * @throws StaleHAGroupStoreRecordVersionException when the version is stale - * @throws InvalidClusterRoleTransitionException when the transition is not valid + * @throws StaleHAGroupStoreRecordVersionException when the version is stale, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. + * @throws InvalidClusterRoleTransitionException when the transition is not valid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. * @throws SQLException when there is an error with the database operation */ public void setHAGroupStatusToAbortToStandby(final String haGroupName) @@ -368,8 +386,12 @@ public void setHAGroupStatusToAbortToStandby(final String haGroupName) * * @param haGroupName name of the HA group * @throws IOException when HAGroupStoreClient is not healthy. - * @throws InvalidClusterRoleTransitionException when the current state - * cannot transition to a degraded reader state + * @throws StaleHAGroupStoreRecordVersionException when the version is stale, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. + * @throws InvalidClusterRoleTransitionException when the transition is not valid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. */ public void setReaderToDegraded(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, @@ -394,8 +416,12 @@ public void setReaderToDegraded(final String haGroupName) * * @param haGroupName name of the HA group * @throws IOException when HAGroupStoreClient is not healthy. - * @throws InvalidClusterRoleTransitionException when the current state - * cannot transition to a healthy reader state + * @throws StaleHAGroupStoreRecordVersionException when the version is stale, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. + * @throws InvalidClusterRoleTransitionException when the transition is not valid, + * the state might have been updated by some other RS, + * check the state again and retry if the use case still needs it. */ public void setReaderToHealthy(final String haGroupName) throws IOException, StaleHAGroupStoreRecordVersionException, @@ -504,11 +530,10 @@ private HAGroupStoreClient getHAGroupStoreClient(final String haGroupName) * @return HAGroupStoreClient instance for the specified HA group * @throws IOException when HAGroupStoreClient is not initialized */ - private synchronized HAGroupStoreClient + private HAGroupStoreClient getHAGroupStoreClientAndSetupFailoverManagement(final String haGroupName) throws IOException { HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); - // Only setup failover management once per HA group using atomic add operation if (failoverManagedHAGroups.add(haGroupName)) { // add() returns true if the element was not already present @@ -563,35 +588,41 @@ public void onStateChange(String haGroupName, Long lastSyncStateTimeInMs) { HAGroupStoreRecord.HAGroupState targetState = null; HAGroupStoreRecord.HAGroupState currentLocalState = null; - - try { - // Get current local state - HAGroupStoreRecord currentRecord = client.getHAGroupStoreRecord(); - if (currentRecord == null) { - LOGGER.error("Current HAGroupStoreRecord is null for HA group: {} " - + "in Failover Management, failover may be stalled", haGroupName); + // We retry 2 times in case there is some Exception in the setHAGroupStatusIfNeeded method. + int retries = 2; + while (retries-- > 0) { + try { + // Get current local state + HAGroupStoreRecord currentRecord = client.getHAGroupStoreRecord(); + if (currentRecord == null) { + LOGGER.error("Current HAGroupStoreRecord is null for HA group: {} " + + "in Failover Management, failover may be stalled", haGroupName); + return; + } + + // Resolve target state using TargetStateResolver + currentLocalState = currentRecord.getHAGroupState(); + targetState = resolver.determineTarget(currentLocalState); + + if (targetState == null) { + return; + } + + // Execute transition if valid + client.setHAGroupStatusIfNeeded(targetState); + + LOGGER.info("Failover management transition: peer {} -> {}, " + + "local {} -> {} for HA group: {}", + toState, toState, currentLocalState, targetState, haGroupName); return; + } catch (Exception e) { + if (isStateAlreadyUpdated(client, haGroupName, targetState)) { + return; + } + LOGGER.error("Failed to set HAGroupStatusIfNeeded for HA group: {} " + + "in Failover Management, event reaction/failover may be stalled: {}", + haGroupName, e); } - - // Resolve target state using TargetStateResolver - currentLocalState = currentRecord.getHAGroupState(); - targetState = resolver.determineTarget(currentLocalState); - - if (targetState == null) { - return; - } - - // Execute transition if valid - client.setHAGroupStatusIfNeeded(targetState); - - LOGGER.info("Failover management transition: peer {} -> {}, " - + "local {} -> {} for HA group: {}", - toState, toState, currentLocalState, targetState, haGroupName); - - } catch (Exception e) { - LOGGER.error("Failed to set HAGroupStatusIfNeeded for HA group: {} " - + "in Failover Management, event reaction/failover may be stalled", - haGroupName, e); } } } @@ -609,4 +640,33 @@ public void setupPeerFailoverManagement(String haGroupName) throws IOException { LOGGER.info("Setup peer failover management for HA group: {} with {} state transitions", haGroupName, PEER_STATE_TRANSITIONS.size()); } + + /** + * Checks if the state is already updated in the local cache. + * + * @param client HAGroupStoreClient to check the state for + * @param haGroupName name of the HA group + * @param targetState the target state to check against + * @return true if the state is already updated to targetState, false otherwise + */ + private static boolean isStateAlreadyUpdated(HAGroupStoreClient client, + String haGroupName, + HAGroupState targetState) { + try { + HAGroupStoreRecord currentRecord = client.getHAGroupStoreRecord(); + if (currentRecord != null && currentRecord.getHAGroupState() == targetState) { + LOGGER.info("HAGroupStoreRecord for HA group {} is already updated" + + "with state {}, no need to update", + haGroupName, targetState); + return true; + } + } catch (IOException e) { + LOGGER.error("Failed to get HAGroupStoreRecord for HA group: {} " + + "in Failover Management, event reaction/failover may be stalled", + haGroupName, e); + } + return false; + } + + } \ No newline at end of file diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java index 63fe00fc608..dc0be9d2edb 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java @@ -569,9 +569,6 @@ public void testSetHAGroupStatusIfNeededNoUpdateWhenNotNeeded() throws Exception HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); - // Get the current record - HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord(); - // Try to set to ACTIVE_IN_SYNC immediately (should not update due to timing) haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); @@ -596,8 +593,10 @@ public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception { createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS + DEFAULT_ZK_SESSION_TIMEOUT); - + + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS + + (long) Math.ceil(DEFAULT_ZK_SESSION_TIMEOUT + * HAGroupStoreClient.ZK_SESSION_TIMEOUT_MULTIPLIER)); haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -663,7 +662,6 @@ public void testSetHAGroupStatusIfNeededMultipleTransitions() throws Exception { createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // First transition: ACTIVE -> ACTIVE_TO_STANDBY haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY); @@ -1012,7 +1010,29 @@ public void testPeriodicSyncJobExecutorStartsAndSyncsData() throws Exception { // All fields successfully verified - sync job is working correctly } - // 6. Test cleanup - verify executor shuts down properly when we exit try-with-resources + // 7. Test that no update happens when system table is already in sync with ZK + // Wait for another sync cycle to ensure the optimization is working + Thread.sleep(16000); // Wait for another sync cycle (15s + 1s buffer) + + // Verify system table still has the same data (no redundant updates) + try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection( + JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT * FROM " + SYSTEM_HA_GROUP_NAME + + " WHERE HA_GROUP_NAME = '" + haGroupName + "'")) { + assertTrue("System table should still have record", rs.next()); + + // Verify all fields remain the same (no unnecessary update occurred) + assertEquals("VERSION should remain the same", 5L, rs.getLong("VERSION")); + assertEquals("CLUSTER_ROLE_1 should remain the same", "STANDBY", rs.getString("CLUSTER_ROLE_1")); + assertEquals("CLUSTER_ROLE_2 should remain the same", "STANDBY_TO_ACTIVE", rs.getString("CLUSTER_ROLE_2")); + assertEquals("CLUSTER_URL_1 should remain the same", updatedClusterUrl, rs.getString("CLUSTER_URL_1")); + assertEquals("CLUSTER_URL_2 should remain the same", updatedPeerClusterUrl, rs.getString("CLUSTER_URL_2")); + + // This verifies that the equals() check is working and preventing redundant updates + } + + // 8. Test cleanup - verify executor shuts down properly when we exit try-with-resources // The close() will be called automatically, and we can verify shutdown in a separate assertion haGroupStoreClient.close(); // Explicit close to test shutdown assertTrue("Sync executor should be shutdown after close", syncExecutor.isShutdown()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java index b2dd68650b0..c8c34c3f1f2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.exception.InvalidClusterRoleTransitionException; import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; @@ -34,7 +35,9 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import java.io.IOException; import java.lang.reflect.Field; +import java.sql.SQLException; import java.util.List; import java.util.Map; import java.util.Optional; @@ -606,6 +609,25 @@ public void testE2EFailoverWithAutomaticStateTransitions() throws Exception { cluster2HAManager.setReaderToHealthy(haGroupName); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); + // Simulates action taken by reader to complete the replay and become new ACTIVE + HAGroupStateListener listener = (haGroupName1, + fromState, + toState, + modifiedTime, + clusterType, + lastSyncStateTimeInMs) -> { + try { + if (toState == HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE) { + cluster2HAManager.setHAGroupStatusToSync(haGroupName1); + } + } catch (Exception e) { + fail("Peer Cluster should be able to move to ACTIVE_IN_SYNC" + e.getMessage()); + } + }; + cluster2HAManager.subscribeToTargetState(haGroupName, + HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, + ClusterType.LOCAL, listener); + // === INITIAL STATE VERIFICATION === Optional cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); Optional cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); @@ -618,48 +640,10 @@ public void testE2EFailoverWithAutomaticStateTransitions() throws Exception { HAGroupStoreRecord.HAGroupState.STANDBY, cluster2Record.get().getHAGroupState()); - // === STEP 1: Operator initiates failover on cluster1 (active) === + // === Operator initiates failover on cluster1 (active) === cluster1HAManager.initiateFailoverOnActiveCluster(haGroupName); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - // Verify cluster1 is now in transition state - cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); - assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); - assertEquals("Cluster1 should be in ACTIVE_IN_SYNC_TO_STANDBY state", - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, cluster1Record.get().getHAGroupState()); - - - // === STEP 2: Verify automatic peer reaction === - // Cluster2 (standby) should automatically move to STANDBY_TO_ACTIVE - // Allow extra time for failover management to react - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); - - cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); - assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); - assertEquals("Cluster2 should automatically transition to STANDBY_TO_ACTIVE", - HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, cluster2Record.get().getHAGroupState()); - - // === STEP 3: Complete cluster2 transition to active === - // Explicitly call setHAGroupStatusToSync on cluster2 - cluster2HAManager.setHAGroupStatusToSync(haGroupName); - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - - // Verify cluster2 is now ACTIVE_IN_SYNC - cluster2Record = cluster2HAManager.getHAGroupStoreRecord(haGroupName); - assertTrue("Cluster2 record should be present", cluster2Record.isPresent()); - assertEquals("Cluster2 should be in ACTIVE_IN_SYNC state", - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, cluster2Record.get().getHAGroupState()); - - // === STEP 4: Verify automatic cluster1-to-standby completion === - // Cluster1 (original active) should automatically move to STANDBY - // Allow extra time for failover management to react to peer ACTIVE_IN_SYNC - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS * 2); - - cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); - assertTrue("Cluster1 record should be present", cluster1Record.isPresent()); - assertEquals("Cluster1 should automatically transition to STANDBY", - HAGroupStoreRecord.HAGroupState.STANDBY, cluster1Record.get().getHAGroupState()); - // === FINAL VERIFICATION === // Verify complete role swap has occurred cluster1Record = cluster1HAManager.getHAGroupStoreRecord(haGroupName); From 9be70af652cb988ec2d406e1223020f8a26cd9db Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Fri, 17 Oct 2025 11:02:46 -0700 Subject: [PATCH 4/7] PHOENIX-7566 Fixing checkstyle logic --- .../org/apache/phoenix/jdbc/HAGroupStoreManager.java | 2 +- .../org/apache/phoenix/jdbc/HAGroupStoreClientIT.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java index 7a022f9db2f..3af63cad620 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java @@ -530,7 +530,7 @@ private HAGroupStoreClient getHAGroupStoreClient(final String haGroupName) * @return HAGroupStoreClient instance for the specified HA group * @throws IOException when HAGroupStoreClient is not initialized */ - private HAGroupStoreClient + private HAGroupStoreClient getHAGroupStoreClientAndSetupFailoverManagement(final String haGroupName) throws IOException { HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java index dc0be9d2edb..ad017da2a40 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java @@ -592,10 +592,11 @@ public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception { this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); - HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); - - Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS - + (long) Math.ceil(DEFAULT_ZK_SESSION_TIMEOUT + HAGroupStoreClient haGroupStoreClient + = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); + + Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS + + (long) Math.ceil(DEFAULT_ZK_SESSION_TIMEOUT * HAGroupStoreClient.ZK_SESSION_TIMEOUT_MULTIPLIER)); haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); From 2554614d33f73cfc1d265d4b93028fe25e3d0c5d Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Sun, 26 Oct 2025 12:48:38 -0700 Subject: [PATCH 5/7] Fixing checkstyle and blank lines --- .../apache/phoenix/jdbc/HAGroupStoreClient.java | 3 +-- .../apache/phoenix/jdbc/HAGroupStoreManager.java | 4 ++-- .../phoenix/hbase/index/IndexRegionObserver.java | 14 ++++++++------ ...gionServerEndpointWithConsistentFailoverIT.java | 6 +++--- .../IndexRegionObserverMutationBlockingIT.java | 4 ++-- .../phoenix/jdbc/FailoverPhoenixConnection2IT.java | 2 +- .../apache/phoenix/jdbc/HAGroupStoreClientIT.java | 4 ++-- .../apache/phoenix/jdbc/HAGroupStoreManagerIT.java | 2 +- .../jdbc/HighAvailabilityTestingUtility.java | 14 +++++++------- .../org/apache/phoenix/jdbc/PhoenixHAAdminIT.java | 6 +++--- 10 files changed, 30 insertions(+), 29 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java index 2ef8526bb7b..dcb13f129b0 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java @@ -85,7 +85,7 @@ * Uses {@link PathChildrenCache} from {@link org.apache.curator.framework.CuratorFramework}. */ public class HAGroupStoreClient implements Closeable { - + public static final String ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE = "phoenix" + ZKPaths.PATH_SEPARATOR + "consistentHA"; public static final String PHOENIX_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS @@ -159,7 +159,6 @@ public static HAGroupStoreClient getInstanceForZkUrl(Configuration conf, String if (result == null || !result.isHealthy) { result = new HAGroupStoreClient(conf, null, null, haGroupName, zkUrl); if (!result.isHealthy) { - result.close(); result = null; } else { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java index e6459591d40..2d84e7a2846 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java @@ -183,7 +183,7 @@ public static HAGroupStoreManager getInstanceForZkUrl(final Configuration conf, return new HAGroupStoreManager(conf, url); }); } - + @VisibleForTesting HAGroupStoreManager(final Configuration conf) { this(conf, getLocalZkUrl(conf)); @@ -245,7 +245,7 @@ public boolean isHAGroupOnClientStale(String haGroupName) throws IOException { //based on whether the SCN falls within its consistency point, and will require a change //in the logic her, with much bigger change. HAGroupStoreRecord hagroupStoreRecord = haGroupStoreClient.getHAGroupStoreRecord(); - return (HighAvailabilityPolicy.valueOf(hagroupStoreRecord.getPolicy()) + return (HighAvailabilityPolicy.valueOf(hagroupStoreRecord.getPolicy()) == HighAvailabilityPolicy.FAILOVER) && !(hagroupStoreRecord.getHAGroupState().getClusterRole().isActive()); } diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java index a8be9530936..efcf0fb8058 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java @@ -640,16 +640,18 @@ public void preBatchMutate(ObserverContext c, final Set haGroupNames = extractHAGroupNameAttribute(miniBatchOp); // Check if mutation is blocked for any of the HAGroupNames for (String haGroupName : haGroupNames) { - //TODO: Below approach might be slow need to figure out faster way, slower part is - //getting haGroupStoreClient We can also cache roleRecord (I tried it and still its - //slow due to haGroupStoreClient initialization) and caching will give us old result - //in case one cluster is unreachable instead of UNKNOWN. + //TODO: Below approach might be slow need to figure out faster way, + // slower part is getting haGroupStoreClient We can also cache + // roleRecord (I tried it and still it's slow due to haGroupStoreClient + // initialization) and caching will give us old result in case one cluster + // is unreachable instead of UNKNOWN. boolean isHAGroupOnClientStale = haGroupStoreManager .isHAGroupOnClientStale(haGroupName); if (StringUtils.isNotBlank(haGroupName) && isHAGroupOnClientStale) { - throw new StaleClusterRoleRecordException(String.format("HAGroupStoreRecord is " - + "stale for haGroup %s on client", haGroupName)); + throw new StaleClusterRoleRecordException( + String.format("HAGroupStoreRecord is stale for haGroup %s on client" + , haGroupName)); } //Check if mutation's haGroup is stale diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java index 203db482305..ced59be7df2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java @@ -73,7 +73,7 @@ public void setUp() throws Exception { peerZkUrl = CLUSTERS.getZkUrl2(); HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), - ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, + ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); } @@ -110,8 +110,8 @@ public void testGetClusterRoleRecordAndInvalidate() throws Exception { executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord, true); // Update the row - HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl, - CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl, + CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY, null); // Now Invalidate the Cache diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java index a57432670f4..39bc4519eea 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java @@ -105,8 +105,8 @@ public void testMutationBlockedOnDataTableWithIndex() throws Exception { // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state) HAGroupStoreRecord haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, - haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), + haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, + null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZkUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java index 13ddeeec454..69ccb7bf783 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java @@ -642,7 +642,7 @@ public void testStaleCrrVersionWithPointLookUpReadDetection() throws Exception { connection.commit(); } CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY, false); - + //Read operation should refresh the CRR leading to success and also update the CRR in HAGroup //as it is a point lookup try (ResultSet rs = connection.createStatement().executeQuery( diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java index fa09400c2e7..0f481e63fa5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java @@ -429,7 +429,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, @Test public void testHAGroupStoreClientWithRootPathDeletion() throws Exception { String haGroupName = testName.getMethodName(); - + HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); @@ -595,7 +595,7 @@ public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception { this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); - HAGroupStoreClient haGroupStoreClient + HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java index 3a3f1ccb251..35e208455d4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java @@ -786,7 +786,7 @@ public void testE2EStoreAndForwardWithAutomaticStateTransitions() throws Excepti CLUSTERS.getHBaseCluster1().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); CLUSTERS.getHBaseCluster2().getMiniHBaseCluster().getConf().setLong(ZK_SESSION_TIMEOUT, 10*1000); - HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, + HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null); HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java index 4bcbce7561f..7783261ffa5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java @@ -186,9 +186,9 @@ public String getURL(int clusterIndex, ClusterRoleRecord.RegistryType registryTy */ public void initClusterRole(String haGroupName, HighAvailabilityPolicy policy) throws Exception { - HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), + HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), null, policy != null ? policy.name() : HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); - HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, STANDBY.getDefaultHAGroupState(), + HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, STANDBY.getDefaultHAGroupState(), null, policy != null ? policy.name() :HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); upsertGroupRecordInBothSystemTable(haGroupName, ACTIVE, STANDBY, 1L, 1L,null, policy); addOrUpdateRoleRecordToClusters(haGroupName, activeRecord, standbyRecord); @@ -201,7 +201,7 @@ public void initClusterRole(String haGroupName, HighAvailabilityPolicy policy) * @param policy policy to be used for HA group */ public void initClusterRoleRecordFor1Cluster(String haGroupName, HighAvailabilityPolicy policy) throws Exception { - HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), + HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); upsertGroupRecordInASystemTable(haGroupName, ACTIVE, STANDBY, 1L, 1L, null, policy, 1); addOrUpdateRoleRecordToClusters(haGroupName, activeRecord, null); @@ -304,12 +304,12 @@ public void doUpdatesMissedWhenClusterWasDown(HighAvailabilityGroup haGroup, Clu final Pair currentRecord2 = getHaAdmin2().getHAGroupStoreRecordInZooKeeper(haGroupName); HAGroupStoreRecord newRecord; if (clusterIndex == 1) { - newRecord = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), + newRecord = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); refreshSystemTableInOneCluster(haGroupName, role1, role2, newRecord.getAdminCRRVersion(), currentRecord2.getLeft().getAdminCRRVersion(), null, haGroup.getRoleRecord().getPolicy(), clusterIndex); addOrUpdateRoleRecordToClusters(haGroupName, newRecord,null); } else { - newRecord = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), + newRecord = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); refreshSystemTableInOneCluster(haGroupName, role1, role2, currentRecord1.getLeft().getAdminCRRVersion(), newRecord.getAdminCRRVersion(), null, haGroup.getRoleRecord().getPolicy(), clusterIndex); addOrUpdateRoleRecordToClusters(haGroupName, null, newRecord); @@ -463,7 +463,7 @@ public void refreshClusterRoleRecordAfterClusterRestart(HighAvailabilityGroup ha refreshSystemTableInBothClusters(haGroupName, role1, role2, 2, 2, null, haGroup.getRoleRecord().getPolicy()); addOrUpdateRoleRecordToClusters(haGroupName, record1, record2); - + // Adding this wait because HAGroup on client is not quickly refreshed. Thread.sleep(10000); @@ -1002,7 +1002,7 @@ public static void primeHAGroupStoreClientOnCluster(HBaseTestingUtility cluster, LOG.warn("Fail to prime HAGroupStoreClient for {} because {}", haGroupName, e.getMessage()); } } - + /** * This tests that basic operation using a connection works. * diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java index 49de14f579d..37a55a7211f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java @@ -96,7 +96,7 @@ private void cleanupTestZnodes() throws Exception { public void testCreateHAGroupStoreRecordInZooKeeper() throws Exception { String haGroupName = testName.getMethodName(); String peerZKUrl = CLUSTERS.getZkUrl2(); - + HAGroupStoreRecord record = createInitialRecord(haGroupName); // Create the record in ZooKeeper @@ -113,7 +113,7 @@ public void testCreateHAGroupStoreRecordInZooKeeper() throws Exception { @Test public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exception { String haGroupName = testName.getMethodName(); - + HAGroupStoreRecord record1 = createInitialRecord(haGroupName); // Create the first record @@ -470,7 +470,7 @@ private HAGroupStoreRecord createInitialRecord(String haGroupName) { haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, - HighAvailabilityPolicy.FAILOVER.toString(), + HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L); } From 8f548559fa80d8cbbd0a30f4857b76af38c2ebda Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Mon, 27 Oct 2025 09:57:37 -0700 Subject: [PATCH 6/7] Fixing checkstyle and blank lines --- .../org/apache/phoenix/hbase/index/IndexRegionObserver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java index efcf0fb8058..e6b91d5bae8 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java @@ -640,8 +640,8 @@ public void preBatchMutate(ObserverContext c, final Set haGroupNames = extractHAGroupNameAttribute(miniBatchOp); // Check if mutation is blocked for any of the HAGroupNames for (String haGroupName : haGroupNames) { - //TODO: Below approach might be slow need to figure out faster way, - // slower part is getting haGroupStoreClient We can also cache + //TODO: Below approach might be slow need to figure out faster way, + // slower part is getting haGroupStoreClient We can also cache // roleRecord (I tried it and still it's slow due to haGroupStoreClient // initialization) and caching will give us old result in case one cluster // is unreachable instead of UNKNOWN. From 07b9de3358e88f7cedd3c6e9c901eee709c08e99 Mon Sep 17 00:00:00 2001 From: Ritesh Garg Date: Mon, 27 Oct 2025 17:47:32 -0700 Subject: [PATCH 7/7] PHOENIX-7566 Making lastSyncTimeInMs default to 0 and updating calculation logic --- .../phoenix/jdbc/HAGroupStoreClient.java | 21 +++--- .../phoenix/jdbc/HAGroupStoreRecord.java | 4 +- .../apache/phoenix/query/QueryServices.java | 3 + .../phoenix/query/QueryServicesOptions.java | 4 ++ .../replication/ReplicationLogGroup.java | 3 - .../ReplicationLogGroupWriter.java | 6 +- ...erverEndpointWithConsistentFailoverIT.java | 2 +- ...IndexRegionObserverMutationBlockingIT.java | 11 ++-- .../jdbc/HAGroupStateSubscriptionIT.java | 42 ++++++------ .../phoenix/jdbc/HAGroupStoreClientIT.java | 66 +++++++++---------- .../phoenix/jdbc/HAGroupStoreManagerIT.java | 29 ++++---- .../jdbc/HighAvailabilityTestingUtility.java | 26 ++++---- .../apache/phoenix/jdbc/PhoenixHAAdminIT.java | 16 ++--- .../phoenix/jdbc/HAGroupStoreRecordTest.java | 2 +- .../replication/ReplicationLogGroupTest.java | 3 +- 15 files changed, 124 insertions(+), 114 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java index dcb13f129b0..f05b58e4a62 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java @@ -51,6 +51,8 @@ import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole; import org.apache.phoenix.jdbc.ClusterRoleRecord.RegistryType; import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.util.JDBCUtil; @@ -120,6 +122,8 @@ public class HAGroupStoreClient implements Closeable { private final PathChildrenCacheListener peerCustomPathChildrenCacheListener; // Wait time for sync mode private final long waitTimeForSyncModeInMs; + // Rotation time for a log + private final long rotationTimeMs; // State tracking for transition detection private volatile HAGroupState lastKnownLocalState; private volatile HAGroupState lastKnownPeerState; @@ -221,6 +225,9 @@ public static List getHAGroupNames(String zkUrl) throws SQLException { this.zkUrl = zkUrl; this.waitTimeForSyncModeInMs = (long) Math.ceil(conf.getLong(ZK_SESSION_TIMEOUT, DEFAULT_ZK_SESSION_TIMEOUT) * ZK_SESSION_TIMEOUT_MULTIPLIER); + this.rotationTimeMs = + conf.getLong(QueryServices.REPLICATION_LOG_ROTATION_TIME_MS_KEY, + QueryServicesOptions.DEFAULT_REPLICATION_LOG_ROTATION_TIME_MS); // Custom Event Listener this.peerCustomPathChildrenCacheListener = peerPathChildrenCacheListener; try { @@ -329,11 +336,9 @@ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupStat if (currentHAGroupStoreRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC && haGroupState == HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC) { - lastSyncTimeInMs = System.currentTimeMillis(); - } else if (haGroupState == HAGroupState.ACTIVE_IN_SYNC - || !(ClusterRole.ACTIVE.equals(clusterRole) - || ClusterRole.ACTIVE_TO_STANDBY.equals(clusterRole))) { - lastSyncTimeInMs = null; + // We record the last round timestamp by subtracting the rotationTime and then + // taking the beginning of last round (floor) by first integer division and then multiplying again. + lastSyncTimeInMs = ((System.currentTimeMillis() - rotationTimeMs)/rotationTimeMs) * (rotationTimeMs); } HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord( currentHAGroupStoreRecord.getProtocolVersion(), @@ -427,15 +432,11 @@ private void initializeZNodeIfNeeded() throws IOException, SQLException { "System Table HAGroupRecord cannot be null"); HAGroupStoreRecord.HAGroupState defaultHAGroupState = systemTableRecord.getClusterRole().getDefaultHAGroupState(); - Long lastSyncTimeInMs - = defaultHAGroupState.equals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC) - ? System.currentTimeMillis() - : null; HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord( HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, defaultHAGroupState, - lastSyncTimeInMs, + 0L, systemTableRecord.getPolicy().toString(), systemTableRecord.getPeerZKUrl(), systemTableRecord.getClusterUrl(), diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java index 70f4c12053e..54f683247cb 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java @@ -155,7 +155,7 @@ public static HAGroupState from(byte[] bytes) { private final String protocolVersion; private final String haGroupName; private final HAGroupState haGroupState; - private final Long lastSyncStateTimeInMs; + private final long lastSyncStateTimeInMs; private final String policy; private final String peerZKUrl; private final String clusterUrl; @@ -166,7 +166,7 @@ public static HAGroupState from(byte[] bytes) { public HAGroupStoreRecord(@JsonProperty("protocolVersion") String protocolVersion, @JsonProperty("haGroupName") String haGroupName, @JsonProperty("haGroupState") HAGroupState haGroupState, - @JsonProperty("lastSyncStateTimeInMs") Long lastSyncStateTimeInMs, + @JsonProperty("lastSyncStateTimeInMs") long lastSyncStateTimeInMs, @JsonProperty("policy") String policy, @JsonProperty("peerZKUrl") String peerZKUrl, @JsonProperty("clusterUrl") String clusterUrl, diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java index c2d1ccd9c8e..6e00b7499f9 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -564,6 +564,9 @@ public interface QueryServices extends SQLCloseable { // HA Group Store sync job interval in seconds String HA_GROUP_STORE_SYNC_INTERVAL_SECONDS = "phoenix.ha.group.store.sync.interval.seconds"; + public static final String REPLICATION_LOG_ROTATION_TIME_MS_KEY = + "phoenix.replication.log.rotation.time.ms"; + /** * Get executor service used for parallel scans */ diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index c3e88168e49..5326b583b45 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -89,6 +89,7 @@ import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_ENABLED; import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_THREAD_POOL_SIZE; import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_THRESHOLD_MILLISECONDS; +import static org.apache.phoenix.query.QueryServices.REPLICATION_LOG_ROTATION_TIME_MS_KEY; import static org.apache.phoenix.query.QueryServices.ROW_KEY_ORDER_SALTED_TABLE_ATTRIB; import static org.apache.phoenix.query.QueryServices.RPC_TIMEOUT_ATTRIB; import static org.apache.phoenix.query.QueryServices.RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS; @@ -477,6 +478,8 @@ public class QueryServicesOptions { // Default HA Group Store sync job interval in seconds (15 minutes = 900 seconds) public static final int DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS = 900; + public static final long DEFAULT_REPLICATION_LOG_ROTATION_TIME_MS = 60 * 1000L; + private final Configuration config; private QueryServicesOptions(Configuration config) { @@ -592,6 +595,7 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT, DEFAULT_CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT) .setIfUnset(CQSI_THREAD_POOL_METRICS_ENABLED, DEFAULT_CQSI_THREAD_POOL_METRICS_ENABLED) + .setIfUnset(REPLICATION_LOG_ROTATION_TIME_MS_KEY, DEFAULT_REPLICATION_LOG_ROTATION_TIME_MS) .setIfUnset(HA_GROUP_STORE_SYNC_INTERVAL_SECONDS, DEFAULT_HA_GROUP_STORE_SYNC_INTERVAL_SECONDS); diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroup.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroup.java index 0c3eeacfd93..4dde98f9e6b 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroup.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroup.java @@ -61,9 +61,6 @@ public class ReplicationLogGroup { public static final String REPLICATION_NUM_SHARDS_KEY = "phoenix.replication.log.shards"; public static final int DEFAULT_REPLICATION_NUM_SHARDS = 1000; public static final int MAX_REPLICATION_NUM_SHARDS = 100000; - public static final String REPLICATION_LOG_ROTATION_TIME_MS_KEY = - "phoenix.replication.log.rotation.time.ms"; - public static final long DEFAULT_REPLICATION_LOG_ROTATION_TIME_MS = 60 * 1000L; public static final String REPLICATION_LOG_ROTATION_SIZE_BYTES_KEY = "phoenix.replication.log.rotation.size.bytes"; public static final long DEFAULT_REPLICATION_LOG_ROTATION_SIZE_BYTES = 256 * 1024 * 1024L; diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroupWriter.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroupWriter.java index a3810e0fb7f..d5d9cd83c08 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroupWriter.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/ReplicationLogGroupWriter.java @@ -35,6 +35,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.replication.log.LogFileWriter; import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.phoenix.util.EnvironmentEdgeManager; @@ -143,8 +145,8 @@ protected ReplicationLogGroupWriter(ReplicationLogGroup logGroup) { this.logGroup = logGroup; Configuration conf = logGroup.getConfiguration(); this.rotationTimeMs = - conf.getLong(ReplicationLogGroup.REPLICATION_LOG_ROTATION_TIME_MS_KEY, - ReplicationLogGroup.DEFAULT_REPLICATION_LOG_ROTATION_TIME_MS); + conf.getLong(QueryServices.REPLICATION_LOG_ROTATION_TIME_MS_KEY, + QueryServicesOptions.DEFAULT_REPLICATION_LOG_ROTATION_TIME_MS); long rotationSize = conf.getLong(ReplicationLogGroup.REPLICATION_LOG_ROTATION_SIZE_BYTES_KEY, ReplicationLogGroup.DEFAULT_REPLICATION_LOG_ROTATION_SIZE_BYTES); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java index ced59be7df2..fbd8b0c9212 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointWithConsistentFailoverIT.java @@ -87,7 +87,7 @@ public void testGetClusterRoleRecordAndInvalidate() throws Exception { try (PhoenixHAAdmin peerHAAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_RECORD_NAMESPACE)) { HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, - HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), CLUSTERS.getMasterAddress2(), CLUSTERS.getMasterAddress1(), 0L); peerHAAdmin.createHAGroupStoreRecordInZooKeeper(peerHAGroupStoreRecord); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java index 39bc4519eea..e015aa4cf95 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java @@ -61,6 +61,7 @@ public class IndexRegionObserverMutationBlockingIT { private String zkUrl; private String peerZkUrl; + @Rule public TestName testName = new TestName(); private Properties clientProps = new Properties(); @@ -82,6 +83,8 @@ public void setUp() throws Exception { haAdmin = CLUSTERS.getHaAdmin1(); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER); + this.zkUrl = CLUSTERS.getZkUrl1(); + this.peerZkUrl = CLUSTERS.getZkUrl2(); } @Test @@ -106,7 +109,7 @@ public void testMutationBlockedOnDataTableWithIndex() throws Exception { HAGroupStoreRecord haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZkUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); @@ -178,7 +181,7 @@ public void testMutationBlockingTransition() throws Exception { HAGroupStoreRecord haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZkUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -199,7 +202,7 @@ public void testMutationBlockingTransition() throws Exception { haGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, - null, HighAvailabilityPolicy.FAILOVER.toString(), + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZkUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -249,7 +252,7 @@ public void testSystemHAGroupTableMutationsAllowedDuringActiveToStandby() throws HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZkUrl, CLUSTERS.getMasterAddress1(), diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java index cc224139ef5..eb142cfa5e3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStateSubscriptionIT.java @@ -147,12 +147,12 @@ public void testDifferentTargetStatesPerCluster() throws Exception { manager.subscribeToTargetState(haGroupName, ACTIVE_NOT_IN_SYNC, ClusterType.PEER, peerListener); // Trigger transition to STANDBY_TO_ACTIVE on LOCAL cluster - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Trigger transition to STANDBY on PEER cluster - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -193,7 +193,7 @@ public void testUnsubscribeSpecificCluster() throws Exception { manager.unsubscribeFromTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.LOCAL, listener); // First, establish ACTIVE_IN_SYNC state on PEER cluster - HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerActiveRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -201,7 +201,7 @@ public void testUnsubscribeSpecificCluster() throws Exception { assertEquals("Should receive no notifications for ACTIVE_IN_SYNC state", 0, totalNotifications.get()); // Now trigger transition from ACTIVE_IN_SYNC to STANDBY on PEER → should call listener - HAGroupStoreRecord peerStandbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerStandbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerStandbyRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -254,12 +254,12 @@ public void testMultipleListenersMultipleClusters() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.DEGRADED_STANDBY, ClusterType.PEER, listener2); // Trigger transition to DEGRADED_STANDBY on LOCAL - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Trigger transition to DEGRADED_STANDBY on PEER - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.DEGRADED_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -299,12 +299,12 @@ public void testSameListenerDifferentTargetStates() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.ACTIVE_IN_SYNC, ClusterType.PEER, sharedListener); // Trigger target state A on LOCAL - HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); // Trigger target state B on PEER - HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -374,7 +374,7 @@ public void testListenerExceptionIsolation() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.ACTIVE_IN_SYNC, ClusterType.LOCAL, goodListener2); // Trigger transition to target state - HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -460,18 +460,18 @@ public void testHighFrequencyMultiClusterChanges() throws Exception { // Rapidly alternate state changes on both clusters final int changeCount = 5; - HAGroupStoreRecord initialPeerRecord = new HAGroupStoreRecord("1.0", haGroupName,HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord initialPeerRecord = new HAGroupStoreRecord("1.0", haGroupName,HAGroupState.DEGRADED_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(initialPeerRecord); for (int i = 0; i < changeCount; i++) { // Change local cluster HAGroupStoreRecord localRecord = new HAGroupStoreRecord("1.0", haGroupName, - (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.DEGRADED_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localRecord, -1); // Change peer cluster HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("1.0", haGroupName, - (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + (i % 2 == 0) ? HAGroupState.STANDBY : HAGroupState.STANDBY_TO_ACTIVE, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerRecord, -1); // Small delay between changes @@ -547,7 +547,7 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY_TO_ACTIVE, ClusterType.PEER, listener4); // Test initial functionality - trigger ACTIVE_IN_SYNC on LOCAL - HAGroupStoreRecord localActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -555,7 +555,7 @@ public void testSubscriptionCleanupPerCluster() throws Exception { assertEquals("Should have 2 LOCAL ACTIVE_IN_SYNC notifications initially", 2, localActiveNotifications.get()); // Test initial functionality - trigger STANDBY_TO_ACTIVE on PEER - HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerActiveRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerActiveRecord); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -572,11 +572,11 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.unsubscribeFromTargetState(haGroupName, HAGroupState.ACTIVE_IN_SYNC, ClusterType.LOCAL, listener4); // Test after partial unsubscribe - trigger ACTIVE_IN_SYNC on LOCAL again by first changing to some other state. - HAGroupStoreRecord localActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord2, 1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - HAGroupStoreRecord localActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord3, 2); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -584,11 +584,11 @@ public void testSubscriptionCleanupPerCluster() throws Exception { assertEquals("Should have 1 LOCAL ACTIVE_IN_SYNC notification after partial unsubscribe", 1, localActiveNotifications.get()); // Test after partial unsubscribe - trigger STANDBY_TO_ACTIVE on PEER again - HAGroupStoreRecord peerActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerActiveRecord2 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerActiveRecord2, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - HAGroupStoreRecord peerActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerActiveRecord3 = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY_TO_ACTIVE, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerActiveRecord3, 1); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -605,11 +605,11 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.unsubscribeFromTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.PEER, listener4); // Test after complete unsubscribe - trigger ACTIVE_NOT_IN_SYNC on both clusters - HAGroupStoreRecord localActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord localActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, localActiveRecord4, 3); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); - HAGroupStoreRecord peerActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord peerActiveRecord4 = new HAGroupStoreRecord("1.0", haGroupName, ACTIVE_NOT_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, peerActiveRecord4, 2); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -630,7 +630,7 @@ public void testSubscriptionCleanupPerCluster() throws Exception { manager.subscribeToTargetState(haGroupName, HAGroupState.STANDBY, ClusterType.LOCAL, newTestListener); // Trigger STANDBY state and verify new subscription works - HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); + HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord("1.0", haGroupName, HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, standbyRecord, 4); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java index 0f481e63fa5..aeee4f3851e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java @@ -136,7 +136,7 @@ public void testHAGroupStoreClientWithBothNullZKUrl() throws Exception { public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidUrl() throws Exception { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl); @@ -150,7 +150,7 @@ public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidU // Now update peerZKUrl to null and rebuild record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), null, this.masterUrl, null, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -158,7 +158,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Now update System table to contain valid peer ZK URL and also change local cluster role to STANDBY record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -171,7 +171,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Now update local HAGroupStoreRecord to STANDBY to verify that HAGroupStoreClient is working as normal record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -185,7 +185,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // This URL can also be considered unreachable url due to a connectivity issue. String invalidUrl = "invalidURL"; record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), invalidUrl, this.masterUrl, invalidUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -224,7 +224,7 @@ public void testHAGroupStoreClientWithSingleHAGroupStoreRecord() throws Exceptio // Create and store HAGroupStoreRecord with ACTIVE state HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -233,7 +233,7 @@ public void testHAGroupStoreClientWithSingleHAGroupStoreRecord() throws Exceptio // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -244,7 +244,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -254,7 +254,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Change it again to ACTIVE_TO_STANDBY so that we can validate watcher works repeatedly record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -264,7 +264,7 @@ record = new HAGroupStoreRecord("v1.0", haGroupName, // Change it back to ACTIVE to verify transition works record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -300,10 +300,10 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Setup initial HAGroupStoreRecords HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); HAGroupStoreRecord record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); @@ -320,10 +320,10 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY for only 1 record record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); @@ -338,10 +338,10 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); @@ -355,10 +355,10 @@ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Excep // Change other record to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly record1 = new HAGroupStoreRecord("v1.0", haGroupName1, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); record2 = new HAGroupStoreRecord("v1.0", haGroupName2, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1); @@ -377,7 +377,7 @@ public void testMultiThreadedAccessToHACache() throws Exception { // Setup initial HAGroupStoreRecord HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -403,7 +403,7 @@ public void testMultiThreadedAccessToHACache() throws Exception { // Update HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -431,7 +431,7 @@ public void testHAGroupStoreClientWithRootPathDeletion() throws Exception { String haGroupName = testName.getMethodName(); HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1); @@ -452,7 +452,7 @@ public void testHAGroupStoreClientWithRootPathDeletion() throws Exception { assertNotNull(currentRecord.getLastSyncStateTimeInMs()); record1 = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1); @@ -468,7 +468,7 @@ public void testThrowsExceptionWithZKDisconnectionAndThenConnection() throws Exc // Setup initial HAGroupStoreRecord HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record); @@ -538,7 +538,7 @@ public void testSetHAGroupStatusIfNeededUpdateExistingRecord() throws Exception // Create initial record HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); @@ -564,7 +564,7 @@ public void testSetHAGroupStatusIfNeededNoUpdateWhenNotNeeded() throws Exception // Create initial record with current timestamp HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); Stat initialRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight(); @@ -591,7 +591,7 @@ public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception { String haGroupName = testName.getMethodName(); // Create initial record HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); @@ -615,7 +615,7 @@ public void testSetHAGroupStatusIfNeededWithInvalidTransition() throws Exception // Create initial record with ACTIVE state HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); @@ -661,7 +661,7 @@ public void testSetHAGroupStatusIfNeededMultipleTransitions() throws Exception { // Create initial record with old timestamp HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord); @@ -756,13 +756,13 @@ public void testGetClusterRoleRecordNormalCase() throws Exception { // Create HAGroupStoreRecord for local cluster HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord); // Create HAGroupStoreRecord for peer cluster HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.STANDBY, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.peerMasterUrl, this.masterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerRecord); @@ -789,7 +789,7 @@ public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws E // Create HAGroupStoreRecord for local cluster only (no peer record) HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord); @@ -815,7 +815,7 @@ public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws E private HAGroupStoreRecord createHAGroupStoreRecord(String haGroupName) { return new HAGroupStoreRecord("v1.0", haGroupName, - HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null, HighAvailabilityPolicy.FAILOVER.toString(), + HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.masterUrl, this.peerMasterUrl, 0L); } @@ -970,7 +970,7 @@ public void testPeriodicSyncJobExecutorStartsAndSyncsData() throws Exception { // Also create a peer ZK record with STANDBY_TO_ACTIVE role to test peer role sync HAGroupStoreRecord peerZkRecord = new HAGroupStoreRecord("v2.0", haGroupName, - HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, null, + HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE, 0L, HighAvailabilityPolicy.FAILOVER.toString(), updatedClusterUrl, this.peerMasterUrl, updatedClusterUrl, 5L); createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerZkRecord); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java index 35e208455d4..5e1306f079a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java @@ -109,7 +109,7 @@ public void testMutationBlockingWithSingleHAGroup() throws Exception { // Update to ACTIVE_TO_STANDBY role (should block mutations) HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0); Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS); @@ -129,7 +129,7 @@ public void testMutationBlockingWithMultipleHAGroups() throws Exception { this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null); HAGroupStoreRecord activeRecord1 = new HAGroupStoreRecord( "1.0", haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(activeRecord1); @@ -143,7 +143,7 @@ public void testMutationBlockingWithMultipleHAGroups() throws Exception { // Update only second group to ACTIVE_NOT_IN_SYNC_TO_STANDBY HAGroupStoreRecord transitionRecord2 = new HAGroupStoreRecord( "1.0", haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName2, transitionRecord2, 0); @@ -188,10 +188,9 @@ public void testGetHAGroupStoreRecord() throws Exception { // Complete object comparison field-by-field assertEquals(haGroupName, retrievedOpt.get().getHaGroupName()); assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, retrievedOpt.get().getHAGroupState()); - Long lastSyncStateTimeInMs = retrievedOpt.get().getLastSyncStateTimeInMs(); - Long mtime = currentRecordAndStat.getRight().getMtime(); + long lastSyncStateTimeInMs = retrievedOpt.get().getLastSyncStateTimeInMs(); // Allow a small margin of error - assertTrue(Math.abs(lastSyncStateTimeInMs - mtime) <= 1); + assertEquals(0L, lastSyncStateTimeInMs); assertEquals(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, retrievedOpt.get().getProtocolVersion()); } @@ -212,7 +211,7 @@ public void testGetPeerHAGroupStoreRecord() throws Exception { // Create a HAGroupStoreRecord in the peer cluster HAGroupStoreRecord peerRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord); @@ -239,7 +238,7 @@ public void testGetPeerHAGroupStoreRecord() throws Exception { // Create peer record again with different state HAGroupStoreRecord newPeerRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); peerHaAdmin.createHAGroupStoreRecordInZooKeeper(newPeerRecord); @@ -280,7 +279,7 @@ public void testInvalidateHAGroupStoreClient() throws Exception { // Create a HAGroupStoreRecord first HAGroupStoreRecord record = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(record); @@ -323,7 +322,7 @@ public void testMutationBlockDisabled() throws Exception { // Create HAGroupStoreRecord with ACTIVE_IN_SYNC_TO_STANDBY role HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(transitionRecord); @@ -344,7 +343,7 @@ public void testSetHAGroupStatusToStoreAndForward() throws Exception { // Create an initial HAGroupStoreRecord with ACTIVE status HAGroupStoreRecord initialRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord); @@ -395,7 +394,7 @@ public void testSetHAGroupStatusToSync() throws Exception { assertTrue(updatedRecordOpt.isPresent()); HAGroupStoreRecord updatedRecord = updatedRecordOpt.get(); assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, updatedRecord.getHAGroupState()); - assertNull(updatedRecord.getLastSyncStateTimeInMs()); + assertEquals(initialRecord.getLastSyncStateTimeInMs(), updatedRecord.getLastSyncStateTimeInMs()); } @Test @@ -483,7 +482,7 @@ public void testSetReaderToDegraded() throws Exception { // Update the auto-created record to STANDBY state for testing HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); // Get the record to initialize ZNode from HAGroup so that we can artificially update it via HAAdmin @@ -517,7 +516,7 @@ public void testSetReaderToHealthy() throws Exception { // Update the auto-created record to DEGRADED_STANDBY state for testing HAGroupStoreRecord degradedReaderRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, degradedReaderRecord, 0); @@ -546,7 +545,7 @@ public void testReaderStateTransitionInvalidStates() throws Exception { // Update the auto-created record to ACTIVE_IN_SYNC state (invalid for both operations) HAGroupStoreRecord activeRecord = new HAGroupStoreRecord( "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, - null, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), this.peerZKUrl, this.zkUrl, this.peerZKUrl, 0L); haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, activeRecord, 0); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java index 7783261ffa5..333ef029e7c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java @@ -187,9 +187,9 @@ public String getURL(int clusterIndex, ClusterRoleRecord.RegistryType registryTy public void initClusterRole(String haGroupName, HighAvailabilityPolicy policy) throws Exception { HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), - null, policy != null ? policy.name() : HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + 0L, policy != null ? policy.name() : HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, STANDBY.getDefaultHAGroupState(), - null, policy != null ? policy.name() :HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); + 0L, policy != null ? policy.name() :HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); upsertGroupRecordInBothSystemTable(haGroupName, ACTIVE, STANDBY, 1L, 1L,null, policy); addOrUpdateRoleRecordToClusters(haGroupName, activeRecord, standbyRecord); } @@ -202,7 +202,7 @@ public void initClusterRole(String haGroupName, HighAvailabilityPolicy policy) */ public void initClusterRoleRecordFor1Cluster(String haGroupName, HighAvailabilityPolicy policy) throws Exception { HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), - null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); upsertGroupRecordInASystemTable(haGroupName, ACTIVE, STANDBY, 1L, 1L, null, policy, 1); addOrUpdateRoleRecordToClusters(haGroupName, activeRecord, null); } @@ -305,12 +305,12 @@ public void doUpdatesMissedWhenClusterWasDown(HighAvailabilityGroup haGroup, Clu HAGroupStoreRecord newRecord; if (clusterIndex == 1) { newRecord = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), - null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); refreshSystemTableInOneCluster(haGroupName, role1, role2, newRecord.getAdminCRRVersion(), currentRecord2.getLeft().getAdminCRRVersion(), null, haGroup.getRoleRecord().getPolicy(), clusterIndex); addOrUpdateRoleRecordToClusters(haGroupName, newRecord,null); } else { newRecord = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), - null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); refreshSystemTableInOneCluster(haGroupName, role1, role2, currentRecord1.getLeft().getAdminCRRVersion(), newRecord.getAdminCRRVersion(), null, haGroup.getRoleRecord().getPolicy(), clusterIndex); addOrUpdateRoleRecordToClusters(haGroupName, null, newRecord); } @@ -370,8 +370,8 @@ public void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRole role1, final Pair currentRecord2 = haAdmin2.getHAGroupStoreRecordInZooKeeper(haGroupName); String policyString = policy != null ? policy.name() : HighAvailabilityPolicy.FAILOVER.name(); - HAGroupStoreRecord record1 = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), null, policyString, getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); - HAGroupStoreRecord record2 = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), null, policyString, getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); + HAGroupStoreRecord record1 = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), 0L, policyString, getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + HAGroupStoreRecord record2 = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), 0L, policyString, getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); refreshSystemTableInBothClusters(haGroupName, role1, role2, 1L, 1L, null, haGroup.getRoleRecord().getPolicy()); @@ -407,7 +407,7 @@ public void transitClusterRoleWithCluster1Down(HighAvailabilityGroup haGroup, Cl String haGroupName = haGroup.getGroupInfo().getName(); final Pair currentRecord2 = haAdmin2.getHAGroupStoreRecordInZooKeeper(haGroupName); - HAGroupStoreRecord record2 = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); + HAGroupStoreRecord record2 = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 1L); refreshSystemTableInOneCluster(haGroupName, role1, role2, 1, 1, null, haGroup.getRoleRecord().getPolicy(), 2); addOrUpdateRoleRecordToClusters(haGroupName, null, record2); @@ -419,7 +419,7 @@ public void transitClusterRoleWithCluster1Down(HighAvailabilityGroup haGroup, Cl haGroup.refreshClusterRoleRecord(true); //If cluster 1 is down, server won't be able to reach peer for states and will get version passed in refreshSystemTableInOneCluster and OFFLINE role. - HAGroupStoreRecord record1 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, UNKNOWN.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L);; + HAGroupStoreRecord record1 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, UNKNOWN.getDefaultHAGroupState(), 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L);; // wait for the cluster roles are populated into client side from RegionServer endpoints. ClusterRoleRecord newRoleRecord = generateCRRFromHAGroupStoreRecord(haGroup, record1, record2); @@ -433,7 +433,7 @@ public void transitClusterRoleWithCluster2Down(HighAvailabilityGroup haGroup, Cl String haGroupName = haGroup.getGroupInfo().getName(); final Pair currentRecord1 = haAdmin1.getHAGroupStoreRecordInZooKeeper(haGroupName); - HAGroupStoreRecord record1 = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + HAGroupStoreRecord record1 = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); refreshSystemTableInOneCluster(haGroupName, role1, role2, 1, 1, null, haGroup.getRoleRecord().getPolicy(), 1); addOrUpdateRoleRecordToClusters(haGroupName, record1, null); @@ -445,7 +445,7 @@ public void transitClusterRoleWithCluster2Down(HighAvailabilityGroup haGroup, Cl haGroup.refreshClusterRoleRecord(true); //If cluster 2 is down, server won't be able to reach peer for states and will get version passed in refreshSystemTableInOneCluster and OFFLINE role. - HAGroupStoreRecord record2 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, UNKNOWN.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); + HAGroupStoreRecord record2 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, UNKNOWN.getDefaultHAGroupState(), 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 1L); // wait for the cluster roles are populated into client side from RegionServer endpoints. ClusterRoleRecord newRoleRecord = generateCRRFromHAGroupStoreRecord(haGroup, record1, record2); @@ -458,8 +458,8 @@ public void refreshClusterRoleRecordAfterClusterRestart(HighAvailabilityGroup ha String haGroupName = haGroup.getGroupInfo().getName(); //TODO:- Inducing version change, but how would we know that a URL is changed // as we don't store url in ZNodes so it doesn't increase version for that change - HAGroupStoreRecord record1 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, role1.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 2L); - HAGroupStoreRecord record2 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, role2.getDefaultHAGroupState(), null, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 2L); + HAGroupStoreRecord record1 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, role1.getDefaultHAGroupState(), 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), 2L); + HAGroupStoreRecord record2 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, role2.getDefaultHAGroupState(), 0L, HighAvailabilityPolicy.FAILOVER.name(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), 2L); refreshSystemTableInBothClusters(haGroupName, role1, role2, 2, 2, null, haGroup.getRoleRecord().getPolicy()); addOrUpdateRoleRecordToClusters(haGroupName, record1, record2); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java index 37a55a7211f..99283366363 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java @@ -124,7 +124,7 @@ public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exc HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + 0L, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -168,7 +168,7 @@ public void testUpdateHAGroupStoreRecordInZooKeeper() throws Exception { HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -205,7 +205,7 @@ public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exc HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -216,7 +216,7 @@ public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exc HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -293,7 +293,7 @@ public void testCompleteWorkflowCreateUpdateGet() throws Exception { HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), peerZKUrl, CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -349,7 +349,7 @@ public void testMultiThreadedUpdatesConcurrentVersionConflict() throws Exception HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + 0L, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -428,7 +428,7 @@ public void testMultiThreadedUpdatesWithDifferentVersions() throws Exception { HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY, - null, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), + 0L, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L ); @@ -469,7 +469,7 @@ private HAGroupStoreRecord createInitialRecord(String haGroupName) { "v1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, - null, + 0L, HighAvailabilityPolicy.FAILOVER.toString(), CLUSTERS.getZkUrl2(), CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), 0L); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java index 7729673bcaa..4a33ca18c44 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java @@ -365,6 +365,6 @@ private HAGroupStoreRecord getHAGroupStoreRecord(String haGroupName, String prot String policy, String peerZKUrl, String clusterUrl, String peerClusterUrl, long adminCRRVersion) { return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState, - null, policy, peerZKUrl, clusterUrl, peerClusterUrl, adminCRRVersion); + 0L, policy, peerZKUrl, clusterUrl, peerClusterUrl, adminCRRVersion); } } \ No newline at end of file diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/ReplicationLogGroupTest.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/ReplicationLogGroupTest.java index d3176490d10..9fb31112b92 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/replication/ReplicationLogGroupTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/ReplicationLogGroupTest.java @@ -51,6 +51,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.ServerName; +import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.replication.log.LogFileWriter; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.junit.After; @@ -101,7 +102,7 @@ public void setUp() throws IOException { // Set a short sync timeout for testing conf.setLong(ReplicationLogGroup.REPLICATION_LOG_SYNC_TIMEOUT_KEY, TEST_SYNC_TIMEOUT); // Set rotation time to 10 seconds - conf.setLong(ReplicationLogGroup.REPLICATION_LOG_ROTATION_TIME_MS_KEY, TEST_ROTATION_TIME); + conf.setLong(QueryServices.REPLICATION_LOG_ROTATION_TIME_MS_KEY, TEST_ROTATION_TIME); // Small size threshold for testing conf.setLong(ReplicationLogGroup.REPLICATION_LOG_ROTATION_SIZE_BYTES_KEY, TEST_ROTATION_SIZE_BYTES);