Make the keymeta table initialization happen in a predictable manner (#7433)

Author: haridsv

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -2669,5 +2669,5 @@ List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType, Server
    * Refresh the system key cache on all specified region servers.
    * @param regionServers the list of region servers to refresh the system key cache on
    */
-  void refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) throws IOException;
+  void refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) throws IOException;
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java

@@ -1148,7 +1148,7 @@ public void restoreBackupSystemTable(String snapshotName) throws IOException {
   }
 
   @Override
-  public void refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) throws IOException {
-    get(admin.refreshSystemKeyCacheOnAllServers(regionServers));
+  public void refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) throws IOException {
+    get(admin.refreshSystemKeyCacheOnServers(regionServers));
   }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java

@@ -1879,5 +1879,5 @@ CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNames, Str
    * Refresh the system key cache on all specified region servers.
    * @param regionServers the list of region servers to refresh the system key cache on
    */
-  CompletableFuture<Void> refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers);
+  CompletableFuture<Void> refreshSystemKeyCacheOnServers(Set<ServerName> regionServers);
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java

@@ -687,8 +687,8 @@ public CompletableFuture<Void> updateConfiguration(String groupName) {
   }
 
   @Override
-  public CompletableFuture<Void> refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) {
-    return wrap(rawAdmin.refreshSystemKeyCacheOnAllServers(regionServers));
+  public CompletableFuture<Void> refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) {
+    return wrap(rawAdmin.refreshSystemKeyCacheOnServers(regionServers));
   }
 
   @Override

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java

@@ -4665,7 +4665,7 @@ MasterProtos.RestoreBackupSystemTableResponse> procedureCall(request,
   }
 
   @Override
-  public CompletableFuture<Void> refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) {
+  public CompletableFuture<Void> refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) {
     CompletableFuture<Void> future = new CompletableFuture<>();
     List<CompletableFuture<Void>> futures =
       regionServers.stream().map(this::refreshSystemKeyCache).collect(Collectors.toList());

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java

@@ -127,6 +127,7 @@ public enum LockState {
   private long rootProcId = NO_PROC_ID;
   private long procId = NO_PROC_ID;
   private long submittedTime;
+  private boolean isCriticalSystemTable;
 
   // Runtime state, updated every operation
   private ProcedureState state = ProcedureState.INITIALIZING;
@@ -608,6 +609,14 @@ protected void setParentProcId(long parentProcId) {
     this.parentProcId = parentProcId;
   }
 
+  public void setCriticalSystemTable(boolean isCriticalSystemTable) {
+    this.isCriticalSystemTable = isCriticalSystemTable;
+  }
+
+  public boolean isCriticalSystemTable() {
+    return isCriticalSystemTable;
+  }
+
   protected void setRootProcId(long rootProcId) {
     this.rootProcId = rootProcId;
   }

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java

@@ -1975,6 +1975,7 @@ private Procedure<TEnvironment>[] initializeChildren(RootProcedureState<TEnviron
       subproc.setParentProcId(procedure.getProcId());
       subproc.setRootProcId(rootProcId);
       subproc.setProcId(nextProcId());
+      subproc.setCriticalSystemTable(procedure.isCriticalSystemTable());
       procStack.addSubProcedure(subproc);
     }
 

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureUtil.java

@@ -175,6 +175,10 @@ public static ProcedureProtos.Procedure convertToProtoProcedure(Procedure<?> pro
       builder.setParentId(proc.getParentProcId());
     }
 
+    if (proc.isCriticalSystemTable()) {
+      builder.setIsCryticalSystemTable(true);
+    }
+
     if (proc.hasTimeout()) {
       builder.setTimeout(proc.getTimeout());
     }
@@ -244,6 +248,10 @@ public static Procedure<?> convertToProcedure(ProcedureProtos.Procedure proto)
       proc.setParentProcId(proto.getParentId());
     }
 
+    if (proto.hasIsCryticalSystemTable()) {
+      proc.setCriticalSystemTable(proto.getIsCryticalSystemTable());
+    }
+
     if (proto.hasOwner()) {
       proc.setOwner(proto.getOwner());
     }

hbase-protocol-shaded/src/main/protobuf/server/Procedure.proto

@@ -73,6 +73,9 @@ message Procedure {
   // whether the procedure has been executed
   // since we do not always maintain the stack_id now, we need a separated flag
   optional bool executed = 18 [default = false];
+
+  // Indicates that the procedure belongs to a crytical system table.
+  optional bool is_crytical_system_table = 19 [default = false];
 }
 
 /**

hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaAdminImpl.java

@@ -94,7 +94,7 @@ public boolean rotateSTK() throws IOException {
 
     LOG.info("System Key is rotated, initiating cache refresh on all region servers");
     try {
-      FutureUtils.get(getAsyncAdmin(master).refreshSystemKeyCacheOnAllServers(regionServers));
+      FutureUtils.get(getAsyncAdmin(master).refreshSystemKeyCacheOnServers(regionServers));
     } catch (Exception e) {
       throw new IOException(
         "Failed to initiate System Key cache refresh on one or more region servers", e);