diff --git a/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java b/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java index bd009b0e99..c4f54b0878 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java @@ -58,6 +58,11 @@ public interface StateMachine extends Closeable { Logger LOG = LoggerFactory.getLogger(StateMachine.class); + enum EventStatus { + INITIATED, + COMPLETED + } + /** A registry to support different state machines in multi-raft environment. */ interface Registry extends Function { } @@ -229,20 +234,42 @@ default void notifyLogFailed(Throwable cause, LogEntryProto failedEntry) {} default void notifySnapshotInstalled(InstallSnapshotResult result, long snapshotIndex, RaftPeer peer) {} /** - * Notify the {@link StateMachine} that the server for this division has been shut down. - * @deprecated please use/override {@link #notifyServerShutdown(RoleInfoProto, boolean)} instead + * The same as notifyServerShutdown(roleInfo, false, EventStatus.COMPLETED). + * + * @deprecated please use/override {@link #notifyServerShutdown(RoleInfoProto, boolean, EventStatus)} instead */ @Deprecated - default void notifyServerShutdown(RoleInfoProto roleInfo) { - notifyServerShutdown(roleInfo, false); - } + default void notifyServerShutdown(RoleInfoProto roleInfo) {} /** * Notify the {@link StateMachine} that either the server for this division or all the servers have been shut down. + * + * @deprecated please override {@link #notifyServerShutdown(RoleInfoProto, boolean, EventStatus)} instead + */ + @Deprecated + default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers) { + notifyServerShutdown(roleInfo); + } + + /** + * Notify the {@link StateMachine} server shutdown. + *

+ * {@link EventStatus#INITIATED} is notified before the shutdown has closed any resources, + * in particular, before waiting for the committed log entries to be applied to this {@link StateMachine}, + * which is unbounded in time. An implementation must return promptly for {@link EventStatus#INITIATED}. + *

+ * {@link EventStatus#COMPLETED} is notified after the shutdown has completed. + * * @param roleInfo roleInfo this server - * @param allServer whether all raft servers will be shutdown at this time + * @param allServers whether all raft servers will be shutdown at this time + * @param eventStatus {@link EventStatus#INITIATED} or {@link EventStatus#COMPLETED} */ - default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {} + default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) { + // preserve the behavior prior to Ratis 3.4.0. + if (eventStatus == EventStatus.COMPLETED) { + notifyServerShutdown(roleInfo, allServers); + } + } } /** diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java index 385d338335..41f2fd778b 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java @@ -169,7 +169,6 @@ public void rejected(long term, ResultAndTerm result) throws IOException { @Override public void shutdown() { server.close(); - server.getStateMachine().event().notifyServerShutdown(server.getRoleInfoProto(), false); } @Override diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java index 3b568e6d61..9185e20e50 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java @@ -241,7 +241,8 @@ public long[] getFollowerMatchIndices() { // Disallow appendEntries before start() complete; otherwise, it could fail with illegal lifeCycle transition private final AtomicBoolean startComplete = new AtomicBoolean(false); private final AtomicBoolean firstElectionSinceStartup = new AtomicBoolean(true); - private final CompletableFuture closeFuture = new CompletableFuture<>(); + private final CompletableFuture closeFuture = new CompletableFuture<>(); + private final AtomicBoolean shutdownInitiatedNotified = new AtomicBoolean(); private final TransferLeadership transferLeadership; private final SnapshotManagementRequestHandler snapshotRequestHandler; @@ -299,10 +300,12 @@ public long[] getFollowerMatchIndices() { id + "-client"); this.threadGroup = new ThreadGroup(proxy.getThreadGroup(), getMemberId().toString()); - this.closeFuture.whenComplete((v, e) -> { + this.closeFuture.whenComplete((allServers, e) -> { // A division closed before its role is initialized has no role info to report. if (role.getCurrentRole() != null) { - stateMachine.event().notifyServerShutdown(getRoleInfoProto(), false); + // allServers is null when the close has failed. + stateMachine.event().notifyServerShutdown(getRoleInfoProto(), Boolean.TRUE.equals(allServers), + StateMachine.EventStatus.COMPLETED); } }); } @@ -486,8 +489,7 @@ void groupRemove(boolean deleteDirectory, boolean renameDirectory) { closeFuture.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - LOG.warn("{}: Waiting closing interrupted, will not continue to remove group locally", getMemberId()); - return; + throw new CompletionException(getMemberId() + ": Interrupted groupRemove", e); } catch (ExecutionException e) { throw new CompletionException(getMemberId() + ": Failed to close this division", e); } @@ -525,32 +527,38 @@ void groupRemove(boolean deleteDirectory, boolean renameDirectory) { @Override public void close() { - close(null); + close(null, false); } - void close(Throwable cause) { + void close(Throwable cause, boolean allServers) { + // A division closed before its role is initialized has no role info to report. + if (role.getCurrentRole() != null && shutdownInitiatedNotified.compareAndSet(false, true)) { + getStateMachine().event().notifyServerShutdown(getRoleInfoProto(), allServers, + StateMachine.EventStatus.INITIATED); + } + final State newState = lifeCycle.transition(current -> current.isClosingOrClosed() ? current : current == State.NEW ? State.CLOSED : State.CLOSING); if (newState == State.CLOSED) { - completeClose(cause); + completeClose(cause, allServers); } if (newState != State.CLOSING) { return; } if (cause == null) { - closeImpl(null); + closeImpl(null, allServers); } else { // Close on failure: The caller may hold the server lock or run on an executor that closeImpl must stop Daemon.newBuilder() .setName(getMemberId() + "-close") .setThreadGroup(threadGroup) - .setRunnable(() -> closeImpl(cause)) + .setRunnable(() -> closeImpl(cause, allServers)) .build() .start(); } } - private void closeImpl(Throwable cause) { + private void closeImpl(Throwable cause, boolean allServers) { Throwable failure = cause; try { if (cause != null) { @@ -601,17 +609,21 @@ private void closeImpl(Throwable cause) { LOG.warn("{}: Failed to shutdown serverExecutor", getMemberId(), e); } } catch (RuntimeException | Error e) { - failure = e; + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } throw e; } finally { - completeClose(failure); + completeClose(failure, allServers); } } - private void completeClose(Throwable cause) { + private void completeClose(Throwable cause, boolean allServers) { lifeCycle.transitionIfNotEqual(State.CLOSED); if (cause == null) { - closeFuture.complete(null); + closeFuture.complete(allServers); } else { closeFuture.completeExceptionally(cause); } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java index cff1c3d5a9..a380d43199 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java @@ -139,11 +139,10 @@ private void close(RaftGroupId groupId, CompletableFuture future return; } try { - impl.close(); + impl.close(null, true); } catch (Throwable t) { LOG.warn("{}: Failed to close the division for {}", getId(), groupId, t); } - impl.getStateMachine().event().notifyServerShutdown(impl.getRoleInfoProto(), true); } synchronized List getGroupIds() { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java index f96add5f7b..91aaf1b4c2 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java @@ -271,7 +271,7 @@ void persistMetadata() throws IOException { try { getLog().persistMetadata(RaftStorageMetadata.valueOf(currentTerm.get(), votedFor)); } catch (IOException | RuntimeException e) { - server.close(e); + server.close(e, false); throw e; } } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java index fc00b70bab..02e147fc1f 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java @@ -28,7 +28,9 @@ import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.TransactionContext; +import org.apache.ratis.util.JavaUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +57,19 @@ protected static class StateMachineWithConditionalWait extends public static Map>> futures = new ConcurrentHashMap<>(); public static Map numTxns = new ConcurrentHashMap<>(); private final Map appliedTxns = new ConcurrentHashMap<>(); + private final CompletableFuture shutdownInitiated = new CompletableFuture<>(); + private final CompletableFuture shutdownCompleted = new CompletableFuture<>(); + private final AtomicLong numShutdownInitiated = new AtomicLong(); + + @Override + public void notifyServerShutdown(RaftProtos.RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) { + if (eventStatus == EventStatus.INITIATED) { + numShutdownInitiated.incrementAndGet(); + shutdownInitiated.complete(allServers); + } else { + shutdownCompleted.complete(null); + } + } private synchronized void updateTxns() { long appliedIndex = this.getLastAppliedTermIndex().getIndex() + 1; @@ -119,6 +134,13 @@ public void unblockAllTxns() { } } + /** {@link StateMachineWithConditionalWait} counts transactions in static maps keyed by the (reused) peer ids. */ + @BeforeEach + public void resetTxnCounts() { + StateMachineWithConditionalWait.futures.clear(); + StateMachineWithConditionalWait.numTxns.clear(); + } + @Test public void testStateMachineShutdownWaitsForApplyTxn() throws Exception { final RaftProperties prop = getProperties(); @@ -184,4 +206,60 @@ public void testStateMachineShutdownWaitsForApplyTxn() throws Exception { cluster.shutdown(); } } + + /** + * {@link StateMachine.EventStatus#COMPLETED} is notified only after the shutdown has waited for the committed + * entries to be applied, which is unbounded in time. {@link StateMachine.EventStatus#INITIATED} must instead be + * notified before that wait, so that an application embedding a server can react without waiting for it. + */ + @Test + public void testNotifyShutdownInitiatedBeforeWaitingForApplyTxn() throws Exception { + final RaftProperties prop = getProperties(); + prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, + StateMachineWithConditionalWait.class, StateMachine.class); + final MiniRaftCluster cluster = newCluster(3); + cluster.start(); + RaftTestUtil.waitForLeader(cluster); + final RaftServer.Division leader = cluster.getLeader(); + + // Unblock the leader and the first follower so that entries can be committed. + ((StateMachineWithConditionalWait) leader.getStateMachine()).unblockAllTxns(); + ((StateMachineWithConditionalWait) cluster.getFollowers().get(0).getStateMachine()).unblockAllTxns(); + cluster.getLeaderAndSendFirstMessage(true); + + try (final RaftClient client = cluster.createClient(leader.getId())) { + client.io().send(new RaftTestUtil.SimpleMessage("message")); + final long logIndex = client.io().send(new RaftTestUtil.SimpleMessage("message2")).getLogIndex(); + client.io().watch(logIndex, RaftProtos.ReplicationLevel.ALL_COMMITTED); + + // The second follower is still blocked in applyTransaction. + final RaftServer.Division secondFollower = cluster.getFollowers().get(1); + final StateMachineWithConditionalWait sm = (StateMachineWithConditionalWait) secondFollower.getStateMachine(); + // The shutdown only waits for the entries committed locally, which may lag behind the watch reply. + JavaUtils.attemptUntilTrue(() -> secondFollower.getRaftLog().getLastCommittedIndex() >= logIndex, + 50, HUNDRED_MILLIS, "wait for " + secondFollower.getId() + " to commit " + logIndex, LOG); + Assertions.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex); + + final CompletableFuture killFuture = + CompletableFuture.runAsync(() -> cluster.killServer(secondFollower.getId())); + + try { + // The shutdown-initiated notification must arrive while applyTransaction is still blocked. + Assertions.assertTrue(sm.shutdownInitiated.get( + FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit())); + Assertions.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex); + Assertions.assertFalse(sm.shutdownCompleted.isDone()); + } finally { + // The shutdown completes, and notifies, only once the entries have been applied. + sm.unblockAllTxns(); + } + sm.shutdownCompleted.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()); + killFuture.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()); + Assertions.assertTrue(logIndex <= secondFollower.getInfo().getLastAppliedIndex()); + // The division is notified exactly once, even though more than one shutdown path may notify it. + Assertions.assertEquals(1, sm.numShutdownInitiated.get()); + + cluster.shutdown(); + } + } } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRaftServerMetadataPersistence.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRaftServerMetadataPersistence.java index e5962ef278..994a7310de 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRaftServerMetadataPersistence.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRaftServerMetadataPersistence.java @@ -185,7 +185,7 @@ public void testCloseWithNonIOException() throws Exception { stateMachine, false)) { follower.start(); final IllegalStateException failure = new IllegalStateException("Unexpected division failure"); - follower.close(failure); + follower.close(failure, false); assertStopped(follower, group, stateMachine); final ExecutionException thrown = Assertions.assertThrows(ExecutionException.class, () -> closeFuture(follower).get(5, TimeUnit.SECONDS)); @@ -203,9 +203,11 @@ public void testNormalCloseCompletesFutureAfterClosed() throws Exception { final AtomicReference stateWhenNotified = new AtomicReference<>(); final ShutdownStateMachine stateMachine = new ShutdownStateMachine() { @Override - public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) { - stateWhenNotified.set(server.get().getInfo().getLifeCycleState()); - super.notifyServerShutdown(roleInfo, allServer); + public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) { + if (eventStatus == EventStatus.COMPLETED) { + stateWhenNotified.set(server.get().getInfo().getLifeCycleState()); + } + super.notifyServerShutdown(roleInfo, allServers, eventStatus); } }; @@ -215,7 +217,7 @@ public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) { follower.start(); follower.close(); Assertions.assertTrue(closeFuture(follower).isDone()); - Assertions.assertNull(closeFuture(follower).get(5, TimeUnit.SECONDS)); + Assertions.assertFalse(closeFuture(follower).get(5, TimeUnit.SECONDS)); Assertions.assertEquals(LifeCycle.State.CLOSED, stateWhenNotified.get()); assertStopped(follower, group, stateMachine); } @@ -233,7 +235,7 @@ public void testCloseBeforeStartCompletesFuture() throws Exception { follower.close(); Assertions.assertEquals(LifeCycle.State.CLOSED, follower.getInfo().getLifeCycleState()); Assertions.assertTrue(closeFuture(follower).isDone()); - Assertions.assertNull(closeFuture(follower).get(5, TimeUnit.SECONDS)); + Assertions.assertFalse(closeFuture(follower).get(5, TimeUnit.SECONDS)); Assertions.assertFalse(stateMachine.shutdown.isDone()); Assertions.assertEquals(0, stateMachine.shutdownCount.get()); } @@ -443,10 +445,13 @@ public void testShutdownCallbackFailureDoesNotReplaceMetadataException() throws final AtomicReference uncaught = new AtomicReference<>(); final ShutdownStateMachine stateMachine = new ShutdownStateMachine() { @Override - public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) { + public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) { + super.notifyServerShutdown(roleInfo, allServers, eventStatus); + if (eventStatus != EventStatus.COMPLETED) { + return; + } callbackThread.set(Thread.currentThread()); Thread.currentThread().setUncaughtExceptionHandler((thread, cause) -> uncaught.set(cause)); - super.notifyServerShutdown(roleInfo, allServer); throw new IllegalStateException("Failed shutdown callback"); } }; @@ -573,16 +578,19 @@ public void close() throws IOException { } @Override - public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) { + public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) { + if (eventStatus != EventStatus.COMPLETED) { + return; + } closedWhenNotified = closed; shutdownCount.incrementAndGet(); - shutdown.complete(allServer); + shutdown.complete(allServers); } } @SuppressWarnings("unchecked") - private static CompletableFuture closeFuture(RaftServerImpl server) { - return (CompletableFuture) RaftTestUtil.getDeclaredField(server, "closeFuture"); + private static CompletableFuture closeFuture(RaftServerImpl server) { + return (CompletableFuture) RaftTestUtil.getDeclaredField(server, "closeFuture"); } @SuppressWarnings("unchecked")