Skip to content

Commit 020fabd

Browse files
authored
RATIS-2721. Notify the StateMachine when a server shutdown starts (#1647)
1 parent fdd51ed commit 020fabd

7 files changed

Lines changed: 161 additions & 38 deletions

File tree

‎ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@
5858
public interface StateMachine extends Closeable {
5959
Logger LOG = LoggerFactory.getLogger(StateMachine.class);
6060

61+
enum EventStatus {
62+
INITIATED,
63+
COMPLETED
64+
}
65+
6166
/** A registry to support different state machines in multi-raft environment. */
6267
interface Registry extends Function<RaftGroupId, StateMachine> {
6368
}
@@ -229,20 +234,42 @@ default void notifyLogFailed(Throwable cause, LogEntryProto failedEntry) {}
229234
default void notifySnapshotInstalled(InstallSnapshotResult result, long snapshotIndex, RaftPeer peer) {}
230235

231236
/**
232-
* Notify the {@link StateMachine} that the server for this division has been shut down.
233-
* @deprecated please use/override {@link #notifyServerShutdown(RoleInfoProto, boolean)} instead
237+
* The same as notifyServerShutdown(roleInfo, false, EventStatus.COMPLETED).
238+
*
239+
* @deprecated please use/override {@link #notifyServerShutdown(RoleInfoProto, boolean, EventStatus)} instead
234240
*/
235241
@Deprecated
236-
default void notifyServerShutdown(RoleInfoProto roleInfo) {
237-
notifyServerShutdown(roleInfo, false);
238-
}
242+
default void notifyServerShutdown(RoleInfoProto roleInfo) {}
239243

240244
/**
241245
* Notify the {@link StateMachine} that either the server for this division or all the servers have been shut down.
246+
*
247+
* @deprecated please override {@link #notifyServerShutdown(RoleInfoProto, boolean, EventStatus)} instead
248+
*/
249+
@Deprecated
250+
default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers) {
251+
notifyServerShutdown(roleInfo);
252+
}
253+
254+
/**
255+
* Notify the {@link StateMachine} server shutdown.
256+
* <p>
257+
* {@link EventStatus#INITIATED} is notified before the shutdown has closed any resources,
258+
* in particular, before waiting for the committed log entries to be applied to this {@link StateMachine},
259+
* which is unbounded in time. An implementation must return promptly for {@link EventStatus#INITIATED}.
260+
* <p>
261+
* {@link EventStatus#COMPLETED} is notified after the shutdown has completed.
262+
*
242263
* @param roleInfo roleInfo this server
243-
* @param allServer whether all raft servers will be shutdown at this time
264+
* @param allServers whether all raft servers will be shutdown at this time
265+
* @param eventStatus {@link EventStatus#INITIATED} or {@link EventStatus#COMPLETED}
244266
*/
245-
default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {}
267+
default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) {
268+
// preserve the behavior prior to Ratis 3.4.0.
269+
if (eventStatus == EventStatus.COMPLETED) {
270+
notifyServerShutdown(roleInfo, allServers);
271+
}
272+
}
246273
}
247274

248275
/**

‎ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,6 @@ public void rejected(long term, ResultAndTerm result) throws IOException {
181181
@Override
182182
public void shutdown() {
183183
server.close();
184-
server.getStateMachine().event().notifyServerShutdown(server.getRoleInfoProto(), false);
185184
}
186185

187186
@Override

‎ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,8 @@ public long[] getFollowerMatchIndices() {
229229
// Disallow appendEntries before start() complete; otherwise, it could fail with illegal lifeCycle transition
230230
private final AtomicBoolean startComplete = new AtomicBoolean(false);
231231
private final AtomicBoolean firstElectionSinceStartup = new AtomicBoolean(true);
232-
private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
232+
private final CompletableFuture<Boolean> closeFuture = new CompletableFuture<>();
233+
private final AtomicBoolean shutdownInitiatedNotified = new AtomicBoolean();
233234

234235
private final TransferLeadership transferLeadership;
235236
private final SnapshotManagementRequestHandler snapshotRequestHandler;
@@ -287,10 +288,12 @@ public long[] getFollowerMatchIndices() {
287288
id + "-client");
288289
this.threadGroup = new ThreadGroup(proxy.getThreadGroup(), getMemberId().toString());
289290

290-
this.closeFuture.whenComplete((v, e) -> {
291+
this.closeFuture.whenComplete((allServers, e) -> {
291292
// A division closed before its role is initialized has no role info to report.
292293
if (role.getCurrentRole() != null) {
293-
stateMachine.event().notifyServerShutdown(getRoleInfoProto(), false);
294+
// allServers is null when the close has failed.
295+
stateMachine.event().notifyServerShutdown(getRoleInfoProto(), Boolean.TRUE.equals(allServers),
296+
StateMachine.EventStatus.COMPLETED);
294297
}
295298
});
296299
}
@@ -474,8 +477,7 @@ void groupRemove(boolean deleteDirectory, boolean renameDirectory) {
474477
closeFuture.get();
475478
} catch (InterruptedException e) {
476479
Thread.currentThread().interrupt();
477-
LOG.warn("{}: Waiting closing interrupted, will not continue to remove group locally", getMemberId());
478-
return;
480+
throw new CompletionException(getMemberId() + ": Interrupted groupRemove", e);
479481
} catch (ExecutionException e) {
480482
throw new CompletionException(getMemberId() + ": Failed to close this division", e);
481483
}
@@ -513,32 +515,38 @@ void groupRemove(boolean deleteDirectory, boolean renameDirectory) {
513515

514516
@Override
515517
public void close() {
516-
close(null);
518+
close(null, false);
517519
}
518520

519-
void close(Throwable cause) {
521+
void close(Throwable cause, boolean allServers) {
522+
// A division closed before its role is initialized has no role info to report.
523+
if (role.getCurrentRole() != null && shutdownInitiatedNotified.compareAndSet(false, true)) {
524+
getStateMachine().event().notifyServerShutdown(getRoleInfoProto(), allServers,
525+
StateMachine.EventStatus.INITIATED);
526+
}
527+
520528
final State newState = lifeCycle.transition(current -> current.isClosingOrClosed() ? current
521529
: current == State.NEW ? State.CLOSED : State.CLOSING);
522530
if (newState == State.CLOSED) {
523-
completeClose(cause);
531+
completeClose(cause, allServers);
524532
}
525533
if (newState != State.CLOSING) {
526534
return;
527535
}
528536
if (cause == null) {
529-
closeImpl(null);
537+
closeImpl(null, allServers);
530538
} else {
531539
// Close on failure: The caller may hold the server lock or run on an executor that closeImpl must stop
532540
Daemon.newBuilder()
533541
.setName(getMemberId() + "-close")
534542
.setThreadGroup(threadGroup)
535-
.setRunnable(() -> closeImpl(cause))
543+
.setRunnable(() -> closeImpl(cause, allServers))
536544
.build()
537545
.start();
538546
}
539547
}
540548

541-
private void closeImpl(Throwable cause) {
549+
private void closeImpl(Throwable cause, boolean allServers) {
542550
Throwable failure = cause;
543551
try {
544552
if (cause != null) {
@@ -589,17 +597,21 @@ private void closeImpl(Throwable cause) {
589597
LOG.warn("{}: Failed to shutdown serverExecutor", getMemberId(), e);
590598
}
591599
} catch (RuntimeException | Error e) {
592-
failure = e;
600+
if (failure == null) {
601+
failure = e;
602+
} else {
603+
failure.addSuppressed(e);
604+
}
593605
throw e;
594606
} finally {
595-
completeClose(failure);
607+
completeClose(failure, allServers);
596608
}
597609
}
598610

599-
private void completeClose(Throwable cause) {
611+
private void completeClose(Throwable cause, boolean allServers) {
600612
lifeCycle.transitionIfNotEqual(State.CLOSED);
601613
if (cause == null) {
602-
closeFuture.complete(null);
614+
closeFuture.complete(allServers);
603615
} else {
604616
closeFuture.completeExceptionally(cause);
605617
}

‎ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,11 +139,10 @@ private void close(RaftGroupId groupId, CompletableFuture<RaftServerImpl> future
139139
return;
140140
}
141141
try {
142-
impl.close();
142+
impl.close(null, true);
143143
} catch (Throwable t) {
144144
LOG.warn("{}: Failed to close the division for {}", getId(), groupId, t);
145145
}
146-
impl.getStateMachine().event().notifyServerShutdown(impl.getRoleInfoProto(), true);
147146
}
148147

149148
synchronized List<RaftGroupId> getGroupIds() {

‎ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ void persistMetadata() throws IOException {
271271
try {
272272
getLog().persistMetadata(RaftStorageMetadata.valueOf(currentTerm.get(), votedFor));
273273
} catch (IOException | RuntimeException e) {
274-
server.close(e);
274+
server.close(e, false);
275275
throw e;
276276
}
277277
}

‎ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing;
2929
import org.apache.ratis.statemachine.StateMachine;
3030
import org.apache.ratis.statemachine.TransactionContext;
31+
import org.apache.ratis.util.JavaUtils;
3132
import org.junit.jupiter.api.Assertions;
33+
import org.junit.jupiter.api.BeforeEach;
3234
import org.junit.jupiter.api.Test;
3335
import org.slf4j.Logger;
3436
import org.slf4j.LoggerFactory;
@@ -55,6 +57,19 @@ protected static class StateMachineWithConditionalWait extends
5557
public static Map<Long, Set<CompletableFuture<Message>>> futures = new ConcurrentHashMap<>();
5658
public static Map<RaftPeerId, AtomicLong> numTxns = new ConcurrentHashMap<>();
5759
private final Map<Long, Long> appliedTxns = new ConcurrentHashMap<>();
60+
private final CompletableFuture<Boolean> shutdownInitiated = new CompletableFuture<>();
61+
private final CompletableFuture<Void> shutdownCompleted = new CompletableFuture<>();
62+
private final AtomicLong numShutdownInitiated = new AtomicLong();
63+
64+
@Override
65+
public void notifyServerShutdown(RaftProtos.RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) {
66+
if (eventStatus == EventStatus.INITIATED) {
67+
numShutdownInitiated.incrementAndGet();
68+
shutdownInitiated.complete(allServers);
69+
} else {
70+
shutdownCompleted.complete(null);
71+
}
72+
}
5873

5974
private synchronized void updateTxns() {
6075
long appliedIndex = this.getLastAppliedTermIndex().getIndex() + 1;
@@ -119,6 +134,13 @@ public void unblockAllTxns() {
119134
}
120135
}
121136

137+
/** {@link StateMachineWithConditionalWait} counts transactions in static maps keyed by the (reused) peer ids. */
138+
@BeforeEach
139+
public void resetTxnCounts() {
140+
StateMachineWithConditionalWait.futures.clear();
141+
StateMachineWithConditionalWait.numTxns.clear();
142+
}
143+
122144
@Test
123145
public void testStateMachineShutdownWaitsForApplyTxn() throws Exception {
124146
final RaftProperties prop = getProperties();
@@ -184,4 +206,60 @@ public void testStateMachineShutdownWaitsForApplyTxn() throws Exception {
184206
cluster.shutdown();
185207
}
186208
}
209+
210+
/**
211+
* {@link StateMachine.EventStatus#COMPLETED} is notified only after the shutdown has waited for the committed
212+
* entries to be applied, which is unbounded in time. {@link StateMachine.EventStatus#INITIATED} must instead be
213+
* notified before that wait, so that an application embedding a server can react without waiting for it.
214+
*/
215+
@Test
216+
public void testNotifyShutdownInitiatedBeforeWaitingForApplyTxn() throws Exception {
217+
final RaftProperties prop = getProperties();
218+
prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
219+
StateMachineWithConditionalWait.class, StateMachine.class);
220+
final MiniRaftCluster cluster = newCluster(3);
221+
cluster.start();
222+
RaftTestUtil.waitForLeader(cluster);
223+
final RaftServer.Division leader = cluster.getLeader();
224+
225+
// Unblock the leader and the first follower so that entries can be committed.
226+
((StateMachineWithConditionalWait) leader.getStateMachine()).unblockAllTxns();
227+
((StateMachineWithConditionalWait) cluster.getFollowers().get(0).getStateMachine()).unblockAllTxns();
228+
cluster.getLeaderAndSendFirstMessage(true);
229+
230+
try (final RaftClient client = cluster.createClient(leader.getId())) {
231+
client.io().send(new RaftTestUtil.SimpleMessage("message"));
232+
final long logIndex = client.io().send(new RaftTestUtil.SimpleMessage("message2")).getLogIndex();
233+
client.io().watch(logIndex, RaftProtos.ReplicationLevel.ALL_COMMITTED);
234+
235+
// The second follower is still blocked in applyTransaction.
236+
final RaftServer.Division secondFollower = cluster.getFollowers().get(1);
237+
final StateMachineWithConditionalWait sm = (StateMachineWithConditionalWait) secondFollower.getStateMachine();
238+
// The shutdown only waits for the entries committed locally, which may lag behind the watch reply.
239+
JavaUtils.attemptUntilTrue(() -> secondFollower.getRaftLog().getLastCommittedIndex() >= logIndex,
240+
50, HUNDRED_MILLIS, "wait for " + secondFollower.getId() + " to commit " + logIndex, LOG);
241+
Assertions.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex);
242+
243+
final CompletableFuture<Void> killFuture =
244+
CompletableFuture.runAsync(() -> cluster.killServer(secondFollower.getId()));
245+
246+
try {
247+
// The shutdown-initiated notification must arrive while applyTransaction is still blocked.
248+
Assertions.assertTrue(sm.shutdownInitiated.get(
249+
FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()));
250+
Assertions.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex);
251+
Assertions.assertFalse(sm.shutdownCompleted.isDone());
252+
} finally {
253+
// The shutdown completes, and notifies, only once the entries have been applied.
254+
sm.unblockAllTxns();
255+
}
256+
sm.shutdownCompleted.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit());
257+
killFuture.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit());
258+
Assertions.assertTrue(logIndex <= secondFollower.getInfo().getLastAppliedIndex());
259+
// The division is notified exactly once, even though more than one shutdown path may notify it.
260+
Assertions.assertEquals(1, sm.numShutdownInitiated.get());
261+
262+
cluster.shutdown();
263+
}
264+
}
187265
}

‎ratis-server/src/test/java/org/apache/ratis/server/impl/TestRaftServerMetadataPersistence.java‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ public void testCloseWithNonIOException() throws Exception {
185185
stateMachine, false)) {
186186
follower.start();
187187
final IllegalStateException failure = new IllegalStateException("Unexpected division failure");
188-
follower.close(failure);
188+
follower.close(failure, false);
189189
assertStopped(follower, group, stateMachine);
190190
final ExecutionException thrown = Assertions.assertThrows(ExecutionException.class,
191191
() -> closeFuture(follower).get(5, TimeUnit.SECONDS));
@@ -203,9 +203,11 @@ public void testNormalCloseCompletesFutureAfterClosed() throws Exception {
203203
final AtomicReference<LifeCycle.State> stateWhenNotified = new AtomicReference<>();
204204
final ShutdownStateMachine stateMachine = new ShutdownStateMachine() {
205205
@Override
206-
public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {
207-
stateWhenNotified.set(server.get().getInfo().getLifeCycleState());
208-
super.notifyServerShutdown(roleInfo, allServer);
206+
public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) {
207+
if (eventStatus == EventStatus.COMPLETED) {
208+
stateWhenNotified.set(server.get().getInfo().getLifeCycleState());
209+
}
210+
super.notifyServerShutdown(roleInfo, allServers, eventStatus);
209211
}
210212
};
211213

@@ -215,7 +217,7 @@ public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {
215217
follower.start();
216218
follower.close();
217219
Assertions.assertTrue(closeFuture(follower).isDone());
218-
Assertions.assertNull(closeFuture(follower).get(5, TimeUnit.SECONDS));
220+
Assertions.assertFalse(closeFuture(follower).get(5, TimeUnit.SECONDS));
219221
Assertions.assertEquals(LifeCycle.State.CLOSED, stateWhenNotified.get());
220222
assertStopped(follower, group, stateMachine);
221223
}
@@ -233,7 +235,7 @@ public void testCloseBeforeStartCompletesFuture() throws Exception {
233235
follower.close();
234236
Assertions.assertEquals(LifeCycle.State.CLOSED, follower.getInfo().getLifeCycleState());
235237
Assertions.assertTrue(closeFuture(follower).isDone());
236-
Assertions.assertNull(closeFuture(follower).get(5, TimeUnit.SECONDS));
238+
Assertions.assertFalse(closeFuture(follower).get(5, TimeUnit.SECONDS));
237239
Assertions.assertFalse(stateMachine.shutdown.isDone());
238240
Assertions.assertEquals(0, stateMachine.shutdownCount.get());
239241
}
@@ -443,10 +445,13 @@ public void testShutdownCallbackFailureDoesNotReplaceMetadataException() throws
443445
final AtomicReference<Throwable> uncaught = new AtomicReference<>();
444446
final ShutdownStateMachine stateMachine = new ShutdownStateMachine() {
445447
@Override
446-
public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {
448+
public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) {
449+
super.notifyServerShutdown(roleInfo, allServers, eventStatus);
450+
if (eventStatus != EventStatus.COMPLETED) {
451+
return;
452+
}
447453
callbackThread.set(Thread.currentThread());
448454
Thread.currentThread().setUncaughtExceptionHandler((thread, cause) -> uncaught.set(cause));
449-
super.notifyServerShutdown(roleInfo, allServer);
450455
throw new IllegalStateException("Failed shutdown callback");
451456
}
452457
};
@@ -574,16 +579,19 @@ public void close() throws IOException {
574579
}
575580

576581
@Override
577-
public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {
582+
public void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServers, EventStatus eventStatus) {
583+
if (eventStatus != EventStatus.COMPLETED) {
584+
return;
585+
}
578586
closedWhenNotified = closed;
579587
shutdownCount.incrementAndGet();
580-
shutdown.complete(allServer);
588+
shutdown.complete(allServers);
581589
}
582590
}
583591

584592
@SuppressWarnings("unchecked")
585-
private static CompletableFuture<Void> closeFuture(RaftServerImpl server) {
586-
return (CompletableFuture<Void>) RaftTestUtil.getDeclaredField(server, "closeFuture");
593+
private static CompletableFuture<Boolean> closeFuture(RaftServerImpl server) {
594+
return (CompletableFuture<Boolean>) RaftTestUtil.getDeclaredField(server, "closeFuture");
587595
}
588596

589597
@SuppressWarnings("unchecked")

0 commit comments

Comments
 (0)