2828import org .apache .ratis .statemachine .impl .SimpleStateMachine4Testing ;
2929import org .apache .ratis .statemachine .StateMachine ;
3030import org .apache .ratis .statemachine .TransactionContext ;
31+ import org .apache .ratis .util .JavaUtils ;
3132import org .junit .jupiter .api .Assertions ;
33+ import org .junit .jupiter .api .BeforeEach ;
3234import org .junit .jupiter .api .Test ;
3335import org .slf4j .Logger ;
3436import org .slf4j .LoggerFactory ;
@@ -55,6 +57,19 @@ protected static class StateMachineWithConditionalWait extends
5557 public static Map <Long , Set <CompletableFuture <Message >>> futures = new ConcurrentHashMap <>();
5658 public static Map <RaftPeerId , AtomicLong > numTxns = new ConcurrentHashMap <>();
5759 private final Map <Long , Long > appliedTxns = new ConcurrentHashMap <>();
60+ private final CompletableFuture <Boolean > shutdownInitiated = new CompletableFuture <>();
61+ private final CompletableFuture <Void > shutdownCompleted = new CompletableFuture <>();
62+ private final AtomicLong numShutdownInitiated = new AtomicLong ();
63+
64+ @ Override
65+ public void notifyServerShutdown (RaftProtos .RoleInfoProto roleInfo , boolean allServers , EventStatus eventStatus ) {
66+ if (eventStatus == EventStatus .INITIATED ) {
67+ numShutdownInitiated .incrementAndGet ();
68+ shutdownInitiated .complete (allServers );
69+ } else {
70+ shutdownCompleted .complete (null );
71+ }
72+ }
5873
5974 private synchronized void updateTxns () {
6075 long appliedIndex = this .getLastAppliedTermIndex ().getIndex () + 1 ;
@@ -119,6 +134,13 @@ public void unblockAllTxns() {
119134 }
120135 }
121136
137+ /** {@link StateMachineWithConditionalWait} counts transactions in static maps keyed by the (reused) peer ids. */
138+ @ BeforeEach
139+ public void resetTxnCounts () {
140+ StateMachineWithConditionalWait .futures .clear ();
141+ StateMachineWithConditionalWait .numTxns .clear ();
142+ }
143+
122144 @ Test
123145 public void testStateMachineShutdownWaitsForApplyTxn () throws Exception {
124146 final RaftProperties prop = getProperties ();
@@ -184,4 +206,60 @@ public void testStateMachineShutdownWaitsForApplyTxn() throws Exception {
184206 cluster .shutdown ();
185207 }
186208 }
209+
210+ /**
211+ * {@link StateMachine.EventStatus#COMPLETED} is notified only after the shutdown has waited for the committed
212+ * entries to be applied, which is unbounded in time. {@link StateMachine.EventStatus#INITIATED} must instead be
213+ * notified before that wait, so that an application embedding a server can react without waiting for it.
214+ */
215+ @ Test
216+ public void testNotifyShutdownInitiatedBeforeWaitingForApplyTxn () throws Exception {
217+ final RaftProperties prop = getProperties ();
218+ prop .setClass (MiniRaftCluster .STATEMACHINE_CLASS_KEY ,
219+ StateMachineWithConditionalWait .class , StateMachine .class );
220+ final MiniRaftCluster cluster = newCluster (3 );
221+ cluster .start ();
222+ RaftTestUtil .waitForLeader (cluster );
223+ final RaftServer .Division leader = cluster .getLeader ();
224+
225+ // Unblock the leader and the first follower so that entries can be committed.
226+ ((StateMachineWithConditionalWait ) leader .getStateMachine ()).unblockAllTxns ();
227+ ((StateMachineWithConditionalWait ) cluster .getFollowers ().get (0 ).getStateMachine ()).unblockAllTxns ();
228+ cluster .getLeaderAndSendFirstMessage (true );
229+
230+ try (final RaftClient client = cluster .createClient (leader .getId ())) {
231+ client .io ().send (new RaftTestUtil .SimpleMessage ("message" ));
232+ final long logIndex = client .io ().send (new RaftTestUtil .SimpleMessage ("message2" )).getLogIndex ();
233+ client .io ().watch (logIndex , RaftProtos .ReplicationLevel .ALL_COMMITTED );
234+
235+ // The second follower is still blocked in applyTransaction.
236+ final RaftServer .Division secondFollower = cluster .getFollowers ().get (1 );
237+ final StateMachineWithConditionalWait sm = (StateMachineWithConditionalWait ) secondFollower .getStateMachine ();
238+ // The shutdown only waits for the entries committed locally, which may lag behind the watch reply.
239+ JavaUtils .attemptUntilTrue (() -> secondFollower .getRaftLog ().getLastCommittedIndex () >= logIndex ,
240+ 50 , HUNDRED_MILLIS , "wait for " + secondFollower .getId () + " to commit " + logIndex , LOG );
241+ Assertions .assertTrue (secondFollower .getInfo ().getLastAppliedIndex () < logIndex );
242+
243+ final CompletableFuture <Void > killFuture =
244+ CompletableFuture .runAsync (() -> cluster .killServer (secondFollower .getId ()));
245+
246+ try {
247+ // The shutdown-initiated notification must arrive while applyTransaction is still blocked.
248+ Assertions .assertTrue (sm .shutdownInitiated .get (
249+ FIVE_SECONDS .getDuration (), FIVE_SECONDS .getUnit ()));
250+ Assertions .assertTrue (secondFollower .getInfo ().getLastAppliedIndex () < logIndex );
251+ Assertions .assertFalse (sm .shutdownCompleted .isDone ());
252+ } finally {
253+ // The shutdown completes, and notifies, only once the entries have been applied.
254+ sm .unblockAllTxns ();
255+ }
256+ sm .shutdownCompleted .get (FIVE_SECONDS .getDuration (), FIVE_SECONDS .getUnit ());
257+ killFuture .get (FIVE_SECONDS .getDuration (), FIVE_SECONDS .getUnit ());
258+ Assertions .assertTrue (logIndex <= secondFollower .getInfo ().getLastAppliedIndex ());
259+ // The division is notified exactly once, even though more than one shutdown path may notify it.
260+ Assertions .assertEquals (1 , sm .numShutdownInitiated .get ());
261+
262+ cluster .shutdown ();
263+ }
264+ }
187265}
0 commit comments