Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,33 @@ default void notifyServerShutdown(RoleInfoProto roleInfo) {
* @param allServer whether all raft servers will be shutdown at this time
*/
default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {}

/**
* Notify the {@link StateMachine} that the shutdown of the server for this division has started.
* @deprecated please use/override {@link #notifyServerShutdownStarted(RoleInfoProto, boolean)} instead
*/
@Deprecated
default void notifyServerShutdownStarted(RoleInfoProto roleInfo) {
notifyServerShutdownStarted(roleInfo, false);
}

/**
* Notify the {@link StateMachine} that the shutdown of either the server for this division
* or all the servers has started.
* <p>
* In contrast to {@link #notifyServerShutdown(RoleInfoProto, boolean)}, which is invoked once the shutdown
* has completed, this method is invoked before the shutdown has closed any resources.
* In particular, it is invoked before waiting for the committed log entries to be applied to this
* {@link StateMachine}, which is unbounded in time. An application embedding a {@link RaftServer} can use
* this notification to react immediately, e.g. to reject incoming requests or to terminate itself,
* instead of waiting for the shutdown to complete.
* <p>
* This method is invoked from the shutdown path, so an implementation must return promptly.
*
* @param roleInfo roleInfo this server
* @param allServer whether all raft servers will be shutdown at this time
*/
default void notifyServerShutdownStarted(RoleInfoProto roleInfo, boolean allServer) {}
Comment thread
amaliujia marked this conversation as resolved.
Resolved
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ public void rejected(long term, ResultAndTerm result) throws IOException {

@Override
public void shutdown() {
server.notifyServerShutdownStarted(false);
server.close();
server.getStateMachine().event().notifyServerShutdown(server.getRoleInfoProto(), false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ public long[] getFollowerMatchIndices() {
private final AtomicBoolean startComplete = new AtomicBoolean(false);
private final AtomicBoolean firstElectionSinceStartup = new AtomicBoolean(true);
private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
private final AtomicBoolean shutdownStartedNotified = new AtomicBoolean();

private final TransferLeadership transferLeadership;
private final SnapshotManagementRequestHandler snapshotRequestHandler;
Expand Down Expand Up @@ -756,6 +757,25 @@ RoleInfoProto getRoleInfoProto() {
return role.buildRoleInfoProto(this);
}

/**
* Notify the {@link StateMachine} that this division is about to be closed, before {@link #close()} starts
* releasing any resources. Note that {@link #close()} waits for the committed entries to be applied,
* which is unbounded in time, so this is the only shutdown notification with a bounded delay.
* <p>
* This method may be invoked from more than one shutdown path; the {@link StateMachine} is notified only once.
*/
void notifyServerShutdownStarted(boolean allServer) {
if (!shutdownStartedNotified.compareAndSet(false, true)) {
return;
}
try {
stateMachine.event().notifyServerShutdownStarted(getRoleInfoProto(), allServer);
} catch (Throwable t) {
LOG.warn("{}: Failed to notify {} that the shutdown has started",
getMemberId(), JavaUtils.getClassSimpleName(stateMachine.getClass()), t);
}
}

synchronized void changeToCandidate(boolean forceStartLeaderElection) {
Preconditions.assertTrue(getInfo().isFollower());
role.shutdownFollowerState();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ public synchronized void close() {
return;
}
isClosed = true;
// Notify all the state machines upfront: the close tasks submitted below each wait for the committed entries
// to be applied, so a task queued behind them could otherwise be delayed for an unbounded time.
map.values().forEach(f -> f.thenAccept(impl -> impl.notifyServerShutdownStarted(true)));
ConcurrentUtils.parallelForEachAsync(map.entrySet(),
entry -> close(entry.getKey(), entry.getValue()),
executor.get());
Expand All @@ -138,6 +141,7 @@ private void close(RaftGroupId groupId, CompletableFuture<RaftServerImpl> future
LOG.warn("{}: Failed to join the division for {}", getId(), groupId, t);
return;
}
impl.notifyServerShutdownStarted(true);
try {
impl.close();
} catch (Throwable t) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@
import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing;
import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.util.JavaUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -55,6 +57,20 @@ protected static class StateMachineWithConditionalWait extends
public static Map<Long, Set<CompletableFuture<Message>>> futures = new ConcurrentHashMap<>();
public static Map<RaftPeerId, AtomicLong> numTxns = new ConcurrentHashMap<>();
private final Map<Long, Long> appliedTxns = new ConcurrentHashMap<>();
private final CompletableFuture<Boolean> shutdownStarted = new CompletableFuture<>();
private final CompletableFuture<Void> shutdownCompleted = new CompletableFuture<>();
private final AtomicLong numShutdownStarted = new AtomicLong();

@Override
public void notifyServerShutdownStarted(RaftProtos.RoleInfoProto roleInfo, boolean allServer) {
numShutdownStarted.incrementAndGet();
shutdownStarted.complete(allServer);
}

@Override
public void notifyServerShutdown(RaftProtos.RoleInfoProto roleInfo, boolean allServer) {
shutdownCompleted.complete(null);
}

private synchronized void updateTxns() {
long appliedIndex = this.getLastAppliedTermIndex().getIndex() + 1;
Expand Down Expand Up @@ -119,6 +135,13 @@ public void unblockAllTxns() {
}
}

/** {@link StateMachineWithConditionalWait} counts transactions in static maps keyed by the (reused) peer ids. */
@BeforeEach
public void resetTxnCounts() {
StateMachineWithConditionalWait.futures.clear();
StateMachineWithConditionalWait.numTxns.clear();
}

@Test
public void testStateMachineShutdownWaitsForApplyTxn() throws Exception {
final RaftProperties prop = getProperties();
Expand Down Expand Up @@ -184,4 +207,61 @@ public void testStateMachineShutdownWaitsForApplyTxn() throws Exception {
cluster.shutdown();
}
}

/**
* {@link StateMachine.EventApi#notifyServerShutdown(RaftProtos.RoleInfoProto, boolean)} is invoked only after
* the shutdown has waited for the committed entries to be applied, which is unbounded in time.
* {@link StateMachine.EventApi#notifyServerShutdownStarted(RaftProtos.RoleInfoProto, boolean)} must instead be
* invoked before that wait, so that an application embedding a server can react without waiting for it.
*/
@Test
public void testNotifyShutdownStartedBeforeWaitingForApplyTxn() throws Exception {
final RaftProperties prop = getProperties();
prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
StateMachineWithConditionalWait.class, StateMachine.class);
final MiniRaftCluster cluster = newCluster(3);
cluster.start();
RaftTestUtil.waitForLeader(cluster);
final RaftServer.Division leader = cluster.getLeader();

// Unblock the leader and the first follower so that entries can be committed.
((StateMachineWithConditionalWait) leader.getStateMachine()).unblockAllTxns();
((StateMachineWithConditionalWait) cluster.getFollowers().get(0).getStateMachine()).unblockAllTxns();
cluster.getLeaderAndSendFirstMessage(true);

try (final RaftClient client = cluster.createClient(leader.getId())) {
client.io().send(new RaftTestUtil.SimpleMessage("message"));
final long logIndex = client.io().send(new RaftTestUtil.SimpleMessage("message2")).getLogIndex();
client.io().watch(logIndex, RaftProtos.ReplicationLevel.ALL_COMMITTED);

// The second follower is still blocked in applyTransaction.
final RaftServer.Division secondFollower = cluster.getFollowers().get(1);
final StateMachineWithConditionalWait sm = (StateMachineWithConditionalWait) secondFollower.getStateMachine();
// The shutdown only waits for the entries committed locally, which may lag behind the watch reply.
JavaUtils.attemptUntilTrue(() -> secondFollower.getRaftLog().getLastCommittedIndex() >= logIndex,
50, HUNDRED_MILLIS, "wait for " + secondFollower.getId() + " to commit " + logIndex, LOG);
Assertions.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex);

final CompletableFuture<Void> killFuture =
CompletableFuture.runAsync(() -> cluster.killServer(secondFollower.getId()));

try {
// The shutdown-started notification must arrive while applyTransaction is still blocked.
Assertions.assertTrue(sm.shutdownStarted.get(
FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()));
Assertions.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex);
Assertions.assertFalse(sm.shutdownCompleted.isDone());
} finally {
// The shutdown completes, and notifies, only once the entries have been applied.
sm.unblockAllTxns();
}
sm.shutdownCompleted.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit());
killFuture.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit());
Assertions.assertTrue(logIndex <= secondFollower.getInfo().getLastAppliedIndex());
// The division is notified exactly once, even though more than one shutdown path may notify it.
Assertions.assertEquals(1, sm.numShutdownStarted.get());

cluster.shutdown();
}
}
}
Loading