Skip to content

Commit 3685cd0

Browse files
committed
Fix RPC hang on abrupt connection disconnect.
Notify pending invoke futures when the channel becomes inactive and return BOLT_SEND_FAILED for connection-closed responses, so in-flight requests fail immediately instead of waiting for RPC timeout. Skip suspect-server tracking when ObServerAddr is unset to avoid NPE in direct load reconnect paths.
1 parent 178e1d4 commit 3685cd0

6 files changed

Lines changed: 56 additions & 5 deletions

File tree

src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObClientFuture.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,9 @@ public boolean isDone() {
116116
*/
117117
@Override
118118
public RemotingCommand createConnectionClosedResponse(InetSocketAddress responseHost) {
119-
return null;
119+
String address = responseHost != null ? responseHost.toString() : "unknown";
120+
return ObTablePacket.createTransportErrorPacket(TransportCodes.BOLT_SEND_FAILED,
121+
"connection {" + address + "} closed", null);
120122
}
121123

122124
/*
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*-
2+
* #%L
3+
* OBKV Table Client Framework
4+
* %%
5+
* Copyright (C) 2021 OceanBase
6+
* %%
7+
* OBKV Table Client Framework is licensed under Mulan PSL v2.
8+
* You can use this software according to the terms and conditions of the Mulan PSL v2.
9+
* You may obtain a copy of Mulan PSL v2 at:
10+
* http://license.coscl.org.cn/MulanPSL2
11+
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+
* See the Mulan PSL v2 for more details.
15+
* #L%
16+
*/
17+
18+
package com.alipay.oceanbase.rpc.bolt.transport;
19+
20+
import com.alipay.remoting.Connection;
21+
import com.alipay.remoting.ConnectionEventHandler;
22+
import com.alipay.remoting.config.switches.GlobalSwitch;
23+
import io.netty.channel.ChannelHandlerContext;
24+
25+
/**
26+
* Notify pending RPC futures immediately when the underlying channel becomes inactive.
27+
* The default {@link ConnectionEventHandler} only calls {@link Connection#onClose()} from
28+
* {@code close()}, which is not always invoked when the peer disconnects abruptly.
29+
*/
30+
public class ObConnectionEventHandler extends ConnectionEventHandler {
31+
32+
public ObConnectionEventHandler(GlobalSwitch globalSwitch) {
33+
super(globalSwitch);
34+
}
35+
36+
@Override
37+
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
38+
Connection connection = ctx.channel().attr(Connection.CONNECTION).get();
39+
if (connection != null) {
40+
connection.onClose();
41+
}
42+
super.channelInactive(ctx);
43+
}
44+
}

src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketFactory.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,8 @@ public ObTablePacket createExceptionResponse(int id, ResponseStatus status, Thro
163163
*/
164164
@Override
165165
public ObTablePacket createConnectionClosedResponse(InetSocketAddress address, String message) {
166-
return null;
166+
String errMsg = message != null ? message : "connection {" + address.toString() + "} closed";
167+
return ObTablePacket.createTransportErrorPacket(TransportCodes.BOLT_SEND_FAILED, errMsg,
168+
null);
167169
}
168170
}

src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableConnection.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ private boolean connect() throws Exception {
118118
MONITOR.info(logMessage(null, "CONNECT", endpoint, System.currentTimeMillis() - start));
119119

120120
if (tries >= maxTryTimes) {
121-
if (!obTable.isOdpMode()) {
121+
if (!obTable.isOdpMode() && obTable.getObServerAddr() != null) {
122122
RouteTableRefresher.SuspectObServer suspectAddr = new RouteTableRefresher.SuspectObServer(
123123
obTable.getObServerAddr());
124124
RouteTableRefresher.addIntoSuspectIPs(suspectAddr);

src/main/java/com/alipay/oceanbase/rpc/location/model/RouteTableRefresher.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ private void checkAlive(ObServerAddr addr) {
193193
}
194194

195195
public static void addIntoSuspectIPs(SuspectObServer server) throws InterruptedException {
196+
if (server == null || server.getAddr() == null) {
197+
return;
198+
}
196199
ObServerAddr addr = server.getAddr();
197200
if (suspectServers.get(addr) != null) {
198201
// already in the list, directly return

src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
import com.alipay.oceanbase.rpc.table.api.TableBatchOps;
3434
import com.alipay.oceanbase.rpc.table.api.TableQuery;
3535
import com.alipay.oceanbase.rpc.util.TraceUtil;
36-
import com.alipay.remoting.ConnectionEventHandler;
36+
import com.alipay.oceanbase.rpc.bolt.transport.ObConnectionEventHandler;
3737
import com.alipay.remoting.config.switches.GlobalSwitch;
3838
import com.alipay.remoting.connection.ConnectionFactory;
3939
import com.alipay.remoting.exception.RemotingException;
@@ -101,7 +101,7 @@ public void init() throws Exception {
101101
.newBuilder()
102102
.configWriteBufferWaterMark(getNettyBufferLowWatermark(),
103103
getNettyBufferHighWatermark()).build();
104-
connectionFactory.init(new ConnectionEventHandler(new GlobalSwitch())); // Only for monitoring connection status
104+
connectionFactory.init(new ObConnectionEventHandler(new GlobalSwitch()));
105105
realClient = new ObTableRemoting(new ObPacketFactory(enableRerouting));
106106
connectionPool = new ObTableConnectionPool(this, obTableConnectionPoolSize);
107107
connectionPool.init();

0 commit comments

Comments
 (0)