From 3c9cec1d44491daf2abcc860a1efe9666e9fe6fb Mon Sep 17 00:00:00 2001 From: root Date: Wed, 5 Aug 2015 01:11:49 +0300 Subject: [PATCH] Add SO_REUSEPORT support to socket Introduce new feature for Twemproxy running on newer kernels. With this feature you are able to do upgrades, restarts without any downtime. Kernel does load balancing between processes with the same host:port pairs. Also, helps running on more than single core. Signed-off-by: Donatas Abraitis --- src/nc_conf.c | 10 ++++++++++ src/nc_conf.h | 2 ++ src/nc_proxy.c | 12 +++++++++++- src/nc_server.h | 1 + src/nc_stats.c | 2 +- src/nc_util.c | 14 ++++++++++++++ src/nc_util.h | 1 + 7 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/nc_conf.c b/src/nc_conf.c index fa720796..bcb9910f 100644 --- a/src/nc_conf.c +++ b/src/nc_conf.c @@ -78,6 +78,10 @@ static struct command conf_commands[] = { conf_set_bool, offsetof(struct conf_pool, tcpkeepalive) }, + { string("reuseport"), + conf_set_bool, + offsetof(struct conf_pool, reuseport) }, + { string("redis_auth"), conf_set_string, offsetof(struct conf_pool, redis_auth) }, @@ -199,6 +203,7 @@ conf_pool_init(struct conf_pool *cp, struct string *name) cp->redis = CONF_UNSET_NUM; cp->tcpkeepalive = CONF_UNSET_NUM; + cp->reuseport = CONF_UNSET_NUM; cp->redis_db = CONF_UNSET_NUM; cp->preconnect = CONF_UNSET_NUM; cp->auto_eject_hosts = CONF_UNSET_NUM; @@ -287,6 +292,7 @@ conf_pool_each_transform(void *elem, void *data) sp->hash_tag = cp->hash_tag; sp->tcpkeepalive = cp->tcpkeepalive ? 1 : 0; + sp->reuseport = cp->reuseport ? 1 : 0; sp->redis = cp->redis ? 1 : 0; sp->timeout = cp->timeout; @@ -1242,6 +1248,10 @@ conf_validate_pool(struct conf *cf, struct conf_pool *cp) cp->tcpkeepalive = CONF_DEFAULT_TCPKEEPALIVE; } + if (cp->reuseport == CONF_UNSET_NUM) { + cp->reuseport = CONF_DEFAULT_REUSEPORT; + } + if (cp->redis_db == CONF_UNSET_NUM) { cp->redis_db = CONF_DEFAULT_REDIS_DB; } diff --git a/src/nc_conf.h b/src/nc_conf.h index 296afc46..ae96faea 100644 --- a/src/nc_conf.h +++ b/src/nc_conf.h @@ -55,6 +55,7 @@ #define CONF_DEFAULT_SERVER_CONNECTIONS 1 #define CONF_DEFAULT_KETAMA_PORT 11211 #define CONF_DEFAULT_TCPKEEPALIVE false +#define CONF_DEFAULT_REUSEPORT false struct conf_listen { struct string pname; /* listen: as "hostname:port" */ @@ -95,6 +96,7 @@ struct conf_pool { int server_failure_limit; /* server_failure_limit: */ struct array server; /* servers: conf_server[] */ unsigned valid:1; /* valid? */ + int reuseport; /* set SO_REUSEPORT to socket */ }; struct conf { diff --git a/src/nc_proxy.c b/src/nc_proxy.c index 38cdd9e6..b70be97a 100644 --- a/src/nc_proxy.c +++ b/src/nc_proxy.c @@ -142,6 +142,16 @@ proxy_listen(struct context *ctx, struct conn *p) return NC_ERROR; } + if (pool->reuseport) { + status = nc_set_reuseport(p->sd); + if (status < 0) { + log_error("reuse of port '%.*s' for listening on p %d failed: %s", + pool->addrstr.len, pool->addrstr.data, p->sd, + strerror(errno)); + return NC_ERROR; + } + } + status = bind(p->sd, p->addr, p->addrlen); if (status < 0) { log_error("bind on p %d to addr '%.*s' failed: %s", p->sd, @@ -294,7 +304,7 @@ proxy_accept(struct context *ctx, struct conn *p) return NC_OK; } - /* + /* * Workaround of https://github.com/twitter/twemproxy/issues/97 * * We should never reach here because the check for conn_ncurr_cconn() diff --git a/src/nc_server.h b/src/nc_server.h index 11cdaa77..00c03750 100644 --- a/src/nc_server.h +++ b/src/nc_server.h @@ -121,6 +121,7 @@ struct server_pool { unsigned preconnect:1; /* preconnect? */ unsigned redis:1; /* redis? */ unsigned tcpkeepalive:1; /* tcpkeepalive? */ + unsigned reuseport:1; /* set SO_REUSEPORT to socket */ }; void server_ref(struct conn *conn, void *owner); diff --git a/src/nc_stats.c b/src/nc_stats.c index fa4d92c7..363a9e88 100644 --- a/src/nc_stats.c +++ b/src/nc_stats.c @@ -827,7 +827,7 @@ stats_listen(struct stats *st) log_error("socket failed: %s", strerror(errno)); return NC_ERROR; } - + nc_set_reuseport(st->sd); status = nc_set_reuseaddr(st->sd); if (status < 0) { log_error("set reuseaddr on m %d failed: %s", st->sd, strerror(errno)); diff --git a/src/nc_util.c b/src/nc_util.c index e2768089..d9f1a6b1 100644 --- a/src/nc_util.c +++ b/src/nc_util.c @@ -75,6 +75,20 @@ nc_set_reuseaddr(int sd) return setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &reuse, len); } +int +nc_set_reuseport(int sd) +{ + int reuse; + socklen_t len; + + reuse = 1; + len = sizeof(reuse); +#ifdef SO_REUSEPORT + return setsockopt(sd, SOL_SOCKET, SO_REUSEPORT, &reuse, len); +#endif + return -1; +} + /* * Disable Nagle algorithm on TCP socket. * diff --git a/src/nc_util.h b/src/nc_util.h index 986b7aef..8d433b35 100644 --- a/src/nc_util.h +++ b/src/nc_util.h @@ -81,6 +81,7 @@ int nc_set_blocking(int sd); int nc_set_nonblocking(int sd); int nc_set_reuseaddr(int sd); +int nc_set_reuseport(int sd); int nc_set_tcpnodelay(int sd); int nc_set_linger(int sd, int timeout); int nc_set_sndbuf(int sd, int size);