etcd support enable auth

Signed-off-by: wanghai01 <[email protected]>
opencurve · Aug 31, 2023 · 45e8500 · 45e8500
1 parent 6f5c85b
commit 45e8500
Show file tree

Hide file tree

Showing 15 changed files with 109 additions and 38 deletions.
diff --git a/conf/mds.conf b/conf/mds.conf
@@ -21,6 +21,10 @@ mds.etcd.retry.times=3
 mds.etcd.dlock.timeoutMs=10000
 # dlock lease timeout
 mds.etcd.dlock.ttlSec=10
+# etcd auth options
+etcd.auth.enable=false
+etcd.auth.username=
+etcd.auth.password=
 
 #
 # segment分配量统计相关配置

diff --git a/conf/snapshot_clone_server.conf b/conf/snapshot_clone_server.conf
@@ -86,6 +86,11 @@ etcd.retry.times=3
 etcd.dlock.timeoutMs=10000
 # dlock lease timeout
 etcd.dlock.ttlSec=10
+# etcd auth options
+etcd.auth.enable=false
+etcd.auth.username=
+etcd.auth.password=
+
 
 #
 # leader选举相关参数

diff --git a/curvefs/conf/mds.conf b/curvefs/conf/mds.conf
@@ -35,6 +35,10 @@ etcd.dailtimeoutMs=5000
 etcd.operation.timeoutMs=5000
 # number of times a failed operation can be retried
 etcd.retry.times=3
+# etcd auth options
+etcd.auth.enable=false
+etcd.auth.username=
+etcd.auth.password=
 
 #
 # leader election options

diff --git a/curvefs/docker/debian11/Dockerfile b/curvefs/docker/debian11/Dockerfile
@@ -1,6 +1,7 @@
 FROM opencurvedocker/curve-base:debian11
-COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
+COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 libetcdclient.so /usr/lib/
 COPY curvefs /curvefs
 RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
     && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
+    && cp /curvefs/etcd/sbin/etcdctl /usr/bin/ \
     && cp /curvefs/tools-v2/sbin/curve /usr/bin/
diff --git a/curvefs/src/mds/mds.cpp b/curvefs/src/mds/mds.cpp
@@ -58,7 +58,9 @@ MDS::MDS()
       etcdClient_(),
       leaderElection_(),
       status_(),
-      etcdEndpoint_() {}
+      etcdEndpoint_(),
+      etcdUsername_(),
+      etcdPassword_() {}
 
 MDS::~MDS() {}
 
@@ -355,28 +357,41 @@ void MDS::InitEtcdClient() {
         << ", etcd address: " << std::string(etcdConf.Endpoints, etcdConf.len)
         << ", etcdtimeout: " << etcdConf.DialTimeout
         << ", operation timeout: " << etcdTimeout
-        << ", etcd retrytimes: " << etcdRetryTimes;
+        << ", etcd retrytimes: " << etcdRetryTimes
+        << ", etcd auth enable: " << etcdConf.authEnable;
 
     LOG_IF(FATAL, !CheckEtcd()) << "Check etcd failed";
 
     LOG(INFO) << "Init etcd client succeeded, etcd address: "
               << std::string(etcdConf.Endpoints, etcdConf.len)
               << ", etcdtimeout: " << etcdConf.DialTimeout
               << ", operation timeout: " << etcdTimeout
-              << ", etcd retrytimes: " << etcdRetryTimes;
+              << ", etcd retrytimes: " << etcdRetryTimes
+              << ", etcd auth enable: " << etcdConf.authEnable;
 
     etcdClientInited_ = true;
 }
 
 void MDS::InitEtcdConf(EtcdConf* etcdConf) {
     conf_->GetValueFatalIfFail("etcd.endpoint", &etcdEndpoint_);
+    etcdConf->len = etcdEndpoint_.size();
+    etcdConf->Endpoints = &etcdEndpoint_[0];
     conf_->GetValueFatalIfFail("etcd.dailtimeoutMs", &etcdConf->DialTimeout);
+    // etcd auth config
+    bool authEnable = false;
+    conf_->GetBoolValue("etcd.auth.enable", &authEnable);
+    etcdConf->authEnable = authEnable ? 1 : 0;
+    if (authEnable) {
+        conf_->GetValueFatalIfFail("etcd.auth.username", &etcdUsername_);
+        etcdConf->username = &etcdUsername_[0];
+        etcdConf->usernameLen = etcdUsername_.size();
+        conf_->GetValueFatalIfFail("etcd.auth.password", &etcdPassword_);
+        etcdConf->password = &etcdPassword_[0];
+        etcdConf->passwordLen = etcdPassword_.size();
+    }
 
     LOG(INFO) << "etcd.endpoint: " << etcdEndpoint_;
     LOG(INFO) << "etcd.dailtimeoutMs: " << etcdConf->DialTimeout;
-
-    etcdConf->len = etcdEndpoint_.size();
-    etcdConf->Endpoints = &etcdEndpoint_[0];
 }
 
 bool MDS::CheckEtcd() {

diff --git a/curvefs/src/mds/mds.h b/curvefs/src/mds/mds.h
@@ -181,6 +181,8 @@ class MDS {
     bvar::Status<std::string> status_;
 
     std::string etcdEndpoint_;
+    std::string etcdUsername_;
+    std::string etcdPassword_;
 };
 
 }  // namespace mds

diff --git a/docker/debian11/Dockerfile b/docker/debian11/Dockerfile
@@ -13,8 +13,10 @@ RUN cd /curve-tgt/curve-sdk && \
     make install-programs && \
     rm -rf /curve-tgt
 COPY curvebs /curvebs
+COPY libetcdclient.so /usr/lib/
 RUN mkdir -p /etc/curve /etc/nebd /curve/init.d/ && \
     chmod a+x /entrypoint.sh && \
     cp /curvebs/nbd/sbin/curve-nbd /usr/bin/ && \
     cp /curvebs/tools/sbin/curve_ops_tool /usr/bin/ && \
+    cp /curvebs/etcd/sbin/etcdctl /usr/bin/ && \
     cp /curvebs/tools-v2/sbin/curve /usr/bin/
diff --git a/src/chunkserver/chunkserver.cpp b/src/chunkserver/chunkserver.cpp
@@ -104,7 +104,6 @@ int ChunkServer::Run(int argc, char** argv) {
 
     // 打印参数
     conf.PrintConfig();
-    conf.ExposeMetric("chunkserver_config");
     curve::common::ExposeCurveVersion();
 
     // ============================初始化各模块==========================//

diff --git a/src/mds/server/mds.cpp b/src/mds/server/mds.cpp
@@ -45,9 +45,6 @@ using ::curve::common::BLOCKSIZEKEY;
 using ::curve::common::CHUNKSIZEKEY;
 
 MDS::~MDS() {
-    if (etcdEndpoints_) {
-        delete etcdEndpoints_;
-    }
     if (fileLockManager_) {
         delete fileLockManager_;
     }
@@ -202,14 +199,23 @@ void MDS::Stop() {
 }
 
 void MDS::InitEtcdConf(EtcdConf* etcdConf) {
-    std::string endpoint;
-    conf_->GetValueFatalIfFail("mds.etcd.endpoint", &endpoint);
-    etcdEndpoints_ = new char[endpoint.size()];
-    etcdConf->Endpoints = etcdEndpoints_;
-    std::memcpy(etcdConf->Endpoints, endpoint.c_str(), endpoint.size());
-    etcdConf->len = endpoint.size();
+    conf_->GetValueFatalIfFail("mds.etcd.endpoint", &etcdEndpoints_);
+    etcdConf->len = etcdEndpoints_.size();
+    etcdConf->Endpoints = &etcdEndpoints_[0];
     conf_->GetValueFatalIfFail(
         "mds.etcd.dailtimeoutMs", &etcdConf->DialTimeout);
+    // etcd auth config
+    bool authEnable = false;
+    conf_->GetBoolValue("etcd.auth.enable", &authEnable);
+    etcdConf->authEnable = authEnable ? 1 : 0;
+    if (authEnable) {
+        conf_->GetValueFatalIfFail("etcd.auth.username", &etcdUsername_);
+        etcdConf->username = &etcdUsername_[0];
+        etcdConf->usernameLen = etcdUsername_.size();
+        conf_->GetValueFatalIfFail("etcd.auth.password", &etcdPassword_);
+        etcdConf->password = &etcdPassword_[0];
+        etcdConf->passwordLen = etcdPassword_.size();
+    }
 }
 
 void MDS::StartServer() {
@@ -261,7 +267,8 @@ void MDS::InitEtcdClient(const EtcdConf& etcdConf,
         << ", etcdaddr len: " << etcdConf.len
         << ", etcdtimeout: " << etcdConf.DialTimeout
         << ", operation timeout: " << etcdTimeout
-        << ", etcd retrytimes: " << retryTimes;
+        << ", etcd retrytimes: " << retryTimes
+        << ", auth enable = " << etcdConf.authEnable;
 
 
     std::string out;
@@ -275,7 +282,8 @@ void MDS::InitEtcdClient(const EtcdConf& etcdConf,
             << ", etcdaddr len: " << etcdConf.len
             << ", etcdtimeout: " << etcdConf.DialTimeout
             << ", operation timeout: " << etcdTimeout
-            << ", etcd retrytimes: " << retryTimes;
+            << ", etcd retrytimes: " << retryTimes
+            << ", auth enable = " << etcdConf.authEnable;
 }
 
 void MDS::InitLeaderElection(const LeaderElectionOptions& leaderElectionOp) {

diff --git a/src/mds/server/mds.h b/src/mds/server/mds.h
@@ -235,9 +235,11 @@ class MDS {
     std::shared_ptr<CleanDiscardSegmentTask> cleanDiscardSegmentTask_;
     std::shared_ptr<Coordinator> coordinator_;
     std::shared_ptr<HeartbeatManager> heartbeatManager_;
-    char* etcdEndpoints_;
     FileLockManager* fileLockManager_;
     std::shared_ptr<SnapshotCloneClient> snapshotCloneClient_;
+    std::string etcdEndpoints_;
+    std::string etcdUsername_;
+    std::string etcdPassword_;
 };
 
 bool ParsePoolsetRules(const std::string& str,

diff --git a/src/snapshotcloneserver/snapshotclone_server.cpp b/src/snapshotcloneserver/snapshotclone_server.cpp
@@ -115,21 +115,31 @@ void InitSnapshotCloneServerOptions(std::shared_ptr<Configuration> conf,
                         &(serverOption->dlockOpts.ttlSec));
 }
 
-void InitEtcdConf(std::shared_ptr<Configuration> conf, EtcdConf* etcdConf) {
-    std::string endpoint;
-    conf->GetValueFatalIfFail("etcd.endpoint", &endpoint);
-    char* etcdEndpoints_ = new char[endpoint.size()];
-    etcdConf->Endpoints = etcdEndpoints_;
-    std::memcpy(etcdConf->Endpoints, endpoint.c_str(), endpoint.size());
-    etcdConf->len = endpoint.size();
-    conf->GetValueFatalIfFail("etcd.dailtimeoutMs", &etcdConf->DialTimeout);
+void SnapShotCloneServer::InitEtcdConf(EtcdConf* etcdConf) {
+    conf_->GetValueFatalIfFail("etcd.endpoint", &etcdEndpoints_);
+    etcdConf->len = etcdEndpoints_.size();
+    etcdConf->Endpoints = &etcdEndpoints_[0];
+    conf_->GetValueFatalIfFail(
+        "etcd.dailtimeoutMs", &etcdConf->DialTimeout);
+    // etcd auth config
+    bool authEnable = false;
+    conf_->GetBoolValue("etcd.auth.enable", &authEnable);
+    etcdConf->authEnable = authEnable ? 1 : 0;
+    if (authEnable) {
+        conf_->GetValueFatalIfFail("etcd.auth.username", &etcdUsername_);
+        etcdConf->username = &etcdUsername_[0];
+        etcdConf->usernameLen = etcdUsername_.size();
+        conf_->GetValueFatalIfFail("etcd.auth.password", &etcdPassword_);
+        etcdConf->password = &etcdPassword_[0];
+        etcdConf->passwordLen = etcdPassword_.size();
+    }
 }
 
 void SnapShotCloneServer::InitAllSnapshotCloneOptions(void) {
     InitClientOption(conf_, &(snapshotCloneServerOptions_.clientOptions));
     InitSnapshotCloneServerOptions(conf_,
         &(snapshotCloneServerOptions_.serverOption));
-    InitEtcdConf(conf_, &(snapshotCloneServerOptions_.etcdConf));
+    InitEtcdConf(&(snapshotCloneServerOptions_.etcdConf));
 
     conf_->GetValueFatalIfFail("etcd.operation.timeoutMs",
         &(snapshotCloneServerOptions_.etcdClientTimeout));

diff --git a/src/snapshotcloneserver/snapshotclone_server.h b/src/snapshotcloneserver/snapshotclone_server.h
@@ -122,6 +122,7 @@ class SnapShotCloneServer {
     void RunUntilQuit(void);
 
  private:
+    void InitEtcdConf(EtcdConf* etcdConf);
     bool InitEtcdClient(void);
 
  private:
@@ -153,7 +154,12 @@ class SnapShotCloneServer {
     std::shared_ptr<CloneServiceManager>  cloneServiceManager_;
     std::shared_ptr<SnapshotCloneServiceImpl> service_;
     std::shared_ptr<brpc::Server>          server_;
+
+    std::string etcdEndpoints_;
+    std::string etcdUsername_;
+    std::string etcdPassword_;
 };
+
 }  // namespace snapshotcloneserver
 }  // namespace curve
 

diff --git a/src/tools/metric_name.h b/src/tools/metric_name.h
@@ -37,7 +37,7 @@ const char kCurveVersionMetricName[] = "curve_version";
 
 // snapshot clone server metric name
 const char kSnapshotCloneConfMetricName[] =
-                                "snapshotcloneserver_config_server_address";
+    "snapshot_clone_server_config_server_address";
 const char kSnapshotCloneStatusMetricName[] = "snapshotcloneserver_status";
 const char kSnapshotCloneStatusActive[] = "active";
 

diff --git a/thirdparties/etcdclient/etcdclient.go b/thirdparties/etcdclient/etcdclient.go
@@ -68,6 +68,11 @@ struct EtcdConf {
     char *Endpoints;
     int len;
     int DialTimeout;
+    int authEnable;
+    char *username;
+    int usernameLen;
+    char *password;
+    int passwordLen;
 };
 
 struct Operation {
@@ -199,16 +204,22 @@ func GetErrCode(op string, err error) C.enum_EtcdErrCode {
 }
 
 // TODO(lixiaocui): 日志打印看是否需要glog
+//
 //export NewEtcdClientV3
 func NewEtcdClientV3(conf C.struct_EtcdConf) C.enum_EtcdErrCode {
 	var err error
-	globalClient, err = clientv3.New(clientv3.Config{
+	cfg := clientv3.Config{
 		Endpoints:            GetEndpoint(C.GoStringN(conf.Endpoints, conf.len)),
 		DialTimeout:          time.Duration(int(conf.DialTimeout)) * time.Millisecond,
 		DialOptions:          []grpc.DialOption{grpc.WithBlock()},
 		DialKeepAliveTime:    time.Second,
 		DialKeepAliveTimeout: time.Second,
-	})
+	}
+	if conf.authEnable == 1 {
+		cfg.Username = C.GoStringN(conf.username, conf.usernameLen)
+		cfg.Password = C.GoStringN(conf.password, conf.passwordLen)
+	}
+	globalClient, err = clientv3.New(cfg)
 	return GetErrCode(EtcdNewClient, err)
 }
 
@@ -272,6 +283,7 @@ func EtcdClientGet(timeout C.int, key *C.char,
 }
 
 // TODO(lixiaocui): list可能需要有长度限制
+//
 //export EtcdClientList
 func EtcdClientList(timeout C.int, startKey, endKey *C.char,
 	startLen, endLen C.int) (C.enum_EtcdErrCode, uint64, int64) {
@@ -653,12 +665,12 @@ func EtcdMutexLock(timeout C.int, id C.int64_t) C.enum_EtcdErrCode {
 
 //export EtcdMutexUnlock
 func EtcdMutexUnlock(timeout C.int, id C.int64_t) C.enum_EtcdErrCode {
-   ctx, cancel := context.WithTimeout(context.Background(),
-	   time.Duration(int(timeout))*time.Millisecond)
-   defer cancel()
+	ctx, cancel := context.WithTimeout(context.Background(),
+		time.Duration(int(timeout))*time.Millisecond)
+	defer cancel()
 
-   err := etcdMutex[clientv3.LeaseID(id)].Unlock(ctx)
-   return GetErrCode(EtcdUnlock, err)
+	err := etcdMutex[clientv3.LeaseID(id)].Unlock(ctx)
+	return GetErrCode(EtcdUnlock, err)
 }
 
 //export DestoryEtcdMutex

diff --git a/util/install.sh b/util/install.sh
@@ -338,10 +338,11 @@ main() {
     elif [ "$g_stor" == "bs" ]; then
         install_curvebs
         install_playground
+        install_tools-v2
     else
         install_curvefs
+        install_tools-v2
     fi
-    install_tools-v2
 }
 
 ############################  MAIN()