curvefs mds heartbeat doesn't delete copyset creating.

The empty conf will send to metaserver to delete copysets which don't exist in mds, but when the copyset just created in metaserver and have not add to topology, the metaserver heartbeat will contain these copysets' infomation.

curvefs/src/mds/heartbeat/copyset_conf_generator.cpp

@@ -37,6 +37,11 @@ bool CopysetConfGenerator::GenCopysetConf(
     const ::curvefs::mds::topology::CopySetInfo &reportCopySetInfo,
     const ::curvefs::mds::heartbeat::ConfigChangeInfo &configChInfo,
     ::curvefs::mds::heartbeat::CopySetConf *copysetConf) {
+    // if copyset is creating return false directly
+    if (topo_->IsCopysetCreating(reportCopySetInfo.GetCopySetKey())) {
+        return false;
+    }
+
     // reported copyset not exist in topology
     // in this case an empty configuration will be sent to metaserver
     // to delete it

curvefs/src/mds/topology/topology.cpp

@@ -991,6 +991,13 @@ TopoStatusCode TopologyImpl::AddCopySet(const CopySetInfo &data) {
     }
 }
 
+TopoStatusCode TopologyImpl::AddCopySetCreating(const CopySetKey &key) {
+    WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_);
+    auto iter = copySetCreating_.insert(key);
+    return iter.second ? TopoStatusCode::TOPO_OK :
+        TopoStatusCode::TOPO_ID_DUPLICATED;
+}
+
 TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) {
     WriteLockGuard wlockCopySetMap(copySetMutex_);
     auto it = copySetMap_.find(key);
@@ -1005,6 +1012,11 @@ TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) {
     }
 }
 
+void TopologyImpl::RemoveCopySetCreating(CopySetKey key) {
+    WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_);
+    copySetCreating_.erase(key);
+}
+
 TopoStatusCode TopologyImpl::UpdateCopySetTopo(const CopySetInfo &data) {
     ReadLockGuard rlockCopySetMap(copySetMutex_);
     CopySetKey key(data.GetPoolId(), data.GetId());
@@ -1358,6 +1370,13 @@ std::string TopologyImpl::GetHostNameAndPortById(MetaServerIdType msId) {
     // get hostName of the metaserver
     return server.GetHostName() + ":" + std::to_string(ms.GetInternalPort());
 }
+
+bool TopologyImpl::IsCopysetCreating(const CopySetKey &key) const {
+    ReadLockGuard rlockCopySetCreating(copySetCreatingMutex_);
+    return copySetCreating_.count(key) != 0;
+}
+
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs

curvefs/src/mds/topology/topology.h

@@ -79,13 +79,15 @@ class Topology {
     virtual TopoStatusCode AddServer(const Server &data) = 0;
     virtual TopoStatusCode AddMetaServer(const MetaServer &data) = 0;
     virtual TopoStatusCode AddCopySet(const CopySetInfo &data) = 0;
+    virtual TopoStatusCode AddCopySetCreating(const CopySetKey &key) = 0;
     virtual TopoStatusCode AddPartition(const Partition &data) = 0;
 
     virtual TopoStatusCode RemovePool(PoolIdType id) = 0;
     virtual TopoStatusCode RemoveZone(ZoneIdType id) = 0;
     virtual TopoStatusCode RemoveServer(ServerIdType id) = 0;
     virtual TopoStatusCode RemoveMetaServer(MetaServerIdType id) = 0;
     virtual TopoStatusCode RemoveCopySet(CopySetKey key) = 0;
+    virtual void RemoveCopySetCreating(CopySetKey key) = 0;
     virtual TopoStatusCode RemovePartition(PartitionIdType id) = 0;
 
     virtual TopoStatusCode UpdatePool(const Pool &data) = 0;
@@ -231,6 +233,8 @@ class Topology {
             curvefs::mds::topology::MetadataUsage>* spaces) = 0;
 
     virtual std::string GetHostNameAndPortById(MetaServerIdType msId) = 0;
+
+    virtual bool IsCopysetCreating(const CopySetKey &key) const = 0;
 };
 
 class TopologyImpl : public Topology {
@@ -266,13 +270,15 @@ class TopologyImpl : public Topology {
     TopoStatusCode AddServer(const Server &data) override;
     TopoStatusCode AddMetaServer(const MetaServer &data) override;
     TopoStatusCode AddCopySet(const CopySetInfo &data) override;
+    TopoStatusCode AddCopySetCreating(const CopySetKey &key) override;
     TopoStatusCode AddPartition(const Partition &data) override;
 
     TopoStatusCode RemovePool(PoolIdType id) override;
     TopoStatusCode RemoveZone(ZoneIdType id) override;
     TopoStatusCode RemoveServer(ServerIdType id) override;
     TopoStatusCode RemoveMetaServer(MetaServerIdType id) override;
     TopoStatusCode RemoveCopySet(CopySetKey key) override;
+    void RemoveCopySetCreating(CopySetKey key) override;
     TopoStatusCode RemovePartition(PartitionIdType id) override;
 
     TopoStatusCode UpdatePool(const Pool &data) override;
@@ -445,6 +451,8 @@ class TopologyImpl : public Topology {
 
     std::string GetHostNameAndPortById(MetaServerIdType msId) override;
 
+    bool IsCopysetCreating(const CopySetKey &key) const override;
+
  private:
     TopoStatusCode LoadClusterInfo();
 
@@ -461,6 +469,7 @@ class TopologyImpl : public Topology {
     std::unordered_map<MetaServerIdType, MetaServer> metaServerMap_;
     std::map<CopySetKey, CopySetInfo> copySetMap_;
     std::unordered_map<PartitionIdType, Partition> partitionMap_;
+    std::set<CopySetKey> copySetCreating_;
 
     // cluster info
     ClusterInformation clusterInfo_;
@@ -476,6 +485,7 @@ class TopologyImpl : public Topology {
     mutable RWLock metaServerMutex_;
     mutable RWLock copySetMutex_;
     mutable RWLock partitionMutex_;
+    mutable RWLock copySetCreatingMutex_;
 
     TopologyOption option_;
     curve::common::Thread backEndThread_;

curvefs/src/mds/topology/topology_manager.cpp

@@ -721,6 +721,13 @@ bool TopologyManager::CreateCopysetNodeOnMetaServer(
     return true;
 }
 
+void TopologyManager::ClearCopysetCreating(PoolIdType poolId,
+    const std::set<CopySetIdType> &copysets) {
+    for (const auto &id : copysets) {
+        topology_->RemoveCopySetCreating(CopySetKey(poolId, id));
+    }
+}
+
 TopoStatusCode TopologyManager::CreateCopyset() {
     PoolIdType poolId;
     std::set<MetaServerIdType> metaServerIds;
@@ -752,6 +759,12 @@ TopoStatusCode TopologyManager::CreateCopyset() {
         if (copysetId == static_cast<ServerIdType>(UNINITIALIZE_ID)) {
             return TopoStatusCode::TOPO_ALLOCATE_ID_FAIL;
         }
+        if (TopoStatusCode::TOPO_OK !=
+            topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))) {
+            LOG(WARNING) << "the copyset key = (" << poolId
+                         << ", " << copysetId << ") is already creating.";
+            continue;
+        }
         copysetIds.emplace(copysetId);
     }
 
@@ -761,17 +774,20 @@ TopoStatusCode TopologyManager::CreateCopyset() {
         for (auto id : copysetIds) {
             CopySetInfo copyset(poolId, id);
             copyset.SetCopySetMembers(metaServerIds);
-            TopoStatusCode ret = topology_->AddCopySet(copyset);
+            ret = topology_->AddCopySet(copyset);
             if (TopoStatusCode::TOPO_OK != ret) {
                 LOG(ERROR) << "Add copyset failed after create copyset."
                            << " poolId = " << poolId << ", copysetId = " << id
                            << ", error msg = " << TopoStatusCode_Name(ret);
+                ClearCopysetCreating(poolId, copysetIds);
                 return ret;
             }
         }
     } else {
+        ClearCopysetCreating(poolId, copysetIds);
         return TopoStatusCode::TOPO_CREATE_COPYSET_ON_METASERVER_FAIL;
     }
+    ClearCopysetCreating(poolId, copysetIds);
     return TopoStatusCode::TOPO_OK;
 }
 

curvefs/src/mds/topology/topology_manager.h

@@ -164,6 +164,10 @@ class TopologyManager {
                                 const uint32_t& copysetId,
                                 CopysetValue* copysetValue);
 
+
+    virtual void ClearCopysetCreating(PoolIdType poolId,
+        const std::set<CopySetIdType> &copysets);
+
  private:
     std::shared_ptr<Topology> topology_;
     std::shared_ptr<MetaserverClient> metaserverClient_;

curvefs/test/mds/topology/test_topology.cpp

@@ -1782,6 +1782,21 @@ TEST_F(TestTopology, AddCopySet_StorageFail) {
     ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, ret);
 }
 
+TEST_F(TestTopology, CopySetCreating) {
+    PoolIdType poolId = 0x11;
+    CopySetIdType copysetId = 0x51;
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK,
+        topology_->AddCopySetCreating(CopySetKey(poolId, copysetId)));
+    ASSERT_EQ(TopoStatusCode::TOPO_ID_DUPLICATED,
+        topology_->AddCopySetCreating(CopySetKey(poolId, copysetId)));
+
+    ASSERT_TRUE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId)));
+
+    topology_->RemoveCopySetCreating(CopySetKey(poolId, copysetId));
+    ASSERT_FALSE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId)));
+}
+
 TEST_F(TestTopology, RemoveCopySet_success) {
     PoolIdType poolId = 0x11;
     CopySetIdType copysetId = 0x51;