diff --git a/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp b/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp index d5421ea6d5..4319cb5c68 100644 --- a/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp +++ b/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp @@ -37,6 +37,11 @@ bool CopysetConfGenerator::GenCopysetConf( const ::curvefs::mds::topology::CopySetInfo &reportCopySetInfo, const ::curvefs::mds::heartbeat::ConfigChangeInfo &configChInfo, ::curvefs::mds::heartbeat::CopySetConf *copysetConf) { + // if copyset is creating return false directly + if (topo_->IsCopysetCreating(reportCopySetInfo.GetCopySetKey())) { + return false; + } + // reported copyset not exist in topology // in this case an empty configuration will be sent to metaserver // to delete it diff --git a/curvefs/src/mds/topology/topology.cpp b/curvefs/src/mds/topology/topology.cpp index ccfb263e8d..6895d165f4 100644 --- a/curvefs/src/mds/topology/topology.cpp +++ b/curvefs/src/mds/topology/topology.cpp @@ -991,6 +991,13 @@ TopoStatusCode TopologyImpl::AddCopySet(const CopySetInfo &data) { } } +TopoStatusCode TopologyImpl::AddCopySetCreating(const CopySetKey &key) { + WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_); + auto iter = copySetCreating_.insert(key); + return iter.second ? TopoStatusCode::TOPO_OK : + TopoStatusCode::TOPO_ID_DUPLICATED; +} + TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) { WriteLockGuard wlockCopySetMap(copySetMutex_); auto it = copySetMap_.find(key); @@ -1005,6 +1012,11 @@ TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) { } } +void TopologyImpl::RemoveCopySetCreating(CopySetKey key) { + WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_); + copySetCreating_.erase(key); +} + TopoStatusCode TopologyImpl::UpdateCopySetTopo(const CopySetInfo &data) { ReadLockGuard rlockCopySetMap(copySetMutex_); CopySetKey key(data.GetPoolId(), data.GetId()); @@ -1358,6 +1370,13 @@ std::string TopologyImpl::GetHostNameAndPortById(MetaServerIdType msId) { // get hostName of the metaserver return server.GetHostName() + ":" + std::to_string(ms.GetInternalPort()); } + +bool TopologyImpl::IsCopysetCreating(const CopySetKey &key) const { + ReadLockGuard rlockCopySetCreating(copySetCreatingMutex_); + return copySetCreating_.count(key) != 0; +} + + } // namespace topology } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/topology/topology.h b/curvefs/src/mds/topology/topology.h index 93c730137b..3c89e7ad44 100644 --- a/curvefs/src/mds/topology/topology.h +++ b/curvefs/src/mds/topology/topology.h @@ -79,6 +79,7 @@ class Topology { virtual TopoStatusCode AddServer(const Server &data) = 0; virtual TopoStatusCode AddMetaServer(const MetaServer &data) = 0; virtual TopoStatusCode AddCopySet(const CopySetInfo &data) = 0; + virtual TopoStatusCode AddCopySetCreating(const CopySetKey &key) = 0; virtual TopoStatusCode AddPartition(const Partition &data) = 0; virtual TopoStatusCode RemovePool(PoolIdType id) = 0; @@ -86,6 +87,7 @@ class Topology { virtual TopoStatusCode RemoveServer(ServerIdType id) = 0; virtual TopoStatusCode RemoveMetaServer(MetaServerIdType id) = 0; virtual TopoStatusCode RemoveCopySet(CopySetKey key) = 0; + virtual void RemoveCopySetCreating(CopySetKey key) = 0; virtual TopoStatusCode RemovePartition(PartitionIdType id) = 0; virtual TopoStatusCode UpdatePool(const Pool &data) = 0; @@ -231,6 +233,8 @@ class Topology { curvefs::mds::topology::MetadataUsage>* spaces) = 0; virtual std::string GetHostNameAndPortById(MetaServerIdType msId) = 0; + + virtual bool IsCopysetCreating(const CopySetKey &key) const = 0; }; class TopologyImpl : public Topology { @@ -266,6 +270,7 @@ class TopologyImpl : public Topology { TopoStatusCode AddServer(const Server &data) override; TopoStatusCode AddMetaServer(const MetaServer &data) override; TopoStatusCode AddCopySet(const CopySetInfo &data) override; + TopoStatusCode AddCopySetCreating(const CopySetKey &key) override; TopoStatusCode AddPartition(const Partition &data) override; TopoStatusCode RemovePool(PoolIdType id) override; @@ -273,6 +278,7 @@ class TopologyImpl : public Topology { TopoStatusCode RemoveServer(ServerIdType id) override; TopoStatusCode RemoveMetaServer(MetaServerIdType id) override; TopoStatusCode RemoveCopySet(CopySetKey key) override; + void RemoveCopySetCreating(CopySetKey key) override; TopoStatusCode RemovePartition(PartitionIdType id) override; TopoStatusCode UpdatePool(const Pool &data) override; @@ -445,6 +451,8 @@ class TopologyImpl : public Topology { std::string GetHostNameAndPortById(MetaServerIdType msId) override; + bool IsCopysetCreating(const CopySetKey &key) const override; + private: TopoStatusCode LoadClusterInfo(); @@ -461,6 +469,7 @@ class TopologyImpl : public Topology { std::unordered_map metaServerMap_; std::map copySetMap_; std::unordered_map partitionMap_; + std::set copySetCreating_; // cluster info ClusterInformation clusterInfo_; @@ -476,6 +485,7 @@ class TopologyImpl : public Topology { mutable RWLock metaServerMutex_; mutable RWLock copySetMutex_; mutable RWLock partitionMutex_; + mutable RWLock copySetCreatingMutex_; TopologyOption option_; curve::common::Thread backEndThread_; diff --git a/curvefs/src/mds/topology/topology_manager.cpp b/curvefs/src/mds/topology/topology_manager.cpp index c5a22fa275..101983f5db 100644 --- a/curvefs/src/mds/topology/topology_manager.cpp +++ b/curvefs/src/mds/topology/topology_manager.cpp @@ -721,6 +721,13 @@ bool TopologyManager::CreateCopysetNodeOnMetaServer( return true; } +void TopologyManager::ClearCopysetCreating(PoolIdType poolId, + const std::set ©sets) { + for (const auto &id : copysets) { + topology_->RemoveCopySetCreating(CopySetKey(poolId, id)); + } +} + TopoStatusCode TopologyManager::CreateCopyset() { PoolIdType poolId; std::set metaServerIds; @@ -752,6 +759,12 @@ TopoStatusCode TopologyManager::CreateCopyset() { if (copysetId == static_cast(UNINITIALIZE_ID)) { return TopoStatusCode::TOPO_ALLOCATE_ID_FAIL; } + if (TopoStatusCode::TOPO_OK != + topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))) { + LOG(WARNING) << "the copyset key = (" << poolId + << ", " << copysetId << ") is already creating."; + continue; + } copysetIds.emplace(copysetId); } @@ -761,17 +774,20 @@ TopoStatusCode TopologyManager::CreateCopyset() { for (auto id : copysetIds) { CopySetInfo copyset(poolId, id); copyset.SetCopySetMembers(metaServerIds); - TopoStatusCode ret = topology_->AddCopySet(copyset); + ret = topology_->AddCopySet(copyset); if (TopoStatusCode::TOPO_OK != ret) { LOG(ERROR) << "Add copyset failed after create copyset." << " poolId = " << poolId << ", copysetId = " << id << ", error msg = " << TopoStatusCode_Name(ret); + ClearCopysetCreating(poolId, copysetIds); return ret; } } } else { + ClearCopysetCreating(poolId, copysetIds); return TopoStatusCode::TOPO_CREATE_COPYSET_ON_METASERVER_FAIL; } + ClearCopysetCreating(poolId, copysetIds); return TopoStatusCode::TOPO_OK; } diff --git a/curvefs/src/mds/topology/topology_manager.h b/curvefs/src/mds/topology/topology_manager.h index 6e406484e4..2586b932b9 100644 --- a/curvefs/src/mds/topology/topology_manager.h +++ b/curvefs/src/mds/topology/topology_manager.h @@ -164,6 +164,10 @@ class TopologyManager { const uint32_t& copysetId, CopysetValue* copysetValue); + + virtual void ClearCopysetCreating(PoolIdType poolId, + const std::set ©sets); + private: std::shared_ptr topology_; std::shared_ptr metaserverClient_; diff --git a/curvefs/test/mds/topology/test_topology.cpp b/curvefs/test/mds/topology/test_topology.cpp index 00896da3dd..e58500a89d 100644 --- a/curvefs/test/mds/topology/test_topology.cpp +++ b/curvefs/test/mds/topology/test_topology.cpp @@ -1782,6 +1782,21 @@ TEST_F(TestTopology, AddCopySet_StorageFail) { ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, ret); } +TEST_F(TestTopology, CopySetCreating) { + PoolIdType poolId = 0x11; + CopySetIdType copysetId = 0x51; + + ASSERT_EQ(TopoStatusCode::TOPO_OK, + topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))); + ASSERT_EQ(TopoStatusCode::TOPO_ID_DUPLICATED, + topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))); + + ASSERT_TRUE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId))); + + topology_->RemoveCopySetCreating(CopySetKey(poolId, copysetId)); + ASSERT_FALSE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId))); +} + TEST_F(TestTopology, RemoveCopySet_success) { PoolIdType poolId = 0x11; CopySetIdType copysetId = 0x51;