From beac479b1f4f97609474791e741ea48920130227 Mon Sep 17 00:00:00 2001 From: wanghai01 Date: Tue, 18 Jan 2022 10:59:06 +0800 Subject: [PATCH] curvefs mds heartbeat doesn't delete copyset creating. The empty conf will send to metaserver to delete copysets which don't exist in mds, but when the copyset just created in metaserver and have not add to topology, the metaserver heartbeat will contain these copysets' infomation. --- .../mds/heartbeat/copyset_conf_generator.cpp | 5 +++++ curvefs/src/mds/topology/topology.cpp | 19 +++++++++++++++++++ curvefs/src/mds/topology/topology.h | 10 ++++++++++ curvefs/src/mds/topology/topology_manager.cpp | 18 +++++++++++++++++- curvefs/src/mds/topology/topology_manager.h | 4 ++++ curvefs/test/mds/topology/test_topology.cpp | 15 +++++++++++++++ 6 files changed, 70 insertions(+), 1 deletion(-) diff --git a/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp b/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp index d5421ea6d5..4319cb5c68 100644 --- a/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp +++ b/curvefs/src/mds/heartbeat/copyset_conf_generator.cpp @@ -37,6 +37,11 @@ bool CopysetConfGenerator::GenCopysetConf( const ::curvefs::mds::topology::CopySetInfo &reportCopySetInfo, const ::curvefs::mds::heartbeat::ConfigChangeInfo &configChInfo, ::curvefs::mds::heartbeat::CopySetConf *copysetConf) { + // if copyset is creating return false directly + if (topo_->IsCopysetCreating(reportCopySetInfo.GetCopySetKey())) { + return false; + } + // reported copyset not exist in topology // in this case an empty configuration will be sent to metaserver // to delete it diff --git a/curvefs/src/mds/topology/topology.cpp b/curvefs/src/mds/topology/topology.cpp index ccfb263e8d..6895d165f4 100644 --- a/curvefs/src/mds/topology/topology.cpp +++ b/curvefs/src/mds/topology/topology.cpp @@ -991,6 +991,13 @@ TopoStatusCode TopologyImpl::AddCopySet(const CopySetInfo &data) { } } +TopoStatusCode TopologyImpl::AddCopySetCreating(const CopySetKey &key) { + WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_); + auto iter = copySetCreating_.insert(key); + return iter.second ? TopoStatusCode::TOPO_OK : + TopoStatusCode::TOPO_ID_DUPLICATED; +} + TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) { WriteLockGuard wlockCopySetMap(copySetMutex_); auto it = copySetMap_.find(key); @@ -1005,6 +1012,11 @@ TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) { } } +void TopologyImpl::RemoveCopySetCreating(CopySetKey key) { + WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_); + copySetCreating_.erase(key); +} + TopoStatusCode TopologyImpl::UpdateCopySetTopo(const CopySetInfo &data) { ReadLockGuard rlockCopySetMap(copySetMutex_); CopySetKey key(data.GetPoolId(), data.GetId()); @@ -1358,6 +1370,13 @@ std::string TopologyImpl::GetHostNameAndPortById(MetaServerIdType msId) { // get hostName of the metaserver return server.GetHostName() + ":" + std::to_string(ms.GetInternalPort()); } + +bool TopologyImpl::IsCopysetCreating(const CopySetKey &key) const { + ReadLockGuard rlockCopySetCreating(copySetCreatingMutex_); + return copySetCreating_.count(key) != 0; +} + + } // namespace topology } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/topology/topology.h b/curvefs/src/mds/topology/topology.h index 93c730137b..3c89e7ad44 100644 --- a/curvefs/src/mds/topology/topology.h +++ b/curvefs/src/mds/topology/topology.h @@ -79,6 +79,7 @@ class Topology { virtual TopoStatusCode AddServer(const Server &data) = 0; virtual TopoStatusCode AddMetaServer(const MetaServer &data) = 0; virtual TopoStatusCode AddCopySet(const CopySetInfo &data) = 0; + virtual TopoStatusCode AddCopySetCreating(const CopySetKey &key) = 0; virtual TopoStatusCode AddPartition(const Partition &data) = 0; virtual TopoStatusCode RemovePool(PoolIdType id) = 0; @@ -86,6 +87,7 @@ class Topology { virtual TopoStatusCode RemoveServer(ServerIdType id) = 0; virtual TopoStatusCode RemoveMetaServer(MetaServerIdType id) = 0; virtual TopoStatusCode RemoveCopySet(CopySetKey key) = 0; + virtual void RemoveCopySetCreating(CopySetKey key) = 0; virtual TopoStatusCode RemovePartition(PartitionIdType id) = 0; virtual TopoStatusCode UpdatePool(const Pool &data) = 0; @@ -231,6 +233,8 @@ class Topology { curvefs::mds::topology::MetadataUsage>* spaces) = 0; virtual std::string GetHostNameAndPortById(MetaServerIdType msId) = 0; + + virtual bool IsCopysetCreating(const CopySetKey &key) const = 0; }; class TopologyImpl : public Topology { @@ -266,6 +270,7 @@ class TopologyImpl : public Topology { TopoStatusCode AddServer(const Server &data) override; TopoStatusCode AddMetaServer(const MetaServer &data) override; TopoStatusCode AddCopySet(const CopySetInfo &data) override; + TopoStatusCode AddCopySetCreating(const CopySetKey &key) override; TopoStatusCode AddPartition(const Partition &data) override; TopoStatusCode RemovePool(PoolIdType id) override; @@ -273,6 +278,7 @@ class TopologyImpl : public Topology { TopoStatusCode RemoveServer(ServerIdType id) override; TopoStatusCode RemoveMetaServer(MetaServerIdType id) override; TopoStatusCode RemoveCopySet(CopySetKey key) override; + void RemoveCopySetCreating(CopySetKey key) override; TopoStatusCode RemovePartition(PartitionIdType id) override; TopoStatusCode UpdatePool(const Pool &data) override; @@ -445,6 +451,8 @@ class TopologyImpl : public Topology { std::string GetHostNameAndPortById(MetaServerIdType msId) override; + bool IsCopysetCreating(const CopySetKey &key) const override; + private: TopoStatusCode LoadClusterInfo(); @@ -461,6 +469,7 @@ class TopologyImpl : public Topology { std::unordered_map metaServerMap_; std::map copySetMap_; std::unordered_map partitionMap_; + std::set copySetCreating_; // cluster info ClusterInformation clusterInfo_; @@ -476,6 +485,7 @@ class TopologyImpl : public Topology { mutable RWLock metaServerMutex_; mutable RWLock copySetMutex_; mutable RWLock partitionMutex_; + mutable RWLock copySetCreatingMutex_; TopologyOption option_; curve::common::Thread backEndThread_; diff --git a/curvefs/src/mds/topology/topology_manager.cpp b/curvefs/src/mds/topology/topology_manager.cpp index c5a22fa275..101983f5db 100644 --- a/curvefs/src/mds/topology/topology_manager.cpp +++ b/curvefs/src/mds/topology/topology_manager.cpp @@ -721,6 +721,13 @@ bool TopologyManager::CreateCopysetNodeOnMetaServer( return true; } +void TopologyManager::ClearCopysetCreating(PoolIdType poolId, + const std::set ©sets) { + for (const auto &id : copysets) { + topology_->RemoveCopySetCreating(CopySetKey(poolId, id)); + } +} + TopoStatusCode TopologyManager::CreateCopyset() { PoolIdType poolId; std::set metaServerIds; @@ -752,6 +759,12 @@ TopoStatusCode TopologyManager::CreateCopyset() { if (copysetId == static_cast(UNINITIALIZE_ID)) { return TopoStatusCode::TOPO_ALLOCATE_ID_FAIL; } + if (TopoStatusCode::TOPO_OK != + topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))) { + LOG(WARNING) << "the copyset key = (" << poolId + << ", " << copysetId << ") is already creating."; + continue; + } copysetIds.emplace(copysetId); } @@ -761,17 +774,20 @@ TopoStatusCode TopologyManager::CreateCopyset() { for (auto id : copysetIds) { CopySetInfo copyset(poolId, id); copyset.SetCopySetMembers(metaServerIds); - TopoStatusCode ret = topology_->AddCopySet(copyset); + ret = topology_->AddCopySet(copyset); if (TopoStatusCode::TOPO_OK != ret) { LOG(ERROR) << "Add copyset failed after create copyset." << " poolId = " << poolId << ", copysetId = " << id << ", error msg = " << TopoStatusCode_Name(ret); + ClearCopysetCreating(poolId, copysetIds); return ret; } } } else { + ClearCopysetCreating(poolId, copysetIds); return TopoStatusCode::TOPO_CREATE_COPYSET_ON_METASERVER_FAIL; } + ClearCopysetCreating(poolId, copysetIds); return TopoStatusCode::TOPO_OK; } diff --git a/curvefs/src/mds/topology/topology_manager.h b/curvefs/src/mds/topology/topology_manager.h index 6e406484e4..2586b932b9 100644 --- a/curvefs/src/mds/topology/topology_manager.h +++ b/curvefs/src/mds/topology/topology_manager.h @@ -164,6 +164,10 @@ class TopologyManager { const uint32_t& copysetId, CopysetValue* copysetValue); + + virtual void ClearCopysetCreating(PoolIdType poolId, + const std::set ©sets); + private: std::shared_ptr topology_; std::shared_ptr metaserverClient_; diff --git a/curvefs/test/mds/topology/test_topology.cpp b/curvefs/test/mds/topology/test_topology.cpp index 00896da3dd..e58500a89d 100644 --- a/curvefs/test/mds/topology/test_topology.cpp +++ b/curvefs/test/mds/topology/test_topology.cpp @@ -1782,6 +1782,21 @@ TEST_F(TestTopology, AddCopySet_StorageFail) { ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, ret); } +TEST_F(TestTopology, CopySetCreating) { + PoolIdType poolId = 0x11; + CopySetIdType copysetId = 0x51; + + ASSERT_EQ(TopoStatusCode::TOPO_OK, + topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))); + ASSERT_EQ(TopoStatusCode::TOPO_ID_DUPLICATED, + topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))); + + ASSERT_TRUE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId))); + + topology_->RemoveCopySetCreating(CopySetKey(poolId, copysetId)); + ASSERT_FALSE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId))); +} + TEST_F(TestTopology, RemoveCopySet_success) { PoolIdType poolId = 0x11; CopySetIdType copysetId = 0x51;