From bfd5acbf1bfc24047af857be7218c76cfe27c8e2 Mon Sep 17 00:00:00 2001 From: wanghai01 Date: Thu, 30 Nov 2023 11:33:17 +0800 Subject: [PATCH] curvefs: fix trash will delete file data more than once Signed-off-by: wanghai01 --- curvefs/proto/metaserver.proto | 2 +- curvefs/src/metaserver/inode_manager.cpp | 50 ++--- curvefs/src/metaserver/inode_manager.h | 2 - curvefs/src/metaserver/partition.cpp | 21 +- curvefs/src/metaserver/partition.h | 6 +- curvefs/src/metaserver/trash.cpp | 217 ++++++++++----------- curvefs/src/metaserver/trash.h | 75 +++++-- curvefs/src/metaserver/trash_manager.cpp | 16 +- curvefs/src/metaserver/trash_manager.h | 10 +- curvefs/test/metaserver/partition_test.cpp | 6 - curvefs/test/metaserver/trash_test.cpp | 108 ++++++---- 11 files changed, 268 insertions(+), 245 deletions(-) diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto index 7bc0759841..a1ec4ba69d 100644 --- a/curvefs/proto/metaserver.proto +++ b/curvefs/proto/metaserver.proto @@ -267,7 +267,7 @@ message Inode { optional uint64 rdev = 16; // field 17 is left for compatibility map s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index - optional uint32 dtime = 19; + optional uint64 dtime = 19; optional uint32 openmpcount = 20; // openmpcount mount points had the file open map xattr = 21; repeated uint64 parent = 22; diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index 9fc94643a4..f1b2d87c14 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -283,16 +283,17 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId, VLOG(6) << "DeleteInode, fsId = " << fsId << ", inodeId = " << inodeId; NameLockGuard lg(inodeLock_, GetInodeLockName(fsId, inodeId)); InodeAttr attr; - MetaStatusCode retGetAttr = - inodeStorage_->GetAttr(Key4Inode(fsId, inodeId), &attr); - if (retGetAttr != MetaStatusCode::OK) { - VLOG(9) << "GetInodeAttr fail, fsId = " << fsId - << ", inodeId = " << inodeId - << ", ret = " << MetaStatusCode_Name(retGetAttr); + auto ret = inodeStorage_->GetAttr(Key4Inode(fsId, inodeId), &attr); + if (ret == MetaStatusCode::NOT_FOUND) { + return MetaStatusCode::OK; + } else if (ret != MetaStatusCode::OK) { + LOG(ERROR) << "GetInodeAttr fail, fsId = " << fsId + << ", inodeId = " << inodeId + << ", ret = " << MetaStatusCode_Name(ret); + return ret; } - MetaStatusCode ret = - inodeStorage_->Delete(Key4Inode(fsId, inodeId), logIndex); + ret = inodeStorage_->Delete(Key4Inode(fsId, inodeId), logIndex); if (ret != MetaStatusCode::OK) { LOG(ERROR) << "DeleteInode fail, fsId = " << fsId << ", inodeId = " << inodeId @@ -300,9 +301,13 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId, return ret; } - if (retGetAttr == MetaStatusCode::OK) { + // if nlink is 0 means this inode is already in trash + if (attr.nlink() != 0) { // get attr success --(*type2InodeNum_)[attr.type()]; + } else { + // delete trash item + trash_->Remove(inodeId); } VLOG(6) << "DeleteInode success, fsId = " << fsId << ", inodeId = " << inodeId; @@ -353,8 +358,7 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, if (request.has_nlink()) { if (old.nlink() != 0 && request.nlink() == 0) { - uint32_t now = TimeUtility::GetTimeofDaySec(); - old.set_dtime(now); + old.set_dtime(TimeUtility::GetTimeofDaySec()); needAddTrash = true; } VLOG(9) << "update inode nlink, from " << old.nlink() << " to " @@ -373,7 +377,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, bool fileNeedDeallocate = (needAddTrash && (FsFileType::TYPE_FILE == old.type())); - bool s3NeedTrash = (needAddTrash && (FsFileType::TYPE_S3 == old.type())); std::shared_ptr txn; if (needUpdate) { @@ -388,8 +391,8 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, } } - if (s3NeedTrash) { - trash_->Add(old.fsid(), old.inodeid(), old.dtime()); + if (needAddTrash) { + trash_->Add(old.inodeid(), old.dtime()); --(*type2InodeNum_)[old.type()]; } @@ -610,25 +613,6 @@ MetaStatusCode InodeManager::UpdateInodeWhenCreateOrRemoveSubNode( return MetaStatusCode::OK; } -MetaStatusCode InodeManager::InsertInode(const Inode& inode, int64_t logIndex) { - CHECK_APPLIED(); - VLOG(6) << "InsertInode, " << inode.ShortDebugString(); - - // 2. insert inode - MetaStatusCode ret = inodeStorage_->Insert(inode, logIndex); - if (ret != MetaStatusCode::OK) { - LOG(ERROR) << "InsertInode fail, " << inode.ShortDebugString() - << ", ret = " << MetaStatusCode_Name(ret); - return ret; - } - - if (inode.nlink() == 0) { - trash_->Add(inode.fsid(), inode.inodeid(), inode.dtime()); - } - - return MetaStatusCode::OK; -} - bool InodeManager::GetInodeIdList(std::list* inodeIdList) { return inodeStorage_->GetAllInodeId(inodeIdList); } diff --git a/curvefs/src/metaserver/inode_manager.h b/curvefs/src/metaserver/inode_manager.h index 29b9b13251..54256029e6 100644 --- a/curvefs/src/metaserver/inode_manager.h +++ b/curvefs/src/metaserver/inode_manager.h @@ -108,8 +108,6 @@ class InodeManager { MetaStatusCode UpdateInodeWhenCreateOrRemoveSubNode( const Dentry& dentry, const Time& tm, bool isCreate, int64_t logIndex); - MetaStatusCode InsertInode(const Inode& inode, int64_t logIndex); - bool GetInodeIdList(std::list* inodeIdList); // Update one or more volume extent slice diff --git a/curvefs/src/metaserver/partition.cpp b/curvefs/src/metaserver/partition.cpp index db0e70bc60..7e4481a52c 100644 --- a/curvefs/src/metaserver/partition.cpp +++ b/curvefs/src/metaserver/partition.cpp @@ -69,7 +69,9 @@ Partition::Partition(PartitionInfo partition, dentryStorage_ = std::make_shared(kvStorage_, nameGen_, nDentry); - auto trash = std::make_shared(inodeStorage_); + auto trash = std::make_shared(inodeStorage_, + partitionInfo_.fsid(), partitionInfo_.poolid(), + partitionInfo_.copysetid(), partitionInfo_.partitionid()); inodeManager_ = std::make_shared( inodeStorage_, trash, partitionInfo_.mutable_filetype2inodenum()); txManager_ = std::make_shared(dentryStorage_, partitionInfo_); @@ -359,15 +361,6 @@ MetaStatusCode Partition::PaddingInodeS3ChunkInfo(int32_t fsId, return inodeManager_->PaddingInodeS3ChunkInfo(fsId, inodeId, m, limit); } -MetaStatusCode Partition::InsertInode(const Inode& inode, int64_t logIndex) { - PRECHECK(inode.fsid(), inode.inodeid()); - auto ret = inodeManager_->InsertInode(inode, logIndex); - if (ret == MetaStatusCode::IDEMPOTENCE_OK) { - ret = MetaStatusCode::OK; - } - return ret; -} - bool Partition::GetInodeIdList(std::list* InodeIdList) { return inodeManager_->GetInodeIdList(InodeIdList); } @@ -470,12 +463,12 @@ uint64_t Partition::GetNewInodeId() { return newInodeId; } -uint32_t Partition::GetInodeNum() { - return static_cast(inodeStorage_->Size()); +uint64_t Partition::GetInodeNum() { + return inodeStorage_->Size(); } -uint32_t Partition::GetDentryNum() { - return static_cast(dentryStorage_->Size()); +uint64_t Partition::GetDentryNum() { + return dentryStorage_->Size(); } bool Partition::EmptyInodeStorage() { return inodeStorage_->Empty(); } diff --git a/curvefs/src/metaserver/partition.h b/curvefs/src/metaserver/partition.h index cd6d2e4dd7..df930bda52 100644 --- a/curvefs/src/metaserver/partition.h +++ b/curvefs/src/metaserver/partition.h @@ -138,8 +138,6 @@ class Partition { virtual MetaStatusCode GetAllBlockGroup( std::vector* deallocatableBlockGroupVec); - MetaStatusCode InsertInode(const Inode& inode, int64_t logIndex); - bool GetInodeIdList(std::list* InodeIdList); // if partition has no inode or no dentry, it is deletable @@ -167,9 +165,9 @@ class Partition { // if no available inode id in this partiton ,return UINT64_MAX uint64_t GetNewInodeId(); - uint32_t GetInodeNum(); + uint64_t GetInodeNum(); - uint32_t GetDentryNum(); + uint64_t GetDentryNum(); bool EmptyInodeStorage(); diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index 1175376e72..62cf47a89d 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -25,6 +25,8 @@ #include "curvefs/proto/mds.pb.h" #include "curvefs/src/metaserver/inode_storage.h" #include "src/common/timeutility.h" +#include "curvefs/src/metaserver/copyset/copyset_node_manager.h" +#include "curvefs/src/metaserver/copyset/meta_operator.h" using ::curve::common::TimeUtility; @@ -52,165 +54,153 @@ void TrashImpl::Init(const TrashOption &option) { options_ = option; s3Adaptor_ = option.s3Adaptor; mdsClient_ = option.mdsClient; + copysetNode_ = copyset::CopysetNodeManager::GetInstance(). + GetSharedCopysetNode(poolId_, copysetId_); isStop_ = false; } -void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) { - TrashItem item; - item.fsId = fsId; - item.inodeId = inodeId; - item.dtime = dtime; +void TrashImpl::StopScan() { + isStop_ = true; +} +bool TrashImpl::IsStop() { + return isStop_; +} + +void TrashImpl::Add(uint64_t inodeId, uint64_t dtime) { + if (isStop_) { + return; + } LockGuard lg(itemsMutex_); + trashItems_[inodeId] = dtime; + VLOG(6) << "Add Trash Item success, fsId = " << fsId_ + << ", partitionId = " << partitionId_ + << ", inodeId = " << inodeId + << ", dtime = " << dtime; +} + +void TrashImpl::Remove(uint64_t inodeId) { if (isStop_) { return; } - trashItems_.push_back(item); - VLOG(6) << "Add Trash Item success, item.fsId = " << item.fsId - << ", item.inodeId = " << item.inodeId - << ", item.dtime = " << item.dtime; + LockGuard lg(itemsMutex_); + trashItems_.erase(inodeId); + VLOG(6) << "Remove Trash Item success, fsId = " << fsId_ + << ", partitionId = " << partitionId_ + << ", inodeId = " << inodeId; } void TrashImpl::ScanTrash() { LockGuard lgScan(scanMutex_); - std::list temp; + // only scan on leader + if (copysetNode_ == nullptr || !copysetNode_->IsLeaderTerm()) { + return; + } + + std::unordered_map temp; { LockGuard lgItems(itemsMutex_); trashItems_.swap(temp); } - for (auto it = temp.begin(); it != temp.end();) { - if (isStop_) { + for (auto& it : temp) { + if (isStop_ || !copysetNode_->IsLeaderTerm()) { return; } - if (NeedDelete(*it)) { - MetaStatusCode ret = DeleteInodeAndData(*it); - if (MetaStatusCode::NOT_FOUND == ret) { - it = temp.erase(it); - continue; - } + if (NeedDelete(it.second)) { + MetaStatusCode ret = DeleteInodeAndData(it.first); if (ret != MetaStatusCode::OK) { - LOG(ERROR) << "DeleteInodeAndData fail, fsId = " << it->fsId - << ", inodeId = " << it->inodeId + LOG(ERROR) << "DeleteInodeAndData fail, fsId = " << fsId_ + << ", inodeId = " << it.first << ", ret = " << MetaStatusCode_Name(ret); - it++; continue; } - VLOG(6) << "Trash Delete Inode, fsId = " << it->fsId - << ", inodeId = " << it->inodeId; - it = temp.erase(it); - } else { - it++; + LOG(INFO) << "Trash delete inode, fsId = " << fsId_ + << ", partitionId = " << partitionId_ + << ", inodeId = " << it.first; + temp.erase(it.first); } } { LockGuard lgItems(itemsMutex_); - trashItems_.splice(trashItems_.end(), temp); + trashItems_.insert(temp.begin(), temp.end()); } } -void TrashImpl::StopScan() { - isStop_ = true; -} - -bool TrashImpl::IsStop() { - return isStop_; -} - -bool TrashImpl::NeedDelete(const TrashItem &item) { - uint32_t now = TimeUtility::GetTimeofDaySec(); - Inode inode; - MetaStatusCode ret = - inodeStorage_->Get(Key4Inode(item.fsId, item.inodeId), &inode); - if (MetaStatusCode::NOT_FOUND == ret) { - LOG(WARNING) << "GetInode find inode not exist, fsId = " << item.fsId - << ", inodeId = " << item.inodeId - << ", ret = " << MetaStatusCode_Name(ret); - return true; - } else if (ret != MetaStatusCode::OK) { - LOG(WARNING) << "GetInode fail, fsId = " << item.fsId - << ", inodeId = " << item.inodeId - << ", ret = " << MetaStatusCode_Name(ret); - return false; - } - +bool TrashImpl::NeedDelete(uint64_t dtime) { // for compatibility, if fs recycleTimeHour is 0, use old trash logic // if fs recycleTimeHour is 0, use trash wait until expiredAfterSec // if fs recycleTimeHour is not 0, return true - uint64_t recycleTimeHour = GetFsRecycleTimeHour(item.fsId); + uint64_t recycleTimeHour = GetFsRecycleTimeHour(fsId_); if (recycleTimeHour == 0) { - return ((now - item.dtime) >= FLAGS_trash_expiredAfterSec); + return ((TimeUtility::GetTimeofDaySec() - dtime) >= + FLAGS_trash_expiredAfterSec); } else { return true; } } uint64_t TrashImpl::GetFsRecycleTimeHour(uint32_t fsId) { - FsInfo fsInfo; uint64_t recycleTimeHour = 0; - if (fsInfoMap_.find(fsId) == fsInfoMap_.end()) { - auto ret = mdsClient_->GetFsInfo(fsId, &fsInfo); + if (fsInfo_.fsid() == 0) { + auto ret = mdsClient_->GetFsInfo(fsId, &fsInfo_); if (ret != FSStatusCode::OK) { if (FSStatusCode::NOT_FOUND == ret) { LOG(ERROR) << "The fs not exist, fsId = " << fsId; return 0; } else { - LOG(ERROR) - << "GetFsInfo failed, FSStatusCode = " << ret - << ", FSStatusCode_Name = " << FSStatusCode_Name(ret) - << ", fsId = " << fsId; + LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret + << ", FSStatusCode_Name = " << FSStatusCode_Name(ret) + << ", fsId = " << fsId; return 0; } } - fsInfoMap_.insert({fsId, fsInfo}); - } else { - fsInfo = fsInfoMap_.find(fsId)->second; } - if (fsInfo.has_recycletimehour()) { - recycleTimeHour = fsInfo.recycletimehour(); - } else { - recycleTimeHour = 0; + if (fsInfo_.has_recycletimehour()) { + recycleTimeHour = fsInfo_.recycletimehour(); } return recycleTimeHour; } -MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) { +MetaStatusCode TrashImpl::DeleteInodeAndData(uint64_t inodeId) { Inode inode; - MetaStatusCode ret = - inodeStorage_->Get(Key4Inode(item.fsId, item.inodeId), &inode); + auto ret = inodeStorage_->Get(Key4Inode(fsId_, inodeId), &inode); + if (ret == MetaStatusCode::NOT_FOUND) { + LOG(WARNING) << "GetInode fail, fsId = " << fsId_ + << ", inodeId = " << inodeId + << ", ret = " << MetaStatusCode_Name(ret); + return MetaStatusCode::OK; + } if (ret != MetaStatusCode::OK) { - LOG(WARNING) << "GetInode fail, fsId = " << item.fsId - << ", inodeId = " << item.inodeId + LOG(WARNING) << "GetInode fail, fsId = " << fsId_ + << ", inodeId = " << inodeId << ", ret = " << MetaStatusCode_Name(ret); return ret; } + // 1. delete data first if (FsFileType::TYPE_FILE == inode.type()) { // TODO(xuchaojie) : delete on volume } else if (FsFileType::TYPE_S3 == inode.type()) { // get s3info from mds - FsInfo fsInfo; - if (fsInfoMap_.find(item.fsId) == fsInfoMap_.end()) { - auto ret = mdsClient_->GetFsInfo(item.fsId, &fsInfo); + if (fsInfo_.fsid() == 0) { + auto ret = mdsClient_->GetFsInfo(fsId_, &fsInfo_); if (ret != FSStatusCode::OK) { if (FSStatusCode::NOT_FOUND == ret) { - LOG(ERROR) << "The fsName not exist, fsId = " << item.fsId; + LOG(ERROR) << "The fsName not exist, fsId = " << fsId_; return MetaStatusCode::S3_DELETE_ERR; } else { LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret << ", FSStatusCode_Name = " << FSStatusCode_Name(ret) - << ", fsId = " << item.fsId; + << ", fsId = " << fsId_; return MetaStatusCode::S3_DELETE_ERR; } } - fsInfoMap_.insert({item.fsId, fsInfo}); - } else { - fsInfo = fsInfoMap_.find(item.fsId)->second; } - const auto& s3Info = fsInfo.detail().s3info(); + const auto& s3Info = fsInfo_.detail().s3info(); // reinit s3 adaptor S3ClientAdaptorOption clientAdaptorOption; s3Adaptor_->GetS3ClientAdaptorOption(&clientAdaptorOption); @@ -219,43 +209,52 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) { clientAdaptorOption.objectPrefix = s3Info.objectprefix(); s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(), s3Info.endpoint(), s3Info.bucketname()); - ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId, - item.inodeId, inode.mutable_s3chunkinfomap()); + ret = inodeStorage_->PaddingInodeS3ChunkInfo(fsId_, inodeId, + inode.mutable_s3chunkinfomap()); if (ret != MetaStatusCode::OK) { - LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId - << ", inodeId = " << item.inodeId - << ", retCode = " << MetaStatusCode_Name(ret); + LOG(ERROR) << "GetInode chunklist fail, fsId = " << fsId_ + << ", inodeId = " << inodeId + << ", retCode = " << MetaStatusCode_Name(ret); return ret; } - if (inode.s3chunkinfomap().empty()) { - LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId - << ", inodeId = " << item.inodeId; - return MetaStatusCode::NOT_FOUND; - } - VLOG(9) << "DeleteInodeAndData, inode: " - << inode.ShortDebugString(); - int retVal = s3Adaptor_->Delete(inode); - if (retVal != 0) { - LOG(ERROR) << "S3ClientAdaptor delete s3 data failed" - << ", ret = " << retVal << ", fsId = " << item.fsId - << ", inodeId = " << item.inodeId; - return MetaStatusCode::S3_DELETE_ERR; + VLOG(9) << "DeleteInodeAndData, inode: " << inode.ShortDebugString(); + if (!inode.s3chunkinfomap().empty()) { + int retVal = s3Adaptor_->Delete(inode); + if (retVal != 0) { + LOG(ERROR) << "S3ClientAdaptor delete s3 data failed" + << ", ret = " << retVal << ", fsId = " << fsId_ + << ", inodeId = " << inodeId; + return MetaStatusCode::S3_DELETE_ERR; + } } } - ret = inodeStorage_->ForceDelete(Key4Inode(item.fsId, item.inodeId)); - if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) { - LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId - << ", inodeId = " << item.inodeId - << ", ret = " << MetaStatusCode_Name(ret); - return ret; + // 2. delete metadata + if (copysetNode_->IsLeaderTerm()) { + return DeleteInode(inodeId); } return MetaStatusCode::OK; } -void TrashImpl::ListItems(std::list *items) { - LockGuard lgScan(scanMutex_); +MetaStatusCode TrashImpl::DeleteInode(uint64_t inodeId) { + DeleteInodeRequest request; + request.set_poolid(poolId_); + request.set_copysetid(copysetId_); + request.set_partitionid(partitionId_); + request.set_fsid(fsId_); + request.set_inodeid(inodeId); + + DeleteInodeResponse response; + DeleteInodeClosure done; + auto DeleteInodeOp = new copyset::DeleteInodeOperator( + copysetNode_.get(), nullptr, &request, &response, &done); + DeleteInodeOp->Propose(); + done.WaitRunned(); + return response.statuscode(); +} + +uint64_t TrashImpl::Size() { LockGuard lgItems(itemsMutex_); - *items = trashItems_; + return trashItems_.size(); } } // namespace metaserver diff --git a/curvefs/src/metaserver/trash.h b/curvefs/src/metaserver/trash.h index de9410ddc3..dacb67df34 100644 --- a/curvefs/src/metaserver/trash.h +++ b/curvefs/src/metaserver/trash.h @@ -34,10 +34,15 @@ #include "curvefs/src/metaserver/inode_storage.h" #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h" #include "curvefs/src/client/rpcclient/mds_client.h" +#include "curvefs/src/metaserver/common/types.h" namespace curvefs { namespace metaserver { +namespace copyset { +class CopysetNode; +} // namespace copyset + using ::curve::common::Configuration; using ::curve::common::Thread; using ::curve::common::Atomic; @@ -47,16 +52,6 @@ using ::curve::common::InterruptibleSleeper; using ::curvefs::client::rpcclient::MdsClient; using ::curvefs::client::rpcclient::MdsClientImpl; -struct TrashItem { - uint32_t fsId; - uint64_t inodeId; - uint32_t dtime; - TrashItem() - : fsId(0), - inodeId(0), - dtime(0) {} -}; - struct TrashOption { uint32_t scanPeriodSec; uint32_t expiredAfterSec; @@ -78,9 +73,11 @@ class Trash { virtual void Init(const TrashOption &option) = 0; - virtual void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) = 0; + virtual void Add(uint64_t inodeId, uint64_t dtime) = 0; + + virtual void Remove(uint64_t inodeId) = 0; - virtual void ListItems(std::list *items) = 0; + virtual uint64_t Size() = 0; virtual void ScanTrash() = 0; @@ -91,16 +88,21 @@ class Trash { class TrashImpl : public Trash { public: - explicit TrashImpl(const std::shared_ptr &inodeStorage) - : inodeStorage_(inodeStorage) {} + explicit TrashImpl(const std::shared_ptr &inodeStorage, + uint32_t fsId = 0, PoolId poolId = 0, CopysetId copysetId = 0, + PartitionId partitionId = 0) : + inodeStorage_(inodeStorage), fsId_(fsId), poolId_(poolId), + copysetId_(copysetId), partitionId_(partitionId) {} ~TrashImpl() {} void Init(const TrashOption &option) override; - void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) override; + void Add(uint64_t inodeId, uint64_t dtime) override; - void ListItems(std::list *items) override; + void Remove(uint64_t inodeId) override; + + uint64_t Size() override; void ScanTrash() override; @@ -108,21 +110,33 @@ class TrashImpl : public Trash { bool IsStop() override; - private: - bool NeedDelete(const TrashItem &item); + // for utests + void SetCopysetNode(const std::shared_ptr &node) { + copysetNode_ = node; + } - MetaStatusCode DeleteInodeAndData(const TrashItem &item); + private: + bool NeedDelete(uint64_t dtime); uint64_t GetFsRecycleTimeHour(uint32_t fsId); + MetaStatusCode DeleteInodeAndData(uint64_t inodeId); + + MetaStatusCode DeleteInode(uint64_t inodeId); + private: std::shared_ptr inodeStorage_; std::shared_ptr s3Adaptor_; std::shared_ptr mdsClient_; - std::unordered_map fsInfoMap_; + std::shared_ptr copysetNode_; + FsInfo fsInfo_; - std::list trashItems_; + uint32_t fsId_; + PoolId poolId_; + CopysetId copysetId_; + PartitionId partitionId_; + std::unordered_map trashItems_; mutable Mutex itemsMutex_; TrashOption options_; @@ -132,6 +146,25 @@ class TrashImpl : public Trash { bool isStop_; }; +class DeleteInodeClosure : public google::protobuf::Closure { + private: + std::mutex mutex_; + std::condition_variable cond_; + bool runned_ = false; + + public: + void Run() override { + std::lock_guard l(mutex_); + runned_ = true; + cond_.notify_one(); + } + + void WaitRunned() { + std::unique_lock ul(mutex_); + cond_.wait(ul, [this]() { return runned_; }); + } +}; + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/trash_manager.cpp b/curvefs/src/metaserver/trash_manager.cpp index 7f6341db1c..796c5e2da9 100644 --- a/curvefs/src/metaserver/trash_manager.cpp +++ b/curvefs/src/metaserver/trash_manager.cpp @@ -49,6 +49,13 @@ void TrashManager::Fini() { LOG(INFO) << "stop trash manager ok."; } +void TrashManager::Add(uint32_t partitionId, + const std::shared_ptr &trash) { + curve::common::WriteLockGuard lg(rwLock_); + trash->Init(options_); + trashs_.emplace(partitionId, trash); +} + void TrashManager::ScanLoop() { while (sleeper_.wait_for(std::chrono::seconds(FLAGS_trash_scanPeriodSec))) { ScanEveryTrash(); @@ -82,18 +89,17 @@ void TrashManager::Remove(uint32_t partitionId) { } } -void TrashManager::ListItems(std::list *items) { - items->clear(); +uint64_t TrashManager::Size() { + uint64_t size = 0; std::map> temp; { curve::common::ReadLockGuard lg(rwLock_); temp = trashs_; } for (auto &pair : temp) { - std::list newItems; - pair.second->ListItems(&newItems); - items->splice(items->end(), newItems); + size += pair.second->Size(); } + return size; } } // namespace metaserver diff --git a/curvefs/src/metaserver/trash_manager.h b/curvefs/src/metaserver/trash_manager.h index ca25c0145d..221b4629d6 100644 --- a/curvefs/src/metaserver/trash_manager.h +++ b/curvefs/src/metaserver/trash_manager.h @@ -44,13 +44,7 @@ class TrashManager { return instance_; } - void Add(uint32_t partitionId, const std::shared_ptr &trash) { - curve::common::WriteLockGuard lg(rwLock_); - trash->Init(options_); - trashs_.emplace(partitionId, trash); - LOG(INFO) << "add partition to trash manager, partitionId = " - << partitionId; - } + void Add(uint32_t partitionId, const std::shared_ptr &trash); void Remove(uint32_t partitionId); @@ -64,7 +58,7 @@ class TrashManager { void ScanEveryTrash(); - void ListItems(std::list *items); + uint64_t Size(); private: void ScanLoop(); diff --git a/curvefs/test/metaserver/partition_test.cpp b/curvefs/test/metaserver/partition_test.cpp index a68087714f..7852786dca 100644 --- a/curvefs/test/metaserver/partition_test.cpp +++ b/curvefs/test/metaserver/partition_test.cpp @@ -405,12 +405,6 @@ TEST_F(PartitionTest, PARTITION_ID_MISSMATCH_ERROR) { UpdateInodeRequest inode3Request = MakeUpdateInodeRequestFromInode(inode3); ASSERT_EQ(partition1.UpdateInode(inode3Request, logIndex_++), MetaStatusCode::PARTITION_ID_MISSMATCH); - - // test InsertInode - ASSERT_EQ(partition1.InsertInode(inode2, logIndex_++), - MetaStatusCode::PARTITION_ID_MISSMATCH); - ASSERT_EQ(partition1.InsertInode(inode3, logIndex_++), - MetaStatusCode::PARTITION_ID_MISSMATCH); } TEST_F(PartitionTest, testGetInodeAttr) { diff --git a/curvefs/test/metaserver/trash_test.cpp b/curvefs/test/metaserver/trash_test.cpp index b4b792e0a7..5af1aad196 100644 --- a/curvefs/test/metaserver/trash_test.cpp +++ b/curvefs/test/metaserver/trash_test.cpp @@ -30,6 +30,8 @@ #include "src/fs/ext4_filesystem_impl.h" #include "curvefs/test/client/rpcclient/mock_mds_client.h" #include "curvefs/test/metaserver/mock_metaserver_s3_adaptor.h" +#include "src/common/timeutility.h" +#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h" using ::testing::_; using ::testing::AtLeast; @@ -39,6 +41,7 @@ using ::testing::ReturnArg; using ::testing::SaveArg; using ::testing::SetArgPointee; using ::testing::StrEq; +using ::testing::Invoke; namespace curvefs { namespace metaserver { @@ -47,11 +50,15 @@ namespace { auto localfs = curve::fs::Ext4FileSystemImpl::getInstance(); } +DECLARE_uint32(trash_expiredAfterSec); +DECLARE_uint32(trash_scanPeriodSec); + using ::curvefs::client::rpcclient::MockMdsClient; using ::curvefs::metaserver::storage::KVStorage; using ::curvefs::metaserver::storage::RandomStoragePath; using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::StorageOptions; +using ::curvefs::metaserver::copyset::MockCopysetNode; DECLARE_uint32(trash_expiredAfterSec); DECLARE_uint32(trash_scanPeriodSec); @@ -59,8 +66,6 @@ DECLARE_uint32(trash_scanPeriodSec); class TestTrash : public ::testing::Test { protected: void SetUp() override { - FLAGS_trash_scanPeriodSec = 1; - FLAGS_trash_expiredAfterSec = 1; dataDir_ = RandomStoragePath(); StorageOptions options; options.dataDir = dataDir_; @@ -73,6 +78,9 @@ class TestTrash : public ::testing::Test { std::make_shared(kvStorage_, nameGenerator, 0); trashManager_ = std::make_shared(); logIndex_ = 0; + copysetNode_ = std::make_shared(); + mdsClient_ = std::make_shared(); + s3Adaptor_ = std::make_shared(); } void TearDown() override { @@ -143,65 +151,81 @@ class TestTrash : public ::testing::Test { std::shared_ptr kvStorage_; std::shared_ptr inodeStorage_; std::shared_ptr trashManager_; + std::shared_ptr copysetNode_; + std::shared_ptr mdsClient_; + std::shared_ptr s3Adaptor_; int64_t logIndex_; }; TEST_F(TestTrash, testAdd3ItemAndDelete) { TrashOption option; - option.mdsClient = std::make_shared(); - option.s3Adaptor = std::make_shared(); + option.mdsClient = mdsClient_; + option.s3Adaptor = s3Adaptor_; + FLAGS_trash_scanPeriodSec = 1; + FLAGS_trash_expiredAfterSec = 1; trashManager_->Init(option); trashManager_->Run(); - auto trash1 = std::make_shared(inodeStorage_); - auto trash2 = std::make_shared(inodeStorage_); + auto trash1 = std::make_shared(inodeStorage_, 1, 1, 1, 1); + auto trash2 = std::make_shared(inodeStorage_, 2, 2, 2, 2); trashManager_->Add(1, trash1); trashManager_->Add(2, trash2); + trash1->SetCopysetNode(copysetNode_); + trash2->SetCopysetNode(copysetNode_); inodeStorage_->Insert(GenInodeHasChunks(1, 1), logIndex_++); inodeStorage_->Insert(GenInodeHasChunks(1, 2), logIndex_++); inodeStorage_->Insert(GenInodeHasChunks(2, 1), logIndex_++); - ASSERT_EQ(inodeStorage_->Size(), 3); - trash1->Add(1, 1, 0); - trash1->Add(1, 2, 0); - trash2->Add(2, 1, 0); - - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::list list; - - trashManager_->ListItems(&list); - - ASSERT_EQ(0, list.size()); + EXPECT_CALL(*copysetNode_, IsLeaderTerm()) + .WillRepeatedly(Return(true)); + FsInfo fsInfo; + fsInfo.set_fsid(1); + fsInfo.set_recycletimehour(0); + EXPECT_CALL(*mdsClient_, GetFsInfo(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fsInfo), Return(FSStatusCode::OK))); + fsInfo.set_fsid(2); + EXPECT_CALL(*mdsClient_, GetFsInfo(2, _)) + .WillOnce(DoAll(SetArgPointee<1>(fsInfo), Return(FSStatusCode::OK))); + EXPECT_CALL(*s3Adaptor_, GetS3ClientAdaptorOption(_)) + .Times(3); + EXPECT_CALL(*s3Adaptor_, Reinit(_, _, _, _, _)) + .Times(3); + EXPECT_CALL(*s3Adaptor_, Delete(_)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(*copysetNode_, Propose(_)) + .WillOnce(Invoke([&](const braft::Task& task) { + ASSERT_EQ(inodeStorage_->Delete(Key4Inode(1, 1), logIndex_++), + MetaStatusCode::OK); + LOG(INFO) << "trash deleteInode 1:1"; + task.done->Run(); + })) + .WillOnce(Invoke([&](const braft::Task& task) { + ASSERT_EQ(inodeStorage_->Delete(Key4Inode(1, 2), logIndex_++), + MetaStatusCode::OK); + LOG(INFO) << "trash deleteInode 1:2"; + task.done->Run(); + })) + .WillOnce(Invoke([&](const braft::Task& task) { + ASSERT_EQ(inodeStorage_->Delete(Key4Inode(2, 1), logIndex_++), + MetaStatusCode::OK); + LOG(INFO) << "trash deleteInode 2:1"; + task.done->Run(); + })); + + uint64_t dtime = curve::common::TimeUtility::GetTimeofDaySec() - 2; + trash1->Add(1, dtime); + trash1->Add(2, dtime); + trash2->Add(1, dtime); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + ASSERT_EQ(0, trashManager_->Size()); ASSERT_EQ(inodeStorage_->Size(), 0); trashManager_->Fini(); } -TEST_F(TestTrash, testAdd3ItemAndNoDelete) { - TrashOption option; - option.mdsClient = std::make_shared(); - option.s3Adaptor = std::make_shared(); - trashManager_->Init(option); - trashManager_->Run(); - - auto trash1 = std::make_shared(inodeStorage_); - trashManager_->Add(1, trash1); - - inodeStorage_->Insert(GenInode(1, 1), logIndex_++); - inodeStorage_->Insert(GenInode(1, 2), logIndex_++); - inodeStorage_->Insert(GenInode(2, 1), logIndex_++); - ASSERT_EQ(inodeStorage_->Size(), 3); - trash1->Add(1, 1, 0); - trash1->Add(1, 2, 0); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::list list; - - trashManager_->ListItems(&list); - ASSERT_EQ(0, list.size()); - ASSERT_EQ(inodeStorage_->Size(), 3); - trashManager_->Fini(); -} - } // namespace metaserver } // namespace curvefs