From 2c618ca32361b24937830aef186982eec036c1ed Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Fri, 1 Dec 2023 22:18:09 +0800 Subject: [PATCH] curvefs/metaserver: fix trash bugs --- .../src/metaserver/copyset/copyset_node.cpp | 4 +- curvefs/src/metaserver/inode_manager.cpp | 17 +++-- curvefs/src/metaserver/inode_storage.cpp | 67 +++++++++++++++- curvefs/src/metaserver/inode_storage.h | 16 ++++ curvefs/src/metaserver/metastore.cpp | 26 +++++++ curvefs/src/metaserver/metastore.h | 2 + curvefs/src/metaserver/storage/converter.cpp | 1 + .../metaserver/storage/rocksdb_storage.cpp | 76 +++++++++++++++++++ .../src/metaserver/storage/rocksdb_storage.h | 33 ++++++++ curvefs/src/metaserver/storage/storage.h | 19 +++++ curvefs/src/metaserver/trash.cpp | 35 +++++++-- curvefs/src/metaserver/trash.h | 12 ++- curvefs/src/metaserver/trash_manager.cpp | 1 + curvefs/src/metaserver/trash_manager.h | 8 ++ .../test/metaserver/inode_storage_test.cpp | 21 +++++ curvefs/test/metaserver/trash_test.cpp | 10 +-- 16 files changed, 324 insertions(+), 24 deletions(-) diff --git a/curvefs/src/metaserver/copyset/copyset_node.cpp b/curvefs/src/metaserver/copyset/copyset_node.cpp index 449886204f..44ccaba398 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node.cpp @@ -170,8 +170,8 @@ bool CopysetNode::Start() { LOG(ERROR) << "Fail to init raft node, copyset: " << name_; return false; } - - LOG(INFO) << "Run copyset success, copyset: " << name_; + metaStore_->LoadDeletedInodes(); + LOG(INFO) << "Run copyset start success, copyset: " << name_; return true; } diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index 9fc94643a4..adae6a6f8b 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -312,7 +312,7 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId, MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, int64_t logIndex) { CHECK_APPLIED(); - VLOG(9) << "update inode, fsid: " << request.fsid() + VLOG(0) << "update inode, fsid: " << request.fsid() << ", inodeid: " << request.inodeid(); NameLockGuard lg(inodeLock_, GetInodeLockName(request.fsid(), request.inodeid())); @@ -388,11 +388,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, } } - if (s3NeedTrash) { - trash_->Add(old.fsid(), old.inodeid(), old.dtime()); - --(*type2InodeNum_)[old.type()]; - } - const S3ChunkInfoMap &map2add = request.s3chunkinfoadd(); const S3ChunkInfoList *list2add; VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size " @@ -443,7 +438,15 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, return MetaStatusCode::STORAGE_INTERNAL_ERROR; } } - VLOG(9) << "UpdateInode success, " << request.ShortDebugString(); + + if (s3NeedTrash) { + inodeStorage_->UpdateDeletingKey(old, logIndex); + // wuhongsong：需要打开 + // trash_->Add(old.fsid(), old.inodeid(), old.dtime()); + --(*type2InodeNum_)[old.type()]; + } + + VLOG(0) << "UpdateInode success, " << request.ShortDebugString(); return MetaStatusCode::OK; } diff --git a/curvefs/src/metaserver/inode_storage.cpp b/curvefs/src/metaserver/inode_storage.cpp index 8e5da8a913..68d49642d8 100644 --- a/curvefs/src/metaserver/inode_storage.cpp +++ b/curvefs/src/metaserver/inode_storage.cpp @@ -185,9 +185,74 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) { return MetaStatusCode::STORAGE_INTERNAL_ERROR; } +MetaStatusCode InodeStorage::UpdateDeletingKey( + const Inode& inode, int64_t logIndex) { + WriteLockGuard lg(rwLock_); + Key4Inode key(inode.fsid(), inode.inodeid()); + std::string skey = conv_.SerializeToString(key); + VLOG(9) << "update deleting key, " << inode.inodeid(); + const char* step = "Begin transaction"; + std::shared_ptr txn; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + auto rc = txn->HSetDeleting(table4Inode_, skey , inode); + step = "insert inode "; + if (rc.ok()) { + rc = SetAppliedIndex(txn.get(), logIndex); + step = "Insert applied index to transaction"; + } + if (rc.ok()) { + rc = txn->Commit(); + step = "commit"; + } + if (rc.ok()) { + VLOG(0) << "set deleting key ok"; + return MetaStatusCode::OK; + } + LOG(ERROR) << step << "failed, status = " << rc.ToString(); + if (txn != nullptr && !txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << rc.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + +MetaStatusCode InodeStorage::ClearDelKey(const Key4Inode& key) { + WriteLockGuard lg(rwLock_); + std::string skey = conv_.SerializeToString(key); + VLOG(9) << "clear deleting key start, " << skey; + std::shared_ptr txn = nullptr; + const char* step = "Begin transaction"; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + step = "Delete inode from transaction"; + auto s = txn->HClearDeleting(table4Inode_, skey); + if (s.ok()) { + step = "Delete inode"; + s = txn->Commit(); + } + if (s.ok()) { + VLOG(9) << "clear deleting key ok, " << skey; + return MetaStatusCode::OK; + } + LOG(ERROR) << step << " failed, status = " << s.ToString(); + if (!txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << s.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) { ReadLockGuard lg(rwLock_); std::string skey = conv_.SerializeToString(key); + Status s = kvStorage_->HGet(table4Inode_, skey, inode); if (s.ok()) { return MetaStatusCode::OK; @@ -471,7 +536,6 @@ MetaStatusCode InodeStorage::Clear() { // because if we fail stop, we will replay // raft logs and clear it again WriteLockGuard lg(rwLock_); - Status s = kvStorage_->HClear(table4Inode_); if (!s.ok()) { LOG(ERROR) << "InodeStorage clear inode table failed, status = " @@ -492,7 +556,6 @@ MetaStatusCode InodeStorage::Clear() { << s.ToString(); return MetaStatusCode::STORAGE_INTERNAL_ERROR; } - s = kvStorage_->HClear(table4InodeAuxInfo_); if (!s.ok()) { LOG(ERROR) diff --git a/curvefs/src/metaserver/inode_storage.h b/curvefs/src/metaserver/inode_storage.h index 38ad3c5f56..cd0559cfe7 100644 --- a/curvefs/src/metaserver/inode_storage.h +++ b/curvefs/src/metaserver/inode_storage.h @@ -43,6 +43,7 @@ #include "curvefs/src/metaserver/storage/utils.h" #include "src/common/concurrent/rw_lock.h" +#define DELETING_PREFIX "deleting_" namespace curvefs { namespace metaserver { @@ -76,6 +77,16 @@ class InodeStorage { */ MetaStatusCode Insert(const Inode& inode, int64_t logIndex); + /** + * @brief update deleting inode key in storage + * @param[in] inode: the inode want to update + * @param[in] logIndex: the index of raft log + * @return + */ + MetaStatusCode UpdateDeletingKey(const Inode& inode, int64_t logIndex); + + MetaStatusCode ClearDelKey(const Key4Inode& key); + /** * @brief get inode from storage * @param[in] key: the key of inode want to get @@ -187,6 +198,11 @@ class InodeStorage { MetaStatusCode GetAllBlockGroup( std::vector* deallocatableBlockGroupVec); +/* + std::shared_ptr GetKVStorage() { + return kvStorage_; + } +*/ private: MetaStatusCode UpdateInodeS3MetaSize(Transaction txn, uint32_t fsId, uint64_t inodeId, uint64_t size4add, diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index a2fd20323c..4175e54c38 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include // NOLINT #include @@ -60,6 +61,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage; using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList; using ::curvefs::metaserver::storage::MemoryStorage; +using ::curvefs::metaserver::storage::NameGenerator; using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::StorageOptions; @@ -147,6 +149,7 @@ bool MetaStoreImpl::Load(const std::string &pathname) { } startCompacts(); + return true; } @@ -866,5 +869,28 @@ bool MetaStoreImpl::InitStorage() { return kvStorage_->Open(); } +void MetaStoreImpl::LoadDeletedInodes() { + VLOG(3) << "LoadDeletedInodes start"; + std::list items; + kvStorage_->LoadDeletedInodes(items); + VLOG(3) << "build trash items size: " << items.size(); + std::vector names; + for (auto iter : items) { + curve::common::SplitString(iter , ":", &names); + char* tmp = const_cast(names[2].c_str()); + uint32_t partitionId = *reinterpret_cast(tmp); + VLOG(3) << "partitionId is: " << partitionId; + std::shared_ptr trash = + TrashManager::GetInstance().GetTrash(partitionId); + if (nullptr == trash) { + LOG(INFO) << "trash is null: " << partitionId; + continue; + } + trash->Add(std::stoul(names[names.size() - 2 ]), + std::stoull(names[names.size() - 1 ]), 0, true); + } + VLOG(3) << "LoadDeletedInodes over."; +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/metastore.h b/curvefs/src/metaserver/metastore.h index a13c0a4980..9569526a2b 100644 --- a/curvefs/src/metaserver/metastore.h +++ b/curvefs/src/metaserver/metastore.h @@ -117,6 +117,7 @@ class MetaStore { virtual bool SaveData(const std::string& dir, std::vector* files) = 0; virtual bool Clear() = 0; + virtual void LoadDeletedInodes() {} virtual bool Destroy() = 0; virtual MetaStatusCode CreatePartition( const CreatePartitionRequest* request, @@ -223,6 +224,7 @@ class MetaStoreImpl : public MetaStore { std::vector* files) override; bool Clear() override; bool Destroy() override; + void LoadDeletedInodes() override; MetaStatusCode CreatePartition(const CreatePartitionRequest* request, CreatePartitionResponse* response, diff --git a/curvefs/src/metaserver/storage/converter.cpp b/curvefs/src/metaserver/storage/converter.cpp index 2e48b20d50..6185fe4e87 100644 --- a/curvefs/src/metaserver/storage/converter.cpp +++ b/curvefs/src/metaserver/storage/converter.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.cpp b/curvefs/src/metaserver/storage/rocksdb_storage.cpp index 5875ba6817..bf94183c02 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.cpp +++ b/curvefs/src/metaserver/storage/rocksdb_storage.cpp @@ -26,6 +26,7 @@ #include #include +#include "src/common/string_util.h" #include "src/common/timeutility.h" #include "curvefs/src/metaserver/storage/utils.h" #include "curvefs/src/metaserver/storage/storage.h" @@ -218,6 +219,7 @@ Status RocksDBStorage::Get(const std::string& name, if (s.ok() && !value->ParseFromString(svalue)) { return Status::ParsedFailed(); } + return ToStorageStatus(s); } @@ -241,6 +243,47 @@ Status RocksDBStorage::Set(const std::string& name, return ToStorageStatus(s); } +Status RocksDBStorage::SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered) { + std::string svalue; + if (!inited_) { + return Status::DBClosed(); + } else if (!value.SerializeToString(&svalue)) { + return Status::SerializedFailed(); + } + + auto handle = GetColumnFamilyHandle(ordered); + std::string ikey = ToInternalKey(name, key, ordered); + std::string deletingKey = "deleting_" + ikey; + VLOG(9) << "whs set deleting key = " << deletingKey + << ", ikey " << ikey << ", " << options_.dataDir; + RocksDBPerfGuard guard(OP_PUT); + ROCKSDB_NAMESPACE::Status s = InTransaction_ ? + txn_->Put(handle, deletingKey, svalue) : + db_->Put(dbWriteOptions_, handle, deletingKey, svalue); + return ToStorageStatus(s); +} + +Status RocksDBStorage::ClearDeleting(const std::string& name, + const std::string& key, + bool ordered) { + if (!inited_) { + return Status::DBClosed(); + } + std::string ikey = ToInternalKey(name, key, ordered); + std::string deletingKey = "deleting_" + ikey; + VLOG(9) << "whs clear deleting key = " << deletingKey + << ", ikey " << ikey << ", " << options_.dataDir; + auto handle = GetColumnFamilyHandle(ordered); + RocksDBPerfGuard guard(OP_DELETE); + ROCKSDB_NAMESPACE::Status s = InTransaction_ ? + txn_->Delete(handle, deletingKey) : + db_->Delete(dbWriteOptions_, handle, deletingKey); + return ToStorageStatus(s); +} + Status RocksDBStorage::Del(const std::string& name, const std::string& key, bool ordered) { @@ -273,6 +316,14 @@ std::shared_ptr RocksDBStorage::GetAll(const std::string& name, this, std::move(ikey), 0, status, ordered); } +std::shared_ptr RocksDBStorage::GetPrefix(const std::string& prefix, + bool ordered) { + int status = inited_ ? 0 : -1; + return std::make_shared( + this, std::move(prefix), 0, status, ordered); +} + + size_t RocksDBStorage::Size(const std::string& name, bool ordered) { auto iterator = GetAll(name, ordered); if (iterator->Status() != 0) { @@ -505,6 +556,31 @@ bool RocksDBStorage::Recover(const std::string& dir) { return true; } +void RocksDBStorage::LoadDeletedInodes(std::list& item) { + LOG(INFO) << "LoadAll storage from: " << options_.dataDir; + std::string sprefix = "deleting_"; + int counts = 0; + rocksdb::Iterator* it1 = db_->NewIterator(rocksdb::ReadOptions()); + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key = it1->key().ToString(); + if (curve::common::StringStartWith(key, sprefix)) { + VLOG(3) << "load key: " << key; + item.push_back(key); + } + VLOG(3) << "load key: " << key; + counts++; + } + auto iter = GetPrefix(sprefix, false); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string key = iter->Key(); + // whs item.push_back(key); + VLOG(3) << "load key: " << key; + counts++; + } + LOG(INFO) << "LoadAll storage from. size: " << item.size() + << ", " << counts << ", " << options_.dataDir; +} + } // namespace storage } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.h b/curvefs/src/metaserver/storage/rocksdb_storage.h index e0023dd8e2..6ef22bfdc2 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.h +++ b/curvefs/src/metaserver/storage/rocksdb_storage.h @@ -23,6 +23,7 @@ #ifndef CURVEFS_SRC_METASERVER_STORAGE_ROCKSDB_STORAGE_H_ #define CURVEFS_SRC_METASERVER_STORAGE_ROCKSDB_STORAGE_H_ +#include #include #include #include @@ -79,6 +80,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { bool Close() override; + void LoadDeletedInodes(std::list& item) override; + STORAGE_TYPE Type() override; StorageOptions GetStorageOptions() const override; @@ -92,6 +95,13 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const std::string& key, const ValueType& value) override; + Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) override; + + Status HClearDeleting(const std::string& name, + const std::string& key) override; + Status HDel(const std::string& name, const std::string& key) override; std::shared_ptr HGetAll(const std::string& name) override; @@ -100,6 +110,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { Status HClear(const std::string& name) override; + std::shared_ptr GetPrefix(const std::string& prefix, + bool ordered) override; // ordered Status SGet(const std::string& name, const std::string& key, @@ -156,6 +168,16 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const ValueType& value, bool ordered); + Status SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered); + + Status ClearDeleting(const std::string& name, + const std::string& key, + bool ordered); + + Status Del(const std::string& name, const std::string& key, bool ordered); @@ -219,6 +241,17 @@ inline Status RocksDBStorage::HSet(const std::string& name, return Set(name, key, value, false); } +inline Status RocksDBStorage::HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) { + return SetDeleting(name, key, value, false); +} + +inline Status RocksDBStorage::HClearDeleting(const std::string& name, + const std::string& key) { + return ClearDeleting(name, key, false); +} + inline Status RocksDBStorage::HDel(const std::string& name, const std::string& key) { return Del(name, key, false); diff --git a/curvefs/src/metaserver/storage/storage.h b/curvefs/src/metaserver/storage/storage.h index 97cef01fca..b8410ce6c3 100644 --- a/curvefs/src/metaserver/storage/storage.h +++ b/curvefs/src/metaserver/storage/storage.h @@ -23,6 +23,7 @@ #ifndef CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ #define CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ +#include #include #include #include @@ -51,6 +52,22 @@ class BaseStorage { const std::string& key, const ValueType& value) = 0; + virtual Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) { + return Status::NotSupported(); + } + + virtual Status HClearDeleting(const std::string& name, + const std::string& key) { + return Status::NotSupported(); + } + + virtual std::shared_ptr GetPrefix( + const std::string& prefix, bool ordered) { + return nullptr; + } + virtual Status HDel(const std::string& name, const std::string& key) = 0; virtual std::shared_ptr HGetAll(const std::string& name) = 0; @@ -101,6 +118,8 @@ class KVStorage : public BaseStorage { virtual bool Close() = 0; + virtual void LoadDeletedInodes(std::list& item) {} + virtual StorageOptions GetStorageOptions() const = 0; virtual std::shared_ptr BeginTransaction() = 0; diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index 1175376e72..a2d00146f3 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -55,20 +55,23 @@ void TrashImpl::Init(const TrashOption &option) { isStop_ = false; } -void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) { +void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, + uint32_t dtime, bool deleted) { TrashItem item; item.fsId = fsId; item.inodeId = inodeId; item.dtime = dtime; + item.deleted = deleted; LockGuard lg(itemsMutex_); if (isStop_) { return; } trashItems_.push_back(item); - VLOG(6) << "Add Trash Item success, item.fsId = " << item.fsId + VLOG(6) << "Add trash item success, item.fsId = " << item.fsId << ", item.inodeId = " << item.inodeId - << ", item.dtime = " << item.dtime; + << ", item.dtime = " << item.dtime + << ", item.deleted = " << item.deleted; } void TrashImpl::ScanTrash() { @@ -81,11 +84,19 @@ void TrashImpl::ScanTrash() { for (auto it = temp.begin(); it != temp.end();) { if (isStop_) { + LOG(INFO) << "ScanTrash stop."; return; } + VLOG(9) << "ScanTrash , " << "item.fsId = " << it->fsId + << ", item.inodeId = " << it->inodeId + << ", item.dtime = " << it->dtime + << ", item.deleted = " << it->deleted; if (NeedDelete(*it)) { MetaStatusCode ret = DeleteInodeAndData(*it); if (MetaStatusCode::NOT_FOUND == ret) { + VLOG(3) << "ScanTrash, inode not exist, fsId = " << it->fsId + << ", inodeId = " << it->inodeId; + ClearDeleted(*it); it = temp.erase(it); continue; } @@ -96,10 +107,13 @@ void TrashImpl::ScanTrash() { it++; continue; } - VLOG(6) << "Trash Delete Inode, fsId = " << it->fsId + VLOG(9) << "Trash Delete Inode, fsId = " << it->fsId << ", inodeId = " << it->inodeId; + ClearDeleted(*it); it = temp.erase(it); } else { + VLOG(9) << "ScanTrash, inode not expired, fsId = " << it->fsId + << ", inodeId = " << it->inodeId; it++; } } @@ -110,6 +124,14 @@ void TrashImpl::ScanTrash() { } } +void TrashImpl::ClearDeleted(const TrashItem &item) { + if (item.deleted) { + VLOG(9) << "ClearDeleted: " << item.fsId << ", " << item.inodeId + << ", " << item.dtime << ", " << item.deleted; + inodeStorage_->ClearDelKey(Key4Inode(item.fsId, item.inodeId)); + } +} + void TrashImpl::StopScan() { isStop_ = true; } @@ -176,7 +198,8 @@ uint64_t TrashImpl::GetFsRecycleTimeHour(uint32_t fsId) { return recycleTimeHour; } -MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) { +MetaStatusCode TrashImpl:: +DeleteInodeAndData(const TrashItem &item) { Inode inode; MetaStatusCode ret = inodeStorage_->Get(Key4Inode(item.fsId, item.inodeId), &inode); @@ -230,6 +253,8 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) { if (inode.s3chunkinfomap().empty()) { LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId << ", inodeId = " << item.inodeId; + // Todo(hzwuhongsong ：the s3chunkinfomap is empty too as empty file + // empty file will never be deleted return MetaStatusCode::NOT_FOUND; } VLOG(9) << "DeleteInodeAndData, inode: " diff --git a/curvefs/src/metaserver/trash.h b/curvefs/src/metaserver/trash.h index de9410ddc3..9c7bb215f8 100644 --- a/curvefs/src/metaserver/trash.h +++ b/curvefs/src/metaserver/trash.h @@ -51,10 +51,12 @@ struct TrashItem { uint32_t fsId; uint64_t inodeId; uint32_t dtime; + bool deleted; TrashItem() : fsId(0), inodeId(0), - dtime(0) {} + dtime(0), + deleted(false) {} }; struct TrashOption { @@ -78,7 +80,8 @@ class Trash { virtual void Init(const TrashOption &option) = 0; - virtual void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) = 0; + virtual void Add(uint32_t fsId, uint64_t inodeId, + uint32_t dtime, bool deleted = false) = 0; virtual void ListItems(std::list *items) = 0; @@ -98,7 +101,8 @@ class TrashImpl : public Trash { void Init(const TrashOption &option) override; - void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) override; + void Add(uint32_t fsId, uint64_t inodeId, + uint32_t dtime, bool deleted) override; void ListItems(std::list *items) override; @@ -115,6 +119,8 @@ class TrashImpl : public Trash { uint64_t GetFsRecycleTimeHour(uint32_t fsId); + void ClearDeleted(const TrashItem &item); + private: std::shared_ptr inodeStorage_; std::shared_ptr s3Adaptor_; diff --git a/curvefs/src/metaserver/trash_manager.cpp b/curvefs/src/metaserver/trash_manager.cpp index 7f6341db1c..cbb18fbe19 100644 --- a/curvefs/src/metaserver/trash_manager.cpp +++ b/curvefs/src/metaserver/trash_manager.cpp @@ -56,6 +56,7 @@ void TrashManager::ScanLoop() { } void TrashManager::ScanEveryTrash() { + VLOG(0) << "whs scan trash start."; std::map> temp; { curve::common::ReadLockGuard lg(rwLock_); diff --git a/curvefs/src/metaserver/trash_manager.h b/curvefs/src/metaserver/trash_manager.h index ca25c0145d..b35107ab6f 100644 --- a/curvefs/src/metaserver/trash_manager.h +++ b/curvefs/src/metaserver/trash_manager.h @@ -27,6 +27,8 @@ #include #include +#include "src/common/string_util.h" + #include "src/common/concurrent/concurrent.h" #include "curvefs/src/metaserver/trash.h" @@ -52,6 +54,11 @@ class TrashManager { << partitionId; } + std::shared_ptr GetTrash(uint32_t partitionId) { + curve::common::ReadLockGuard lg(rwLock_); + return trashs_.find(partitionId)->second; + } + void Remove(uint32_t partitionId); void Init(const TrashOption &options) { @@ -79,6 +86,7 @@ class TrashManager { InterruptibleSleeper sleeper_; std::map> trashs_; + curve::common::RWLock rwLock_; }; diff --git a/curvefs/test/metaserver/inode_storage_test.cpp b/curvefs/test/metaserver/inode_storage_test.cpp index 4ea0bc3f54..ea68d160e7 100644 --- a/curvefs/test/metaserver/inode_storage_test.cpp +++ b/curvefs/test/metaserver/inode_storage_test.cpp @@ -1033,5 +1033,26 @@ TEST_F(InodeStorageTest, Test_UpdateDeallocatableBlockGroup) { ASSERT_EQ(1, deallocatableBlockGroupVec.size()); } +TEST_F(InodeStorageTest, test_deleting_key) { + InodeStorage storage(kvStorage_, nameGenerator_, 0); + ASSERT_TRUE(storage.Init()); + Inode inode1 = GenInode(1, 1); + Inode inode2 = GenInode(2, 2); + + // insert + ASSERT_EQ(storage.Insert(inode1, logIndex_++), MetaStatusCode::OK); + ASSERT_EQ(storage.Insert(inode2, logIndex_++), MetaStatusCode::OK); + + ASSERT_EQ(storage.UpdateDeletingKey( + inode1,logIndex_++), MetaStatusCode::OK); + ASSERT_EQ(storage.ClearDelKey(Key4Inode( + inode1.fsid(), inode1.inodeid())), MetaStatusCode::OK); + + ASSERT_EQ(storage.UpdateDeletingKey( + inode2, logIndex_++), MetaStatusCode::OK); + ASSERT_EQ(storage.ClearDelKey(Key4Inode( + inode2.fsid(), inode2.inodeid())), MetaStatusCode::OK); +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/test/metaserver/trash_test.cpp b/curvefs/test/metaserver/trash_test.cpp index b4b792e0a7..301ef5b2e5 100644 --- a/curvefs/test/metaserver/trash_test.cpp +++ b/curvefs/test/metaserver/trash_test.cpp @@ -163,9 +163,9 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) { ASSERT_EQ(inodeStorage_->Size(), 3); - trash1->Add(1, 1, 0); - trash1->Add(1, 2, 0); - trash2->Add(2, 1, 0); + trash1->Add(1, 1, 0, false); + trash1->Add(1, 2, 0, false); + trash2->Add(2, 1, 0, false); std::this_thread::sleep_for(std::chrono::seconds(5)); std::list list; @@ -192,8 +192,8 @@ TEST_F(TestTrash, testAdd3ItemAndNoDelete) { inodeStorage_->Insert(GenInode(1, 2), logIndex_++); inodeStorage_->Insert(GenInode(2, 1), logIndex_++); ASSERT_EQ(inodeStorage_->Size(), 3); - trash1->Add(1, 1, 0); - trash1->Add(1, 2, 0); + trash1->Add(1, 1, 0, false); + trash1->Add(1, 2, 0, false); std::this_thread::sleep_for(std::chrono::seconds(5)); std::list list;