From 7cb5bd1dceb8ac52c80589079f6349659ea7a045 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Tue, 28 Nov 2023 11:03:38 +0800 Subject: [PATCH] curvefs/metaserver: fix trash bugs --- .../src/metaserver/copyset/copyset_node.cpp | 5 +- curvefs/src/metaserver/inode_manager.cpp | 17 ++-- curvefs/src/metaserver/inode_storage.cpp | 73 ++++++++++++++- curvefs/src/metaserver/inode_storage.h | 14 +++ curvefs/src/metaserver/metastore.cpp | 39 +++++++- curvefs/src/metaserver/metastore.h | 2 + curvefs/src/metaserver/storage/converter.cpp | 29 ++++++ .../metaserver/storage/rocksdb_storage.cpp | 90 ++++++++++++++++++- .../src/metaserver/storage/rocksdb_storage.h | 32 +++++++ curvefs/src/metaserver/storage/storage.h | 19 ++++ curvefs/src/metaserver/trash.cpp | 59 ++++++++++-- curvefs/src/metaserver/trash.h | 10 ++- curvefs/src/metaserver/trash_manager.cpp | 1 + curvefs/src/metaserver/trash_manager.h | 8 ++ 14 files changed, 376 insertions(+), 22 deletions(-) diff --git a/curvefs/src/metaserver/copyset/copyset_node.cpp b/curvefs/src/metaserver/copyset/copyset_node.cpp index 449886204f..d327d6b82e 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node.cpp @@ -161,6 +161,7 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) { } bool CopysetNode::Start() { + VLOG(3) << "copyset is starting, copyset: " << name_; if (!raftNode_) { LOG(ERROR) << "RaftNode didn't created, copyset: " << name_; return false; @@ -170,8 +171,8 @@ bool CopysetNode::Start() { LOG(ERROR) << "Fail to init raft node, copyset: " << name_; return false; } - - LOG(INFO) << "Run copyset success, copyset: " << name_; + metaStore_->LoadDeletedInodes(); + VLOG(3) << "copyset start success, copyset: " << name_; return true; } diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index 9fc94643a4..adae6a6f8b 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -312,7 +312,7 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId, MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, int64_t logIndex) { CHECK_APPLIED(); - VLOG(9) << "update inode, fsid: " << request.fsid() + VLOG(0) << "update inode, fsid: " << request.fsid() << ", inodeid: " << request.inodeid(); NameLockGuard lg(inodeLock_, GetInodeLockName(request.fsid(), request.inodeid())); @@ -388,11 +388,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, } } - if (s3NeedTrash) { - trash_->Add(old.fsid(), old.inodeid(), old.dtime()); - --(*type2InodeNum_)[old.type()]; - } - const S3ChunkInfoMap &map2add = request.s3chunkinfoadd(); const S3ChunkInfoList *list2add; VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size " @@ -443,7 +438,15 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, return MetaStatusCode::STORAGE_INTERNAL_ERROR; } } - VLOG(9) << "UpdateInode success, " << request.ShortDebugString(); + + if (s3NeedTrash) { + inodeStorage_->UpdateDeletingKey(old, logIndex); + // wuhongsong：需要打开 + // trash_->Add(old.fsid(), old.inodeid(), old.dtime()); + --(*type2InodeNum_)[old.type()]; + } + + VLOG(0) << "UpdateInode success, " << request.ShortDebugString(); return MetaStatusCode::OK; } diff --git a/curvefs/src/metaserver/inode_storage.cpp b/curvefs/src/metaserver/inode_storage.cpp index 8e5da8a913..14cc912a1e 100644 --- a/curvefs/src/metaserver/inode_storage.cpp +++ b/curvefs/src/metaserver/inode_storage.cpp @@ -185,6 +185,77 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) { return MetaStatusCode::STORAGE_INTERNAL_ERROR; } +MetaStatusCode InodeStorage::UpdateDeletingKey(const Inode& inode, int64_t logIndex) { + WriteLockGuard lg(rwLock_); + Key4Inode key(inode.fsid(), inode.inodeid()); + std::string skey = conv_.SerializeToString(key); + VLOG(0) << "whs update deleting key, " << inode.inodeid(); + const char* step = "Begin transaction"; + std::shared_ptr txn; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + auto rc = txn->HSetDeleting(table4Inode_, skey , inode); + step = "insert inode "; + if (rc.ok()) { + // delete key + // rc = DeleteInternal(txn.get(), key); + // 这里的删除只能最后删除，不然在DeleteInodeAndData中有问题 + // rc = txn->HDel(table4Inode_, skey); + step = "delete inode "; + } + if (rc.ok()) { + rc = SetAppliedIndex(txn.get(), logIndex); + step = "Insert applied index to transaction"; + } + if (rc.ok()) { + rc = txn->Commit(); + step = "commit"; + } + if (rc.ok()) { + VLOG(0) << "set deleting key ok"; + return MetaStatusCode::OK; + } + LOG(ERROR) << step << "failed, status = " << rc.ToString(); + if (txn != nullptr && !txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << rc.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + +MetaStatusCode InodeStorage::ClearDelKey(const Key4Inode& key) { + WriteLockGuard lg(rwLock_); + std::string skey = conv_.SerializeToString(key); + VLOG(0) << "clear deleting key start, " << skey; + std::shared_ptr txn = nullptr; + const char* step = "Begin transaction"; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + step = "Delete inode from transaction"; + auto s = txn->HClearDeleting(table4Inode_, skey); + if (s.ok()) { + step = "Delete inode"; + s = txn->Commit(); + } + if (s.ok()) { + VLOG(0) << "clear deleting key ok"; + return MetaStatusCode::OK; + } + LOG(ERROR) << step << " failed, status = " << s.ToString(); + if (txn != nullptr && !txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << s.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + +} + MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) { ReadLockGuard lg(rwLock_); std::string skey = conv_.SerializeToString(key); @@ -471,7 +542,6 @@ MetaStatusCode InodeStorage::Clear() { // because if we fail stop, we will replay // raft logs and clear it again WriteLockGuard lg(rwLock_); - Status s = kvStorage_->HClear(table4Inode_); if (!s.ok()) { LOG(ERROR) << "InodeStorage clear inode table failed, status = " @@ -492,7 +562,6 @@ MetaStatusCode InodeStorage::Clear() { << s.ToString(); return MetaStatusCode::STORAGE_INTERNAL_ERROR; } - s = kvStorage_->HClear(table4InodeAuxInfo_); if (!s.ok()) { LOG(ERROR) diff --git a/curvefs/src/metaserver/inode_storage.h b/curvefs/src/metaserver/inode_storage.h index 38ad3c5f56..87cef76f0f 100644 --- a/curvefs/src/metaserver/inode_storage.h +++ b/curvefs/src/metaserver/inode_storage.h @@ -43,6 +43,7 @@ #include "curvefs/src/metaserver/storage/utils.h" #include "src/common/concurrent/rw_lock.h" +#define DELETING_PREFIX "deleting_" namespace curvefs { namespace metaserver { @@ -76,6 +77,15 @@ class InodeStorage { */ MetaStatusCode Insert(const Inode& inode, int64_t logIndex); + /** + * @brief update deleting inode key in storage + * @param[in] inode: the inode want to update + * @param[in] logIndex: the index of raft log + * @return + */ + MetaStatusCode UpdateDeletingKey(const Inode& inode, int64_t logIndex); + + MetaStatusCode ClearDelKey(const Key4Inode& key); /** * @brief get inode from storage * @param[in] key: the key of inode want to get @@ -187,6 +197,10 @@ class InodeStorage { MetaStatusCode GetAllBlockGroup( std::vector* deallocatableBlockGroupVec); + std::shared_ptr GetKVStorage() { + return kvStorage_; + } + private: MetaStatusCode UpdateInodeS3MetaSize(Transaction txn, uint32_t fsId, uint64_t inodeId, uint64_t size4add, diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index a2fd20323c..1511530c3e 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include // NOLINT #include @@ -60,6 +61,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage; using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList; using ::curvefs::metaserver::storage::MemoryStorage; +using ::curvefs::metaserver::storage::NameGenerator; using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::StorageOptions; @@ -87,6 +89,8 @@ MetaStoreImpl::MetaStoreImpl(copyset::CopysetNode *node, storageOptions_(storageOptions) {} bool MetaStoreImpl::Load(const std::string &pathname) { +LOG(ERROR) << "whs load start"; + // Load from raft snap file to memory WriteLockGuard writeLockGuard(rwLock_); MetaStoreFStream fstream(&partitionMap_, kvStorage_, @@ -147,6 +151,9 @@ bool MetaStoreImpl::Load(const std::string &pathname) { } startCompacts(); + + +LOG(ERROR) << "whs load end"; return true; } @@ -863,7 +870,37 @@ bool MetaStoreImpl::InitStorage() { return false; } - return kvStorage_->Open(); + + if (!kvStorage_->Open()) { + return false; + } + + return true; +} + +void MetaStoreImpl::LoadDeletedInodes() { + LOG(ERROR) << "InitStorage start"; + std::list items; + // whs: 好像很慢很慢, 11条目录都加载了好久 + kvStorage_->LoadDeletedInodes(items); + VLOG(3) << "build trash items size: " << items.size(); + std::vector names; + for (auto iter : items) { + curve::common::SplitString(iter , ":", &names); + char* tmp = const_cast(names[2].c_str()); + uint32_t partitionId = *reinterpret_cast(tmp); + VLOG(0) << "whs partitionId: " << partitionId; + std::shared_ptr trash = + TrashManager::GetInstance().GetTrash(partitionId); + if (nullptr == trash) { + VLOG(0) << "whs trash is null: " << partitionId; + continue; + } + trash->Add(std::stoul(names[names.size() - 2 ]), + std::stoull(names[names.size() - 1 ]), 0, true); + } + + VLOG(3) << "build trash items over."; } } // namespace metaserver diff --git a/curvefs/src/metaserver/metastore.h b/curvefs/src/metaserver/metastore.h index a13c0a4980..89a645bee5 100644 --- a/curvefs/src/metaserver/metastore.h +++ b/curvefs/src/metaserver/metastore.h @@ -117,6 +117,7 @@ class MetaStore { virtual bool SaveData(const std::string& dir, std::vector* files) = 0; virtual bool Clear() = 0; + virtual void LoadDeletedInodes() {}; virtual bool Destroy() = 0; virtual MetaStatusCode CreatePartition( const CreatePartitionRequest* request, @@ -223,6 +224,7 @@ class MetaStoreImpl : public MetaStore { std::vector* files) override; bool Clear() override; bool Destroy() override; + void LoadDeletedInodes() override; MetaStatusCode CreatePartition(const CreatePartitionRequest* request, CreatePartitionResponse* response, diff --git a/curvefs/src/metaserver/storage/converter.cpp b/curvefs/src/metaserver/storage/converter.cpp index 2e48b20d50..a4293d9ab6 100644 --- a/curvefs/src/metaserver/storage/converter.cpp +++ b/curvefs/src/metaserver/storage/converter.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -118,9 +119,37 @@ size_t NameGenerator::GetFixedLength() { } std::string NameGenerator::Format(KEY_TYPE type, uint32_t partitionId) { + VLOG(0) << "whs NameGenerator::Format, " << partitionId; char buf[sizeof(partitionId)]; std::memcpy(buf, reinterpret_cast(&partitionId), sizeof(buf)); + VLOG(0) << "whs NameGenerator::Format1, " << buf; + + uint32_t num = *reinterpret_cast(buf); + + VLOG(0) << "whs NameGenerator::Format1.1, " << num << ", " << sizeof(buf) << ", " << sizeof(partitionId) << sizeof(int); + std::string partitionIdStr = std::to_string(partitionId); + VLOG(0) << "whs NameGenerator::Format2, " << partitionIdStr; + + absl::string_view tmp = absl::string_view(buf, sizeof(buf)); + + std::string tmp2(tmp); + VLOG(0) << "whs NameGenerator::Format4, " << tmp2; + + std::string binaryValue = std::bitset(partitionId).to_string(); + VLOG(0) << "whs NameGenerator::Format5, " << binaryValue; + + // std::bitset bitsetValue(tmp2); + // uint32_t p = bitsetValue.to_ulong(); + + + + // int p2 = bitsetValue.to_ulong(); + + + return absl::StrCat(type, kDelimiter, absl::string_view(buf, sizeof(buf))); + // return absl::StrCat(type, kDelimiter, partitionIdStr); + // return absl::StrCat(type, kDelimiter, binaryValue); } Key4Inode::Key4Inode() : fsId(0), inodeId(0) {} diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.cpp b/curvefs/src/metaserver/storage/rocksdb_storage.cpp index 5875ba6817..de62287093 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.cpp +++ b/curvefs/src/metaserver/storage/rocksdb_storage.cpp @@ -26,6 +26,7 @@ #include #include +#include "src/common/string_util.h" #include "src/common/timeutility.h" #include "curvefs/src/metaserver/storage/utils.h" #include "curvefs/src/metaserver/storage/storage.h" @@ -187,7 +188,7 @@ std::string RocksDBStorage::ToInternalKey(const std::string& name, std::ostringstream oss; oss << iname << kDelimiter_ << key; std::string ikey = oss.str(); - VLOG(9) << "ikey = " << ikey << " (ordered = " << ordered + VLOG(0) << "whs ikey = " << ikey << " (ordered = " << ordered << ", name = " << name << ", key = " << key << ")" << ", size = " << ikey.size(); return ikey; @@ -209,6 +210,20 @@ Status RocksDBStorage::Get(const std::string& name, ROCKSDB_NAMESPACE::Status s; std::string svalue; std::string ikey = ToInternalKey(name, key, ordered); +VLOG(0) << "whs get key = " << ikey << ", " << options_.dataDir << ordered; +// + LOG(INFO) << "LoadAll storage from: " << options_.dataDir; + rocksdb::Iterator* it1 = db_->NewIterator(rocksdb::ReadOptions()); + int counts = 0; + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key = it1->key().ToString(); + LOG(ERROR) << "whs get all key: " << key; + counts++; + } + LOG(INFO) << "LoadAll storage from. size: " + << ", " << counts << ", " << options_.dataDir; + +// auto handle = GetColumnFamilyHandle(ordered); { RocksDBPerfGuard guard(OP_GET); @@ -218,6 +233,7 @@ Status RocksDBStorage::Get(const std::string& name, if (s.ok() && !value->ParseFromString(svalue)) { return Status::ParsedFailed(); } + return ToStorageStatus(s); } @@ -235,12 +251,57 @@ Status RocksDBStorage::Set(const std::string& name, auto handle = GetColumnFamilyHandle(ordered); std::string ikey = ToInternalKey(name, key, ordered); RocksDBPerfGuard guard(OP_PUT); +VLOG(0) << "whs set key = " << ikey << ", " << options_.dataDir << ", " << ordered; ROCKSDB_NAMESPACE::Status s = InTransaction_ ? txn_->Put(handle, ikey, svalue) : db_->Put(dbWriteOptions_, handle, ikey, svalue); return ToStorageStatus(s); } +Status RocksDBStorage::SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered) { + std::string svalue; + if (!inited_) { + return Status::DBClosed(); + } else if (!value.SerializeToString(&svalue)) { + return Status::SerializedFailed(); + } + + auto handle = GetColumnFamilyHandle(ordered); + + + std::string ikey = ToInternalKey(name, key, ordered); + std::string deletingKey = "deleting_" + ikey; +VLOG(0) << "whs set deleting key = " << deletingKey << ", ikey " << ikey << ", " << options_.dataDir; + RocksDBPerfGuard guard(OP_PUT); + ROCKSDB_NAMESPACE::Status s = InTransaction_ ? + txn_->Put(handle, deletingKey, svalue) : + db_->Put(dbWriteOptions_, handle, deletingKey, svalue); + return ToStorageStatus(s); + +} + +Status RocksDBStorage::ClearDeleting(const std::string& name, + const std::string& key, + bool ordered) { + if (!inited_) { + return Status::DBClosed(); + } + + std::string ikey = ToInternalKey(name, key, ordered); + std::string deletingKey = "deleting_" + ikey; + VLOG(0) << "whs clear deleting key = " << deletingKey << ", ikey " << ikey << ", " << options_.dataDir; + auto handle = GetColumnFamilyHandle(ordered); + RocksDBPerfGuard guard(OP_DELETE); + ROCKSDB_NAMESPACE::Status s = InTransaction_ ? + txn_->Delete(handle, deletingKey) : + db_->Delete(dbWriteOptions_, handle, deletingKey); + return ToStorageStatus(s); + +} + Status RocksDBStorage::Del(const std::string& name, const std::string& key, bool ordered) { @@ -273,6 +334,14 @@ std::shared_ptr RocksDBStorage::GetAll(const std::string& name, this, std::move(ikey), 0, status, ordered); } +std::shared_ptr RocksDBStorage::GetPrefix(const std::string& prefix, + bool ordered) { + int status = inited_ ? 0 : -1; + return std::make_shared( + this, std::move(prefix), 0, status, ordered); +} + + size_t RocksDBStorage::Size(const std::string& name, bool ordered) { auto iterator = GetAll(name, ordered); if (iterator->Status() != 0) { @@ -505,6 +574,25 @@ bool RocksDBStorage::Recover(const std::string& dir) { return true; } +void RocksDBStorage::LoadDeletedInodes(std::list& item) { + LOG(INFO) << "LoadAll storage from: " << options_.dataDir; + std::string sprefix = "deleting_"; + rocksdb::Iterator* it1 = db_->NewIterator(rocksdb::ReadOptions()); + int counts = 0; + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key = it1->key().ToString(); + if (curve::common::StringStartWith(key, sprefix)) { + VLOG(0) << "whs recovery: " << key; + item.push_back(key); + } + LOG(ERROR) << "whs recovery: " << key; + counts++; + } + LOG(INFO) << "LoadAll storage from. size: " << item.size() + << ", " << counts << ", " << options_.dataDir; + // GetPrefix(prefix, false) +} + } // namespace storage } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.h b/curvefs/src/metaserver/storage/rocksdb_storage.h index e0023dd8e2..feec88ea29 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.h +++ b/curvefs/src/metaserver/storage/rocksdb_storage.h @@ -79,6 +79,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { bool Close() override; + void LoadDeletedInodes(std::list& item) override; + STORAGE_TYPE Type() override; StorageOptions GetStorageOptions() const override; @@ -92,6 +94,13 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const std::string& key, const ValueType& value) override; + Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) override; + + Status HClearDeleting(const std::string& name, + const std::string& key) override; + Status HDel(const std::string& name, const std::string& key) override; std::shared_ptr HGetAll(const std::string& name) override; @@ -100,6 +109,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { Status HClear(const std::string& name) override; + std::shared_ptr GetPrefix(const std::string& prefix, + bool ordered) override; // ordered Status SGet(const std::string& name, const std::string& key, @@ -156,6 +167,16 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const ValueType& value, bool ordered); + Status SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered); + + Status ClearDeleting(const std::string& name, + const std::string& key, + bool ordered); + + Status Del(const std::string& name, const std::string& key, bool ordered); @@ -219,6 +240,17 @@ inline Status RocksDBStorage::HSet(const std::string& name, return Set(name, key, value, false); } +inline Status RocksDBStorage::HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) { + return SetDeleting(name, key, value, false); +} + +inline Status RocksDBStorage::HClearDeleting(const std::string& name, + const std::string& key) { + return ClearDeleting(name, key, false); +} + inline Status RocksDBStorage::HDel(const std::string& name, const std::string& key) { return Del(name, key, false); diff --git a/curvefs/src/metaserver/storage/storage.h b/curvefs/src/metaserver/storage/storage.h index 97cef01fca..fa3891e84f 100644 --- a/curvefs/src/metaserver/storage/storage.h +++ b/curvefs/src/metaserver/storage/storage.h @@ -23,6 +23,7 @@ #ifndef CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ #define CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ +#include #include #include #include @@ -51,6 +52,22 @@ class BaseStorage { const std::string& key, const ValueType& value) = 0; + virtual Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) { + return Status::NotSupported(); + } + + virtual Status HClearDeleting(const std::string& name, + const std::string& key) { + return Status::NotSupported(); + } + + virtual std::shared_ptr GetPrefix( + const std::string& prefix, bool ordered) { + return nullptr; + } + virtual Status HDel(const std::string& name, const std::string& key) = 0; virtual std::shared_ptr HGetAll(const std::string& name) = 0; @@ -101,6 +118,8 @@ class KVStorage : public BaseStorage { virtual bool Close() = 0; + virtual void LoadDeletedInodes(std::list& item) {}; + virtual StorageOptions GetStorageOptions() const = 0; virtual std::shared_ptr BeginTransaction() = 0; diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index 1175376e72..5dcea30986 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -55,38 +55,55 @@ void TrashImpl::Init(const TrashOption &option) { isStop_ = false; } -void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) { +void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime, bool deleted) { TrashItem item; item.fsId = fsId; item.inodeId = inodeId; item.dtime = dtime; + item.deleted = deleted; LockGuard lg(itemsMutex_); if (isStop_) { return; } trashItems_.push_back(item); - VLOG(6) << "Add Trash Item success, item.fsId = " << item.fsId + VLOG(0) << "whs add trash item success, item.fsId = " << item.fsId << ", item.inodeId = " << item.inodeId - << ", item.dtime = " << item.dtime; + << ", item.dtime = " << item.dtime + << ", item.deleted = " << item.deleted; } void TrashImpl::ScanTrash() { + VLOG(0) << "whs ScanTrash start."; LockGuard lgScan(scanMutex_); std::list temp; { LockGuard lgItems(itemsMutex_); trashItems_.swap(temp); } + for (auto it = temp.begin(); it != temp.end(); it++) { + VLOG(0) << "11whs ScanTrash , " << "item.fsId = " << it->fsId + << ", item.inodeId = " << it->inodeId + << ", item.dtime = " << it->dtime << ", " << temp.size() << ", " << it->deleted; + } for (auto it = temp.begin(); it != temp.end();) { if (isStop_) { + VLOG(0) << "whs ScanTrash stop."; return; } + VLOG(0) << "whs ScanTrash , " << "item.fsId = " << it->fsId + << ", item.inodeId = " << it->inodeId + << ", item.dtime = " << it->dtime; if (NeedDelete(*it)) { MetaStatusCode ret = DeleteInodeAndData(*it); if (MetaStatusCode::NOT_FOUND == ret) { + VLOG(0) << "whs ScanTrash, inode not exist, fsId = " << it->fsId + << ", inodeId = " << it->inodeId; + ClearDeleted(*it); it = temp.erase(it); + VLOG(0) << "whs ScanTrash, inode not exist, fsId = " << it->fsId + << ", inodeId = " << it->inodeId; continue; } if (ret != MetaStatusCode::OK) { @@ -96,18 +113,45 @@ void TrashImpl::ScanTrash() { it++; continue; } - VLOG(6) << "Trash Delete Inode, fsId = " << it->fsId + VLOG(0) << "whs Trash Delete Inode, fsId = " << it->fsId << ", inodeId = " << it->inodeId; + ClearDeleted(*it); it = temp.erase(it); } else { + VLOG(0) << "whs ScanTrash, inode not expired, fsId = " << it->fsId + << ", inodeId = " << it->inodeId; it++; } } { + LockGuard lgItems(itemsMutex_); + + for (auto it = temp.begin(); it != temp.end(); it++) { + VLOG(0) << "00whs ScanTrash , " << "item.fsId = " << it->fsId + << ", item.inodeId = " << it->inodeId + << ", item.dtime = " << it->dtime; + } + + trashItems_.splice(trashItems_.end(), temp); } + VLOG(0) << "whs ScanTrash end."; + +} + +MetaStatusCode TrashImpl::ClearDeleted(const TrashItem &item) { + if (item.deleted) { + VLOG(0) << "ClearDeleted: " << item.fsId << ", " << item.inodeId + << ", " << item.dtime << ", " << item.deleted ; + inodeStorage_->ClearDelKey(Key4Inode(item.fsId, item.inodeId)); + } else { + + VLOG(0) << "ClearDeleted: " << item.fsId << ", " << item.inodeId + << ", " << item.dtime << ", " << item.deleted ; + } + return MetaStatusCode::OK; } void TrashImpl::StopScan() { @@ -176,7 +220,8 @@ uint64_t TrashImpl::GetFsRecycleTimeHour(uint32_t fsId) { return recycleTimeHour; } -MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) { +MetaStatusCode TrashImpl:: +DeleteInodeAndData(const TrashItem &item) { Inode inode; MetaStatusCode ret = inodeStorage_->Get(Key4Inode(item.fsId, item.inodeId), &inode); @@ -230,9 +275,11 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) { if (inode.s3chunkinfomap().empty()) { LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId << ", inodeId = " << item.inodeId; + // Todo(hzwuhongsong ：the s3chunkinfomap is empty too as empty file + // empty file will never be deleted return MetaStatusCode::NOT_FOUND; } - VLOG(9) << "DeleteInodeAndData, inode: " + VLOG(0) << "whs: DeleteInodeAndData, inode: " << inode.ShortDebugString(); int retVal = s3Adaptor_->Delete(inode); if (retVal != 0) { diff --git a/curvefs/src/metaserver/trash.h b/curvefs/src/metaserver/trash.h index de9410ddc3..88f08c9f61 100644 --- a/curvefs/src/metaserver/trash.h +++ b/curvefs/src/metaserver/trash.h @@ -51,10 +51,12 @@ struct TrashItem { uint32_t fsId; uint64_t inodeId; uint32_t dtime; + bool deleted; TrashItem() : fsId(0), inodeId(0), - dtime(0) {} + dtime(0), + deleted(false) {} }; struct TrashOption { @@ -78,7 +80,7 @@ class Trash { virtual void Init(const TrashOption &option) = 0; - virtual void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) = 0; + virtual void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime, bool deleted = false) = 0; virtual void ListItems(std::list *items) = 0; @@ -98,7 +100,7 @@ class TrashImpl : public Trash { void Init(const TrashOption &option) override; - void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) override; + void Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime, bool deleted) override; void ListItems(std::list *items) override; @@ -115,6 +117,8 @@ class TrashImpl : public Trash { uint64_t GetFsRecycleTimeHour(uint32_t fsId); + MetaStatusCode ClearDeleted(const TrashItem &item); + private: std::shared_ptr inodeStorage_; std::shared_ptr s3Adaptor_; diff --git a/curvefs/src/metaserver/trash_manager.cpp b/curvefs/src/metaserver/trash_manager.cpp index 7f6341db1c..cbb18fbe19 100644 --- a/curvefs/src/metaserver/trash_manager.cpp +++ b/curvefs/src/metaserver/trash_manager.cpp @@ -56,6 +56,7 @@ void TrashManager::ScanLoop() { } void TrashManager::ScanEveryTrash() { + VLOG(0) << "whs scan trash start."; std::map> temp; { curve::common::ReadLockGuard lg(rwLock_); diff --git a/curvefs/src/metaserver/trash_manager.h b/curvefs/src/metaserver/trash_manager.h index ca25c0145d..b35107ab6f 100644 --- a/curvefs/src/metaserver/trash_manager.h +++ b/curvefs/src/metaserver/trash_manager.h @@ -27,6 +27,8 @@ #include #include +#include "src/common/string_util.h" + #include "src/common/concurrent/concurrent.h" #include "curvefs/src/metaserver/trash.h" @@ -52,6 +54,11 @@ class TrashManager { << partitionId; } + std::shared_ptr GetTrash(uint32_t partitionId) { + curve::common::ReadLockGuard lg(rwLock_); + return trashs_.find(partitionId)->second; + } + void Remove(uint32_t partitionId); void Init(const TrashOption &options) { @@ -79,6 +86,7 @@ class TrashManager { InterruptibleSleeper sleeper_; std::map> trashs_; + curve::common::RWLock rwLock_; };