From e4441a720abb7a3a13e9bd1b6a2e381e6e1fe0b5 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Mon, 20 Nov 2023 14:28:53 +0800 Subject: [PATCH] curvefs/client: fix trash bugs --- .../src/metaserver/copyset/copyset_node.cpp | 2 +- curvefs/src/metaserver/inode_manager.cpp | 18 +++-- curvefs/src/metaserver/inode_storage.cpp | 38 ++++++++++ curvefs/src/metaserver/inode_storage.h | 9 +++ curvefs/src/metaserver/metastore.cpp | 29 +++++++- curvefs/src/metaserver/metastore.h | 2 + .../metaserver/storage/rocksdb_storage.cpp | 71 ++++++++++++++++++- .../src/metaserver/storage/rocksdb_storage.h | 17 +++++ curvefs/src/metaserver/storage/storage.h | 9 +++ curvefs/src/metaserver/trash.cpp | 5 +- curvefs/src/metaserver/trash_manager.h | 44 ++++++++++++ 11 files changed, 233 insertions(+), 11 deletions(-) diff --git a/curvefs/src/metaserver/copyset/copyset_node.cpp b/curvefs/src/metaserver/copyset/copyset_node.cpp index 449886204f..154f4969aa 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node.cpp @@ -147,7 +147,7 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) { LOG(ERROR) << "Failed to create meta store"; return false; } - + InitRaftNodeOptions(); butil::ip_t ip; diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index 91fb83a784..ec334bc509 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -312,7 +312,7 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId, MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, int64_t logIndex) { CHECK_APPLIED(); - VLOG(9) << "update inode, fsid: " << request.fsid() + VLOG(0) << "whs update inode, fsid: " << request.fsid() << ", inodeid: " << request.inodeid(); NameLockGuard lg(inodeLock_, GetInodeLockName(request.fsid(), request.inodeid())); @@ -388,11 +388,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, } } - if (s3NeedTrash) { - trash_->Add(old.fsid(), old.inodeid(), old.dtime()); - --(*type2InodeNum_)[old.type()]; - } - const S3ChunkInfoMap &map2add = request.s3chunkinfoadd(); const S3ChunkInfoList *list2add; VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size " @@ -443,7 +438,16 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, return MetaStatusCode::STORAGE_INTERNAL_ERROR; } } - VLOG(9) << "UpdateInode success, " << request.ShortDebugString(); + + if (s3NeedTrash) { +VLOG(0) << "whs add need trash, " << request.ShortDebugString(); + + inodeStorage_->UpdateDeletingKey(old, logIndex); + trash_->Add(old.fsid(), old.inodeid(), old.dtime()); + --(*type2InodeNum_)[old.type()]; + } + + VLOG(0) << "whs UpdateInode success, " << request.ShortDebugString(); return MetaStatusCode::OK; } diff --git a/curvefs/src/metaserver/inode_storage.cpp b/curvefs/src/metaserver/inode_storage.cpp index 8e5da8a913..ed76d6a0c8 100644 --- a/curvefs/src/metaserver/inode_storage.cpp +++ b/curvefs/src/metaserver/inode_storage.cpp @@ -185,6 +185,44 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) { return MetaStatusCode::STORAGE_INTERNAL_ERROR; } +MetaStatusCode InodeStorage::UpdateDeletingKey(const Inode& inode, int64_t logIndex) { + WriteLockGuard lg(rwLock_); + Key4Inode key(inode.fsid(), inode.inodeid()); + std::string skey = conv_.SerializeToString(key); + // std::string skeyDeleting = absl::StrCat(DELETING_PREFIX, skey); +VLOG(0) << "need UpdateDeletingKey, " << inode.inodeid(); + + const char* step = "Begin transaction"; + std::shared_ptr txn; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + auto rc = txn->HSetDeleting(table4Inode_, skey , inode); + // auto rc = txn->HSet(table4Inode_, skeyDeleting, inode); + step = "insert inode "; + if (rc.ok()) { + rc = SetAppliedIndex(txn.get(), logIndex); + step = "Insert applied index to transaction"; + } + if (rc.ok()) { + rc = DeleteInternal(txn.get(), key);; + step = "delete inode "; + } + if (rc.ok()) { +VLOG(0) << "whs UpdateDeletingKey ok, " << inode.inodeid(); + + return MetaStatusCode::OK; + } + LOG(ERROR) << step << "whs failed, status = " << rc.ToString(); + if (txn != nullptr && !txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << rc.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) { ReadLockGuard lg(rwLock_); std::string skey = conv_.SerializeToString(key); diff --git a/curvefs/src/metaserver/inode_storage.h b/curvefs/src/metaserver/inode_storage.h index 38ad3c5f56..267fba88c4 100644 --- a/curvefs/src/metaserver/inode_storage.h +++ b/curvefs/src/metaserver/inode_storage.h @@ -43,6 +43,7 @@ #include "curvefs/src/metaserver/storage/utils.h" #include "src/common/concurrent/rw_lock.h" +#define DELETING_PREFIX "deleting_" namespace curvefs { namespace metaserver { @@ -76,6 +77,14 @@ class InodeStorage { */ MetaStatusCode Insert(const Inode& inode, int64_t logIndex); + /** + * @brief update deleting inode key in storage + * @param[in] inode: the inode want to update + * @param[in] logIndex: the index of raft log + * @return + */ + MetaStatusCode UpdateDeletingKey(const Inode& inode, int64_t logIndex); + /** * @brief get inode from storage * @param[in] key: the key of inode want to get diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index 875f71cffc..4cd7bd1fb3 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -60,6 +60,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage; using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList; using ::curvefs::metaserver::storage::MemoryStorage; +using ::curvefs::metaserver::storage::NameGenerator; using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::StorageOptions; @@ -87,6 +88,8 @@ MetaStoreImpl::MetaStoreImpl(copyset::CopysetNode *node, storageOptions_(storageOptions) {} bool MetaStoreImpl::Load(const std::string &pathname) { +LOG(ERROR) << "whs load start"; + // Load from raft snap file to memory WriteLockGuard writeLockGuard(rwLock_); MetaStoreFStream fstream(&partitionMap_, kvStorage_, @@ -147,6 +150,13 @@ bool MetaStoreImpl::Load(const std::string &pathname) { } startCompacts(); + +LOG(ERROR) << "InitStorage start"; + + BuildTrashList(); +LOG(ERROR) << "InitStorage start02"; + +LOG(ERROR) << "whs load end"; return true; } @@ -859,7 +869,24 @@ bool MetaStoreImpl::InitStorage() { return false; } - return kvStorage_->Open(); + + if (!kvStorage_->Open()) { + return false; + } + + return true; +} + +void MetaStoreImpl::BuildTrashList() { + + std::shared_ptr nameGen = std::make_shared(0); + + std::shared_ptr inodeStorage = + std::make_shared(kvStorage_, nameGen, 1); + + auto trash = std::make_shared(inodeStorage); + + TrashManager::GetInstance().BuildAbortTrash(kvStorage_, trash); } } // namespace metaserver diff --git a/curvefs/src/metaserver/metastore.h b/curvefs/src/metaserver/metastore.h index a13c0a4980..d544f8c560 100644 --- a/curvefs/src/metaserver/metastore.h +++ b/curvefs/src/metaserver/metastore.h @@ -351,6 +351,8 @@ class MetaStoreImpl : public MetaStore { // REQUIRES: rwLock_ is held with write permission bool ClearInternal(); + void BuildTrashList(); + private: RWLock rwLock_; // protect partitionMap_ std::shared_ptr kvStorage_; diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.cpp b/curvefs/src/metaserver/storage/rocksdb_storage.cpp index 5875ba6817..50e1f69c66 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.cpp +++ b/curvefs/src/metaserver/storage/rocksdb_storage.cpp @@ -187,7 +187,7 @@ std::string RocksDBStorage::ToInternalKey(const std::string& name, std::ostringstream oss; oss << iname << kDelimiter_ << key; std::string ikey = oss.str(); - VLOG(9) << "ikey = " << ikey << " (ordered = " << ordered + VLOG(0) << "whs ikey = " << ikey << " (ordered = " << ordered << ", name = " << name << ", key = " << key << ")" << ", size = " << ikey.size(); return ikey; @@ -241,6 +241,32 @@ Status RocksDBStorage::Set(const std::string& name, return ToStorageStatus(s); } +Status RocksDBStorage::SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered) { + std::string svalue; + if (!inited_) { + return Status::DBClosed(); + } else if (!value.SerializeToString(&svalue)) { + return Status::SerializedFailed(); + } + + auto handle = GetColumnFamilyHandle(ordered); + + + std::string ikey = ToInternalKey(name, key, ordered); + std::string deletingKey = "deleting_" + ikey; +VLOG(0) << "whs set deleting key = " << deletingKey << ", ikey " << ikey; + RocksDBPerfGuard guard(OP_PUT); + ROCKSDB_NAMESPACE::Status s = InTransaction_ ? + txn_->Put(handle, deletingKey, svalue) : + db_->Put(dbWriteOptions_, handle, deletingKey, svalue); + return ToStorageStatus(s); + +} + + Status RocksDBStorage::Del(const std::string& name, const std::string& key, bool ordered) { @@ -273,6 +299,14 @@ std::shared_ptr RocksDBStorage::GetAll(const std::string& name, this, std::move(ikey), 0, status, ordered); } +std::shared_ptr RocksDBStorage::GetPrefix(const std::string& prefix, + bool ordered) { + int status = inited_ ? 0 : -1; + return std::make_shared( + this, std::move(prefix), 0, status, ordered); +} + + size_t RocksDBStorage::Size(const std::string& name, bool ordered) { auto iterator = GetAll(name, ordered); if (iterator->Status() != 0) { @@ -501,6 +535,41 @@ bool RocksDBStorage::Recover(const std::string& dir) { return false; } + + +// 创建迭代器 + rocksdb::Iterator* it1 = db_->NewIterator(rocksdb::ReadOptions()); + // 遍历键 + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key = it1->key().ToString(); + LOG(ERROR) << "whs recovery: " << key; + } + + if (!it1->status().ok()) { + LOG(ERROR) << "whs recovery: "; + + } + // 释放迭代器和数据库资源 + delete it1; + +/* +rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions()); +// 设置前缀 +std::string prefix = "deleting_"; +it->Seek(prefix); +// 遍历获取以指定前缀开头的键值对 +while (it->Valid() && it->key().starts_with(prefix)) { + std::string key = it->key().ToString(); + std::string value = it->value().ToString(); + LOG(ERROR) << "whs recovery: " << key; + + it->Next(); +} +delete it; + +*/ + + LOG(INFO) << "Recovered rocksdb from `" << dir << "`"; return true; } diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.h b/curvefs/src/metaserver/storage/rocksdb_storage.h index e0023dd8e2..b35738342b 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.h +++ b/curvefs/src/metaserver/storage/rocksdb_storage.h @@ -92,6 +92,10 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const std::string& key, const ValueType& value) override; + Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) override; + Status HDel(const std::string& name, const std::string& key) override; std::shared_ptr HGetAll(const std::string& name) override; @@ -100,6 +104,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { Status HClear(const std::string& name) override; + std::shared_ptr GetPrefix(const std::string& prefix, + bool ordered) override; // ordered Status SGet(const std::string& name, const std::string& key, @@ -156,6 +162,11 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const ValueType& value, bool ordered); + Status SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered); + Status Del(const std::string& name, const std::string& key, bool ordered); @@ -219,6 +230,12 @@ inline Status RocksDBStorage::HSet(const std::string& name, return Set(name, key, value, false); } +inline Status RocksDBStorage::HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) { + return SetDeleting(name, key, value, false); +} + inline Status RocksDBStorage::HDel(const std::string& name, const std::string& key) { return Del(name, key, false); diff --git a/curvefs/src/metaserver/storage/storage.h b/curvefs/src/metaserver/storage/storage.h index 97cef01fca..02dc8db9c2 100644 --- a/curvefs/src/metaserver/storage/storage.h +++ b/curvefs/src/metaserver/storage/storage.h @@ -51,6 +51,15 @@ class BaseStorage { const std::string& key, const ValueType& value) = 0; + virtual Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value){return Status::NotSupported();}; + + virtual std::shared_ptr GetPrefix( + const std::string& prefix, bool ordered) { + return nullptr; + } + virtual Status HDel(const std::string& name, const std::string& key) = 0; virtual std::shared_ptr HGetAll(const std::string& name) = 0; diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index 1175376e72..05341a5e07 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -49,6 +49,8 @@ void TrashOption::InitTrashOptionFromConf(std::shared_ptr conf) { } void TrashImpl::Init(const TrashOption &option) { +VLOG(0) << "whs init trash, "; + options_ = option; s3Adaptor_ = option.s3Adaptor; mdsClient_ = option.mdsClient; @@ -66,7 +68,7 @@ void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) { return; } trashItems_.push_back(item); - VLOG(6) << "Add Trash Item success, item.fsId = " << item.fsId + VLOG(0) << "whs Add Trash Item success, item.fsId = " << item.fsId << ", item.inodeId = " << item.inodeId << ", item.dtime = " << item.dtime; } @@ -78,6 +80,7 @@ void TrashImpl::ScanTrash() { LockGuard lgItems(itemsMutex_); trashItems_.swap(temp); } +VLOG(0) << "whs scan trash, "; for (auto it = temp.begin(); it != temp.end();) { if (isStop_) { diff --git a/curvefs/src/metaserver/trash_manager.h b/curvefs/src/metaserver/trash_manager.h index ca25c0145d..82c5fb1090 100644 --- a/curvefs/src/metaserver/trash_manager.h +++ b/curvefs/src/metaserver/trash_manager.h @@ -52,6 +52,15 @@ class TrashManager { << partitionId; } + void BuildAbortTrash(const std::shared_ptr &kvStorage, const std::shared_ptr &trash) { + curve::common::WriteLockGuard lg(rwLock_); + trash->Init(options_); + abortTrash_ = trash; + kvStorage_ = kvStorage; + BuildTrashItems(); + LOG(INFO) << "build abort trash"; + } + void Remove(uint32_t partitionId); void Init(const TrashOption &options) { @@ -66,6 +75,32 @@ class TrashManager { void ListItems(std::list *items); + void BuildTrashItems() { + std::string prefix = "deleting_"; + LOG(ERROR) << "- whs BuildTrashItems start: "; + + // need to edit + auto iter = kvStorage_->GetPrefix(prefix, false); + if (iter->Status() != 0) { + LOG(ERROR) << "BuildTrashItems failed"; + return; + } + LOG(ERROR) << "- whs BuildTrashItems start 02 "; + + size_t size = 0; + + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + size++; + LOG(ERROR) << "- whs BuildTrashItems: " << size; + + LOG(ERROR) << "- whs BuildTrashItems: " << iter->Key(); + + } + + LOG(ERROR) << "- whs BuildTrashItems over "; + + } + private: void ScanLoop(); @@ -79,6 +114,15 @@ class TrashManager { InterruptibleSleeper sleeper_; std::map> trashs_; +/** + * 在metastorage的Load函数中新建trash，并进行初始化 + * + *在ScanEveryTrash函数中，获取abortTrash_中的所有item，然后调用deleteinodeanddata* +*/ +// 新建一个Trash，处理因重启导致的需要重新trash的问题 +// 注意除了新建，还需要init + std::shared_ptr abortTrash_; + std::shared_ptr kvStorage_; curve::common::RWLock rwLock_; };