diff --git a/curvefs/src/metaserver/copyset/copyset_node.cpp b/curvefs/src/metaserver/copyset/copyset_node.cpp index 449886204f..bb755fcc9c 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node.cpp @@ -161,6 +161,7 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) { } bool CopysetNode::Start() { + VLOG(3) << "copyset is starting, copyset: " << name_; if (!raftNode_) { LOG(ERROR) << "RaftNode didn't created, copyset: " << name_; return false; @@ -170,8 +171,8 @@ bool CopysetNode::Start() { LOG(ERROR) << "Fail to init raft node, copyset: " << name_; return false; } - - LOG(INFO) << "Run copyset success, copyset: " << name_; + metaStore_->LoadAll(); + VLOG(3) << "copyset start success, copyset: " << name_; return true; } diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index 9fc94643a4..5318064b83 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -312,7 +312,7 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId, MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, int64_t logIndex) { CHECK_APPLIED(); - VLOG(9) << "update inode, fsid: " << request.fsid() + VLOG(0) << "whs update inode, fsid: " << request.fsid() << ", inodeid: " << request.inodeid(); NameLockGuard lg(inodeLock_, GetInodeLockName(request.fsid(), request.inodeid())); @@ -388,11 +388,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, } } - if (s3NeedTrash) { - trash_->Add(old.fsid(), old.inodeid(), old.dtime()); - --(*type2InodeNum_)[old.type()]; - } - const S3ChunkInfoMap &map2add = request.s3chunkinfoadd(); const S3ChunkInfoList *list2add; VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size " @@ -443,7 +438,16 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, return MetaStatusCode::STORAGE_INTERNAL_ERROR; } } - VLOG(9) << "UpdateInode success, " << request.ShortDebugString(); + + if (s3NeedTrash) { +VLOG(0) << "whs add need trash, " << request.ShortDebugString(); + + inodeStorage_->UpdateDeletingKey(old, logIndex); + trash_->Add(old.fsid(), old.inodeid(), old.dtime()); + --(*type2InodeNum_)[old.type()]; + } + + VLOG(0) << "whs UpdateInode success, " << request.ShortDebugString(); return MetaStatusCode::OK; } diff --git a/curvefs/src/metaserver/inode_storage.cpp b/curvefs/src/metaserver/inode_storage.cpp index 8e5da8a913..2e30a23c15 100644 --- a/curvefs/src/metaserver/inode_storage.cpp +++ b/curvefs/src/metaserver/inode_storage.cpp @@ -185,6 +185,46 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) { return MetaStatusCode::STORAGE_INTERNAL_ERROR; } +MetaStatusCode InodeStorage::UpdateDeletingKey(const Inode& inode, int64_t logIndex) { + WriteLockGuard lg(rwLock_); + Key4Inode key(inode.fsid(), inode.inodeid()); + std::string skey = conv_.SerializeToString(key); + VLOG(9) << "update deleting key, " << inode.inodeid(); + const char* step = "Begin transaction"; + std::shared_ptr txn; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + auto rc = txn->HSetDeleting(table4Inode_, skey , inode); + step = "insert inode "; + if (rc.ok()) { + // delete key + // rc = DeleteInternal(txn.get(), key); + rc = txn->HDel(table4Inode_, skey); + step = "delete inode "; + } + if (rc.ok()) { + rc = SetAppliedIndex(txn.get(), logIndex); + step = "Insert applied index to transaction"; + } + if (rc.ok()) { + rc = txn->Commit(); + step = "commit"; + } + if (rc.ok()) { + VLOG(0) << "update deleting key ok, " << inode.inodeid(); + return MetaStatusCode::OK; + } + LOG(ERROR) << step << "failed, status = " << rc.ToString(); + if (txn != nullptr && !txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << rc.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) { ReadLockGuard lg(rwLock_); std::string skey = conv_.SerializeToString(key); @@ -471,7 +511,6 @@ MetaStatusCode InodeStorage::Clear() { // because if we fail stop, we will replay // raft logs and clear it again WriteLockGuard lg(rwLock_); - Status s = kvStorage_->HClear(table4Inode_); if (!s.ok()) { LOG(ERROR) << "InodeStorage clear inode table failed, status = " @@ -492,7 +531,6 @@ MetaStatusCode InodeStorage::Clear() { << s.ToString(); return MetaStatusCode::STORAGE_INTERNAL_ERROR; } - s = kvStorage_->HClear(table4InodeAuxInfo_); if (!s.ok()) { LOG(ERROR) diff --git a/curvefs/src/metaserver/inode_storage.h b/curvefs/src/metaserver/inode_storage.h index 38ad3c5f56..267fba88c4 100644 --- a/curvefs/src/metaserver/inode_storage.h +++ b/curvefs/src/metaserver/inode_storage.h @@ -43,6 +43,7 @@ #include "curvefs/src/metaserver/storage/utils.h" #include "src/common/concurrent/rw_lock.h" +#define DELETING_PREFIX "deleting_" namespace curvefs { namespace metaserver { @@ -76,6 +77,14 @@ class InodeStorage { */ MetaStatusCode Insert(const Inode& inode, int64_t logIndex); + /** + * @brief update deleting inode key in storage + * @param[in] inode: the inode want to update + * @param[in] logIndex: the index of raft log + * @return + */ + MetaStatusCode UpdateDeletingKey(const Inode& inode, int64_t logIndex); + /** * @brief get inode from storage * @param[in] key: the key of inode want to get diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index a2fd20323c..ddf1df8bdf 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -60,6 +60,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage; using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList; using ::curvefs::metaserver::storage::MemoryStorage; +using ::curvefs::metaserver::storage::NameGenerator; using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::StorageOptions; @@ -87,6 +88,8 @@ MetaStoreImpl::MetaStoreImpl(copyset::CopysetNode *node, storageOptions_(storageOptions) {} bool MetaStoreImpl::Load(const std::string &pathname) { +LOG(ERROR) << "whs load start"; + // Load from raft snap file to memory WriteLockGuard writeLockGuard(rwLock_); MetaStoreFStream fstream(&partitionMap_, kvStorage_, @@ -147,6 +150,9 @@ bool MetaStoreImpl::Load(const std::string &pathname) { } startCompacts(); + + +LOG(ERROR) << "whs load end"; return true; } @@ -863,7 +869,33 @@ bool MetaStoreImpl::InitStorage() { return false; } - return kvStorage_->Open(); + + if (!kvStorage_->Open()) { + return false; + } + + return true; +} + +void MetaStoreImpl::BuildTrashList() { + + std::shared_ptr nameGen = std::make_shared(0); + + std::shared_ptr inodeStorage = + std::make_shared(kvStorage_, nameGen, 1); + + auto trash = std::make_shared(inodeStorage); + + TrashManager::GetInstance().BuildAbortTrash(kvStorage_, trash); +} + +void MetaStoreImpl::LoadAll() { + LOG(ERROR) << "InitStorage start"; + // kvStorage_->LoadAll(); + LOG(ERROR) << "InitStorage start01"; + + BuildTrashList(); + LOG(ERROR) << "InitStorage start02"; } } // namespace metaserver diff --git a/curvefs/src/metaserver/metastore.h b/curvefs/src/metaserver/metastore.h index a13c0a4980..13ac6e111d 100644 --- a/curvefs/src/metaserver/metastore.h +++ b/curvefs/src/metaserver/metastore.h @@ -117,6 +117,7 @@ class MetaStore { virtual bool SaveData(const std::string& dir, std::vector* files) = 0; virtual bool Clear() = 0; + virtual void LoadAll() {}; virtual bool Destroy() = 0; virtual MetaStatusCode CreatePartition( const CreatePartitionRequest* request, @@ -223,6 +224,7 @@ class MetaStoreImpl : public MetaStore { std::vector* files) override; bool Clear() override; bool Destroy() override; + void LoadAll() override; MetaStatusCode CreatePartition(const CreatePartitionRequest* request, CreatePartitionResponse* response, @@ -351,6 +353,8 @@ class MetaStoreImpl : public MetaStore { // REQUIRES: rwLock_ is held with write permission bool ClearInternal(); + void BuildTrashList(); + private: RWLock rwLock_; // protect partitionMap_ std::shared_ptr kvStorage_; diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.cpp b/curvefs/src/metaserver/storage/rocksdb_storage.cpp index 5875ba6817..5906980121 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.cpp +++ b/curvefs/src/metaserver/storage/rocksdb_storage.cpp @@ -26,6 +26,7 @@ #include #include +#include "src/common/string_util.h" #include "src/common/timeutility.h" #include "curvefs/src/metaserver/storage/utils.h" #include "curvefs/src/metaserver/storage/storage.h" @@ -187,7 +188,7 @@ std::string RocksDBStorage::ToInternalKey(const std::string& name, std::ostringstream oss; oss << iname << kDelimiter_ << key; std::string ikey = oss.str(); - VLOG(9) << "ikey = " << ikey << " (ordered = " << ordered + VLOG(0) << "whs ikey = " << ikey << " (ordered = " << ordered << ", name = " << name << ", key = " << key << ")" << ", size = " << ikey.size(); return ikey; @@ -241,6 +242,32 @@ Status RocksDBStorage::Set(const std::string& name, return ToStorageStatus(s); } +Status RocksDBStorage::SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered) { + std::string svalue; + if (!inited_) { + return Status::DBClosed(); + } else if (!value.SerializeToString(&svalue)) { + return Status::SerializedFailed(); + } + + auto handle = GetColumnFamilyHandle(ordered); + + + std::string ikey = ToInternalKey(name, key, ordered); + std::string deletingKey = "deleting_" + ikey; +VLOG(0) << "whs set deleting key = " << deletingKey << ", ikey " << ikey; + RocksDBPerfGuard guard(OP_PUT); + ROCKSDB_NAMESPACE::Status s = InTransaction_ ? + txn_->Put(handle, deletingKey, svalue) : + db_->Put(dbWriteOptions_, handle, deletingKey, svalue); + return ToStorageStatus(s); + +} + + Status RocksDBStorage::Del(const std::string& name, const std::string& key, bool ordered) { @@ -273,6 +300,14 @@ std::shared_ptr RocksDBStorage::GetAll(const std::string& name, this, std::move(ikey), 0, status, ordered); } +std::shared_ptr RocksDBStorage::GetPrefix(const std::string& prefix, + bool ordered) { + int status = inited_ ? 0 : -1; + return std::make_shared( + this, std::move(prefix), 0, status, ordered); +} + + size_t RocksDBStorage::Size(const std::string& name, bool ordered) { auto iterator = GetAll(name, ordered); if (iterator->Status() != 0) { @@ -505,6 +540,21 @@ bool RocksDBStorage::Recover(const std::string& dir) { return true; } +void RocksDBStorage::LoadAll(std::list& item) { + LOG(INFO) << "LoadAll storage from"; + std::string sprefix = "deleting_"; + rocksdb::Iterator* it1 = db_->NewIterator(rocksdb::ReadOptions()); + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key = it1->key().ToString(); + if (curve::common::StringStartWith(key, sprefix)) { + VLOG(9) << "whs recovery: " << key; + item.push_back(key); + } + LOG(ERROR) << "whs recovery: " << key; + } + // GetPrefix(prefix, false) +} + } // namespace storage } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.h b/curvefs/src/metaserver/storage/rocksdb_storage.h index e0023dd8e2..7c2d7deb87 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.h +++ b/curvefs/src/metaserver/storage/rocksdb_storage.h @@ -79,6 +79,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { bool Close() override; + void LoadAll(std::list& item) override; + STORAGE_TYPE Type() override; StorageOptions GetStorageOptions() const override; @@ -92,6 +94,10 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const std::string& key, const ValueType& value) override; + Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) override; + Status HDel(const std::string& name, const std::string& key) override; std::shared_ptr HGetAll(const std::string& name) override; @@ -100,6 +106,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { Status HClear(const std::string& name) override; + std::shared_ptr GetPrefix(const std::string& prefix, + bool ordered) override; // ordered Status SGet(const std::string& name, const std::string& key, @@ -156,6 +164,11 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { const ValueType& value, bool ordered); + Status SetDeleting(const std::string& name, + const std::string& key, + const ValueType& value, + bool ordered); + Status Del(const std::string& name, const std::string& key, bool ordered); @@ -219,6 +232,12 @@ inline Status RocksDBStorage::HSet(const std::string& name, return Set(name, key, value, false); } +inline Status RocksDBStorage::HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value) { + return SetDeleting(name, key, value, false); +} + inline Status RocksDBStorage::HDel(const std::string& name, const std::string& key) { return Del(name, key, false); diff --git a/curvefs/src/metaserver/storage/storage.h b/curvefs/src/metaserver/storage/storage.h index 97cef01fca..ae9559f803 100644 --- a/curvefs/src/metaserver/storage/storage.h +++ b/curvefs/src/metaserver/storage/storage.h @@ -23,6 +23,7 @@ #ifndef CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ #define CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ +#include #include #include #include @@ -51,6 +52,15 @@ class BaseStorage { const std::string& key, const ValueType& value) = 0; + virtual Status HSetDeleting(const std::string& name, + const std::string& key, + const ValueType& value){return Status::NotSupported();}; + + virtual std::shared_ptr GetPrefix( + const std::string& prefix, bool ordered) { + return nullptr; + } + virtual Status HDel(const std::string& name, const std::string& key) = 0; virtual std::shared_ptr HGetAll(const std::string& name) = 0; @@ -101,6 +111,8 @@ class KVStorage : public BaseStorage { virtual bool Close() = 0; + virtual void LoadAll(std::list& item) {}; + virtual StorageOptions GetStorageOptions() const = 0; virtual std::shared_ptr BeginTransaction() = 0; diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index 1175376e72..6e22e828c4 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -66,7 +66,7 @@ void TrashImpl::Add(uint32_t fsId, uint64_t inodeId, uint32_t dtime) { return; } trashItems_.push_back(item); - VLOG(6) << "Add Trash Item success, item.fsId = " << item.fsId + VLOG(9) << "add trash item success, item.fsId = " << item.fsId << ", item.inodeId = " << item.inodeId << ", item.dtime = " << item.dtime; } @@ -78,11 +78,13 @@ void TrashImpl::ScanTrash() { LockGuard lgItems(itemsMutex_); trashItems_.swap(temp); } - for (auto it = temp.begin(); it != temp.end();) { if (isStop_) { return; } + VLOG(9) << "ScanTrash , " << "item.fsId = " << it->fsId + << ", item.inodeId = " << it->inodeId + << ", item.dtime = " << it->dtime; if (NeedDelete(*it)) { MetaStatusCode ret = DeleteInodeAndData(*it); if (MetaStatusCode::NOT_FOUND == ret) { diff --git a/curvefs/src/metaserver/trash_manager.cpp b/curvefs/src/metaserver/trash_manager.cpp index 7f6341db1c..b591cf92c9 100644 --- a/curvefs/src/metaserver/trash_manager.cpp +++ b/curvefs/src/metaserver/trash_manager.cpp @@ -66,6 +66,7 @@ void TrashManager::ScanEveryTrash() { pair.second->ScanTrash(); } } + abortTrash_->ScanTrash(); } void TrashManager::Remove(uint32_t partitionId) { @@ -96,5 +97,21 @@ void TrashManager::ListItems(std::list *items) { } } +void TrashManager::BuildTrashItems() { + VLOG(3) << "build trash items start."; + std::list items; + kvStorage_->LoadAll(items); + std::vector names; + int size = 0; + for (auto iter: items) { + curve::common::SplitString(iter , ":", &names); + VLOG(3) << "build trash items: " << iter << ", size: " << ++size + << ", " << names[names.size() - 1 ] << ", " << names[names.size() - 2 ] ; + abortTrash_->Add(std::stoull(names[names.size() - 2 ]), + std::stoull(names[names.size() - 1 ]), 0); + } + VLOG(3) << "build trash items over."; +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/trash_manager.h b/curvefs/src/metaserver/trash_manager.h index ca25c0145d..5527c0eaf5 100644 --- a/curvefs/src/metaserver/trash_manager.h +++ b/curvefs/src/metaserver/trash_manager.h @@ -27,6 +27,8 @@ #include #include +#include "src/common/string_util.h" + #include "src/common/concurrent/concurrent.h" #include "curvefs/src/metaserver/trash.h" @@ -52,6 +54,15 @@ class TrashManager { << partitionId; } + void BuildAbortTrash(const std::shared_ptr &kvStorage, const std::shared_ptr &trash) { + curve::common::WriteLockGuard lg(rwLock_); + trash->Init(options_); + abortTrash_ = trash; + kvStorage_ = kvStorage; + BuildTrashItems(); + LOG(INFO) << "build abort trash"; + } + void Remove(uint32_t partitionId); void Init(const TrashOption &options) { @@ -66,6 +77,8 @@ class TrashManager { void ListItems(std::list *items); + void BuildTrashItems(); + private: void ScanLoop(); @@ -79,6 +92,15 @@ class TrashManager { InterruptibleSleeper sleeper_; std::map> trashs_; +/** + * 在metastorage的Load函数中新建trash,并进行初始化 + * + *在ScanEveryTrash函数中,获取abortTrash_中的所有item,然后调用deleteinodeanddata* +*/ +// 新建一个Trash,处理因重启导致的需要重新trash的问题 +// 注意除了新建,还需要init + std::shared_ptr abortTrash_; + std::shared_ptr kvStorage_; curve::common::RWLock rwLock_; };