curvefs/metaserver: fix trash bugs

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -161,6 +161,7 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) {
 }
 
 bool CopysetNode::Start() {
+    VLOG(3) << "copyset is starting, copyset: " << name_;
     if (!raftNode_) {
         LOG(ERROR) << "RaftNode didn't created, copyset: " << name_;
         return false;
@@ -170,8 +171,8 @@ bool CopysetNode::Start() {
         LOG(ERROR) << "Fail to init raft node, copyset: " << name_;
         return false;
     }
-
-    LOG(INFO) << "Run copyset success, copyset: " << name_;
+    metaStore_->LoadDeletedInodes();
+    VLOG(3) << "copyset start success, copyset: " << name_;
     return true;
 }
 

curvefs/src/metaserver/inode_manager.cpp

@@ -312,7 +312,7 @@ MetaStatusCode InodeManager::DeleteInode(uint32_t fsId, uint64_t inodeId,
 MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
                                          int64_t logIndex) {
     CHECK_APPLIED();
-    VLOG(9) << "update inode, fsid: " << request.fsid()
+    VLOG(0) << "update inode, fsid: " << request.fsid()
             << ", inodeid: " << request.inodeid();
     NameLockGuard lg(inodeLock_,
                      GetInodeLockName(request.fsid(), request.inodeid()));
@@ -388,11 +388,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
         }
     }
 
-    if (s3NeedTrash) {
-        trash_->Add(old.fsid(), old.inodeid(), old.dtime());
-        --(*type2InodeNum_)[old.type()];
-    }
-
     const S3ChunkInfoMap &map2add = request.s3chunkinfoadd();
     const S3ChunkInfoList *list2add;
     VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size "
@@ -443,7 +438,15 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
             return MetaStatusCode::STORAGE_INTERNAL_ERROR;
         }
     }
-    VLOG(9) << "UpdateInode success, " << request.ShortDebugString();
+
+    if (s3NeedTrash) {
+        inodeStorage_->UpdateDeletingKey(old, logIndex);
+        // wuhongsong：需要打开
+        // trash_->Add(old.fsid(), old.inodeid(), old.dtime());
+        --(*type2InodeNum_)[old.type()];
+    }
+
+    VLOG(0) << "UpdateInode success, " << request.ShortDebugString();
     return MetaStatusCode::OK;
 }
 

curvefs/src/metaserver/inode_storage.cpp

@@ -185,6 +185,77 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) {
     return MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
+MetaStatusCode InodeStorage::UpdateDeletingKey(const Inode& inode, int64_t logIndex) {
+    WriteLockGuard lg(rwLock_);
+    Key4Inode key(inode.fsid(), inode.inodeid());
+    std::string skey = conv_.SerializeToString(key);
+    VLOG(0) << "whs update deleting key, " << inode.inodeid();
+    const char* step = "Begin transaction";
+    std::shared_ptr<storage::StorageTransaction> txn;
+    txn = kvStorage_->BeginTransaction();
+    if (txn == nullptr) {
+        LOG(ERROR) << "Begin transaction failed";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+    auto rc = txn->HSetDeleting(table4Inode_, skey , inode);
+    step = "insert inode ";
+    if (rc.ok()) {
+        // delete key
+       // rc = DeleteInternal(txn.get(), key);
+       // 这里的删除只能最后删除，不然在DeleteInodeAndData中有问题
+       //  rc = txn->HDel(table4Inode_, skey);
+        step = "delete inode ";
+    }
+     if (rc.ok()) {
+        rc = SetAppliedIndex(txn.get(), logIndex);
+        step = "Insert applied index to transaction";
+    }
+    if (rc.ok()) {
+        rc = txn->Commit();
+        step = "commit";
+    }
+    if (rc.ok()) {
+        VLOG(0) << "set deleting key ok";
+        return MetaStatusCode::OK;
+    }
+    LOG(ERROR) << step << "failed, status = " << rc.ToString();
+    if (txn != nullptr && !txn->Rollback().ok()) {
+        LOG(ERROR) << "Rollback delete inode transaction failed, status = "
+                   << rc.ToString();
+    }
+    return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+}
+
+MetaStatusCode InodeStorage::ClearDelKey(const Key4Inode& key) {
+    WriteLockGuard lg(rwLock_);
+    std::string skey = conv_.SerializeToString(key);
+    VLOG(0) << "clear deleting key start, " << skey;
+    std::shared_ptr<storage::StorageTransaction> txn = nullptr;
+    const char* step = "Begin transaction";
+    txn = kvStorage_->BeginTransaction();
+    if (txn == nullptr) {
+        LOG(ERROR) << "Begin transaction failed";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+    step = "Delete inode from transaction";
+    auto s = txn->HClearDeleting(table4Inode_, skey);
+    if (s.ok()) {
+        step = "Delete inode";
+        s = txn->Commit();
+    }
+    if (s.ok()) {
+        VLOG(0) << "clear deleting key ok";
+        return MetaStatusCode::OK;
+    }
+    LOG(ERROR) << step << " failed, status = " << s.ToString();
+    if (txn != nullptr && !txn->Rollback().ok()) {
+        LOG(ERROR) << "Rollback delete inode transaction failed, status = "
+                   << s.ToString();
+    }
+    return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+
+}
+
 MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) {
     ReadLockGuard lg(rwLock_);
     std::string skey = conv_.SerializeToString(key);
@@ -471,7 +542,6 @@ MetaStatusCode InodeStorage::Clear() {
     // because if we fail stop, we will replay
     // raft logs and clear it again
     WriteLockGuard lg(rwLock_);
-
     Status s = kvStorage_->HClear(table4Inode_);
     if (!s.ok()) {
         LOG(ERROR) << "InodeStorage clear inode table failed, status = "
@@ -492,7 +562,6 @@ MetaStatusCode InodeStorage::Clear() {
             << s.ToString();
         return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
-
     s = kvStorage_->HClear(table4InodeAuxInfo_);
     if (!s.ok()) {
         LOG(ERROR)

curvefs/src/metaserver/inode_storage.h

@@ -43,6 +43,7 @@
 #include "curvefs/src/metaserver/storage/utils.h"
 #include "src/common/concurrent/rw_lock.h"
 
+#define DELETING_PREFIX "deleting_"
 namespace curvefs {
 namespace metaserver {
 
@@ -76,6 +77,15 @@ class InodeStorage {
      */
     MetaStatusCode Insert(const Inode& inode, int64_t logIndex);
 
+    /**
+     * @brief update deleting inode key in storage
+     * @param[in] inode: the inode want to update
+     * @param[in] logIndex: the index of raft log
+     * @return
+     */
+    MetaStatusCode UpdateDeletingKey(const Inode& inode, int64_t logIndex);
+
+    MetaStatusCode ClearDelKey(const Key4Inode& key);
     /**
      * @brief get inode from storage
      * @param[in] key: the key of inode want to get
@@ -187,6 +197,10 @@ class InodeStorage {
     MetaStatusCode GetAllBlockGroup(
         std::vector<DeallocatableBlockGroup>* deallocatableBlockGroupVec);
 
+    std::shared_ptr<KVStorage> GetKVStorage() {
+        return kvStorage_;
+    }
+
  private:
     MetaStatusCode UpdateInodeS3MetaSize(Transaction txn, uint32_t fsId,
                                          uint64_t inodeId, uint64_t size4add,

curvefs/src/metaserver/metastore.cpp

@@ -25,6 +25,7 @@
 #include <glog/logging.h>
 #include <sys/types.h>
 
+#include <bitset>
 #include <memory>
 #include <thread>  // NOLINT
 #include <unordered_map>
@@ -60,6 +61,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage;
 using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList;
 
 using ::curvefs::metaserver::storage::MemoryStorage;
+using ::curvefs::metaserver::storage::NameGenerator;
 using ::curvefs::metaserver::storage::RocksDBStorage;
 using ::curvefs::metaserver::storage::StorageOptions;
 
@@ -87,6 +89,8 @@ MetaStoreImpl::MetaStoreImpl(copyset::CopysetNode *node,
       storageOptions_(storageOptions) {}
 
 bool MetaStoreImpl::Load(const std::string &pathname) {
+LOG(ERROR) << "whs load start";
+
     // Load from raft snap file to memory
     WriteLockGuard writeLockGuard(rwLock_);
     MetaStoreFStream fstream(&partitionMap_, kvStorage_,
@@ -147,6 +151,9 @@ bool MetaStoreImpl::Load(const std::string &pathname) {
     }
 
     startCompacts();
+
+
+LOG(ERROR) << "whs load end";
     return true;
 }
 
@@ -863,7 +870,37 @@ bool MetaStoreImpl::InitStorage() {
         return false;
     }
 
-    return kvStorage_->Open();
+
+    if (!kvStorage_->Open()) {
+        return false;
+    }
+
+    return true;
+}
+
+void MetaStoreImpl::LoadDeletedInodes() {
+    LOG(ERROR) << "InitStorage start";
+    std::list<std::string> items;
+    // whs: 好像很慢很慢, 11条目录都加载了好久
+    kvStorage_->LoadDeletedInodes(items);
+    VLOG(3) << "build trash items size: " << items.size();
+    std::vector<std::string> names;
+    for (auto iter : items) {
+        curve::common::SplitString(iter , ":", &names);
+        char* tmp = const_cast<char*>(names[2].c_str());
+        uint32_t partitionId = *reinterpret_cast<uint32_t*>(tmp);
+        VLOG(0) << "whs partitionId: " << partitionId;
+        std::shared_ptr<Trash> trash =
+          TrashManager::GetInstance().GetTrash(partitionId);
+        if (nullptr == trash) {
+            VLOG(0) << "whs trash is null: " << partitionId;
+            continue;
+        }
+        trash->Add(std::stoul(names[names.size() - 2 ]),
+          std::stoull(names[names.size() - 1 ]), 0, true);
+    }
+
+    VLOG(3) << "build trash items over.";
 }
 
 }  // namespace metaserver

curvefs/src/metaserver/metastore.h

@@ -117,6 +117,7 @@ class MetaStore {
     virtual bool SaveData(const std::string& dir,
                           std::vector<std::string>* files) = 0;
     virtual bool Clear() = 0;
+    virtual void LoadDeletedInodes() {};
     virtual bool Destroy() = 0;
     virtual MetaStatusCode CreatePartition(
         const CreatePartitionRequest* request,
@@ -223,6 +224,7 @@ class MetaStoreImpl : public MetaStore {
                   std::vector<std::string>* files) override;
     bool Clear() override;
     bool Destroy() override;
+    void LoadDeletedInodes() override;
 
     MetaStatusCode CreatePartition(const CreatePartitionRequest* request,
                                    CreatePartitionResponse* response,

curvefs/src/metaserver/storage/converter.cpp

@@ -23,6 +23,7 @@
 #include <inttypes.h>
 #include <glog/logging.h>
 
+#include <bitset>
 #include <cstring>
 #include <string>
 #include <vector>
@@ -118,9 +119,37 @@ size_t NameGenerator::GetFixedLength() {
 }
 
 std::string NameGenerator::Format(KEY_TYPE type, uint32_t partitionId) {
+    VLOG(0) << "whs NameGenerator::Format, " << partitionId;
     char buf[sizeof(partitionId)];
     std::memcpy(buf, reinterpret_cast<char*>(&partitionId), sizeof(buf));
+    VLOG(0) << "whs NameGenerator::Format1, " << buf;
+
+   uint32_t num = *reinterpret_cast<uint32_t*>(buf);
+
+    VLOG(0) << "whs NameGenerator::Format1.1, " << num << ", " << sizeof(buf) << ", " << sizeof(partitionId) << sizeof(int);
+    std::string partitionIdStr = std::to_string(partitionId);
+    VLOG(0) << "whs NameGenerator::Format2, " << partitionIdStr;
+
+    absl::string_view tmp = absl::string_view(buf, sizeof(buf));
+
+    std::string tmp2(tmp);
+        VLOG(0) << "whs NameGenerator::Format4, " << tmp2;
+
+    std::string binaryValue = std::bitset<sizeof(int) * 8>(partitionId).to_string();
+        VLOG(0) << "whs NameGenerator::Format5, " << binaryValue;
+
+    // std::bitset<sizeof(int) * 8> bitsetValue(tmp2);
+    // uint32_t p = bitsetValue.to_ulong();
+
+
+
+   // int p2 = bitsetValue.to_ulong();
+
+
+
     return absl::StrCat(type, kDelimiter, absl::string_view(buf, sizeof(buf)));
+    // return absl::StrCat(type, kDelimiter, partitionIdStr);
+    // return absl::StrCat(type, kDelimiter, binaryValue);
 }
 
 Key4Inode::Key4Inode() : fsId(0), inodeId(0) {}