From b0148e250affae2d37b91af4f3ee6bd503f5bdf9 Mon Sep 17 00:00:00 2001 From: koko2pp Date: Mon, 27 Nov 2023 11:58:26 +0800 Subject: [PATCH] Code formatting changes Signed-off-by: koko2pp --- curvefs_python/cbd_client.h | 111 +- curvefs_python/libcurvefs.h | 65 +- nebd/src/common/name_lock.h | 149 +- src/chunkserver/clone_core.cpp | 871 +- src/client/request_sender.h | 293 +- src/snapshotcloneserver/clone/clone_core.cpp | 3270 ++++--- test/chunkserver/copyset_node_test.cpp | 2111 ++--- test/client/copyset_client_test.cpp | 7994 +++++++++-------- test/client/mds_failover_test.cpp | 500 +- .../snapshotcloneserver_common_test.cpp | 1864 ++-- .../snapshotcloneserver_test.cpp | 246 +- .../alloc_statistic_helper_test.cpp | 292 +- .../allocstatistic/alloc_statistic_test.cpp | 395 +- 13 files changed, 9330 insertions(+), 8831 deletions(-) diff --git a/curvefs_python/cbd_client.h b/curvefs_python/cbd_client.h index a5415b26e3..c9c0133ed9 100644 --- a/curvefs_python/cbd_client.h +++ b/curvefs_python/cbd_client.h @@ -29,59 +29,62 @@ #include "curvefs_python/curve_type.h" -namespace curve { -namespace client { - -class FileClient; - -} // namespace client -} // namespace curve - -class CBDClient { - public: - CBDClient(); - ~CBDClient(); - - int Init(const char* configPath); - void UnInit(); - - int Open(const char* filename, UserInfo_t* userInfo); - int Close(int fd); - - int Create(const char* filename, UserInfo_t* userInfo, size_t size); - int Create2(const CreateContext* context); - int Unlink(const char* filename, UserInfo_t* info); - int DeleteForce(const char* filename, UserInfo_t* info); - int Recover(const char* filename, UserInfo_t* info, uint64_t fileId); - int Rename(UserInfo_t* info, const char* oldpath, const char* newpath); - int Extend(const char* filename, UserInfo_t* info, uint64_t size); - - // Synchronous read and write - int Read(int fd, char* buf, unsigned long offset, - unsigned long length); // NOLINT - int Write(int fd, const char* buf, unsigned long offset, - unsigned long length); // NOLINT - - // Asynchronous read and write - int AioRead(int fd, AioContext* aioctx); - int AioWrite(int fd, AioContext* aioctx); - - // Obtain basic information about the file - int StatFile(const char* filename, UserInfo_t* info, FileInfo_t* finfo); - int ChangeOwner(const char* filename, const char* owner, UserInfo_t* info); - - DirInfos_t* OpenDir(const char* dirpath, UserInfo_t* userinfo); - int Listdir(DirInfos_t* dirinfo); - void CloseDir(DirInfos_t* dirinfo); - int Mkdir(const char* dirpath, UserInfo_t* info); - int Rmdir(const char* dirpath, UserInfo_t* info); - - std::string GetClusterId(); - - std::vector ListPoolset(); - - private: - std::unique_ptr client_; +namespace curve +{ + namespace client + { + + class FileClient; + + } // namespace client +} // namespace curve + +class CBDClient +{ +public: + CBDClient(); + ~CBDClient(); + + int Init(const char *configPath); + void UnInit(); + + int Open(const char *filename, UserInfo_t *userInfo); + int Close(int fd); + + int Create(const char *filename, UserInfo_t *userInfo, size_t size); + int Create2(const CreateContext *context); + int Unlink(const char *filename, UserInfo_t *info); + int DeleteForce(const char *filename, UserInfo_t *info); + int Recover(const char *filename, UserInfo_t *info, uint64_t fileId); + int Rename(UserInfo_t *info, const char *oldpath, const char *newpath); + int Extend(const char *filename, UserInfo_t *info, uint64_t size); + + // Synchronous read and write + int Read(int fd, char *buf, unsigned long offset, + unsigned long length); // NOLINT + int Write(int fd, const char *buf, unsigned long offset, + unsigned long length); // NOLINT + + // Asynchronous read and write + int AioRead(int fd, AioContext *aioctx); + int AioWrite(int fd, AioContext *aioctx); + + // Obtain basic information about the file + int StatFile(const char *filename, UserInfo_t *info, FileInfo_t *finfo); + int ChangeOwner(const char *filename, const char *owner, UserInfo_t *info); + + DirInfos_t *OpenDir(const char *dirpath, UserInfo_t *userinfo); + int Listdir(DirInfos_t *dirinfo); + void CloseDir(DirInfos_t *dirinfo); + int Mkdir(const char *dirpath, UserInfo_t *info); + int Rmdir(const char *dirpath, UserInfo_t *info); + + std::string GetClusterId(); + + std::vector ListPoolset(); + +private: + std::unique_ptr client_; }; -#endif // CURVEFS_PYTHON_CBD_CLIENT_H_ +#endif // CURVEFS_PYTHON_CBD_CLIENT_H_ diff --git a/curvefs_python/libcurvefs.h b/curvefs_python/libcurvefs.h index 069c4542f4..b1bdb0275c 100644 --- a/curvefs_python/libcurvefs.h +++ b/curvefs_python/libcurvefs.h @@ -19,7 +19,7 @@ * File Created: Tuesday, 25th September 2018 2:07:05 pm * Author: */ -#ifndef CURVE_LIBCURVE_INTERFACE_H // NOLINT +#ifndef CURVE_LIBCURVE_INTERFACE_H // NOLINT #define CURVE_LIBCURVE_INTERFACE_H #include @@ -31,47 +31,48 @@ #include "curvefs_python/curve_type.h" #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif -int Init(const char* path); -int Open4Qemu(const char* filename); -int Open(const char* filename, UserInfo_t* info); -int Create(const char* filename, UserInfo_t* info, size_t size); + int Init(const char *path); + int Open4Qemu(const char *filename); + int Open(const char *filename, UserInfo_t *info); + int Create(const char *filename, UserInfo_t *info, size_t size); -// Synchronous read and write -int Read(int fd, char* buf, unsigned long offset, - unsigned long length); // NOLINT -int Write(int fd, const char* buf, unsigned long offset, - unsigned long length); // NOLINT + // Synchronous read and write + int Read(int fd, char *buf, unsigned long offset, + unsigned long length); // NOLINT + int Write(int fd, const char *buf, unsigned long offset, + unsigned long length); // NOLINT -// Asynchronous read and write -int AioRead(int fd, AioContext* aioctx); -int AioWrite(int fd, AioContext* aioctx); + // Asynchronous read and write + int AioRead(int fd, AioContext *aioctx); + int AioWrite(int fd, AioContext *aioctx); -// Obtain basic information about the file -int StatFile4Qemu(const char* filename, FileInfo_t* finfo); -int StatFile(const char* filename, UserInfo_t* info, FileInfo_t* finfo); -int ChangeOwner(const char* filename, const char* owner, UserInfo_t* info); -int Close(int fd); + // Obtain basic information about the file + int StatFile4Qemu(const char *filename, FileInfo_t *finfo); + int StatFile(const char *filename, UserInfo_t *info, FileInfo_t *finfo); + int ChangeOwner(const char *filename, const char *owner, UserInfo_t *info); + int Close(int fd); -int Rename(UserInfo_t* info, const char* oldpath, const char* newpath); -int Extend(const char* filename, UserInfo_t* info, uint64_t size); -int Unlink(const char* filename, UserInfo_t* info); -int Recover(const char* filename, UserInfo_t* info, uint64_t fileId); -int DeleteForce(const char* filename, UserInfo_t* info); -DirInfos_t* OpenDir(const char* dirpath, UserInfo_t* userinfo); -void CloseDir(DirInfos_t* dirinfo); -int Listdir(DirInfos_t* dirinfo); -int Mkdir(const char* dirpath, UserInfo_t* info); -int Rmdir(const char* dirpath, UserInfo_t* info); + int Rename(UserInfo_t *info, const char *oldpath, const char *newpath); + int Extend(const char *filename, UserInfo_t *info, uint64_t size); + int Unlink(const char *filename, UserInfo_t *info); + int Recover(const char *filename, UserInfo_t *info, uint64_t fileId); + int DeleteForce(const char *filename, UserInfo_t *info); + DirInfos_t *OpenDir(const char *dirpath, UserInfo_t *userinfo); + void CloseDir(DirInfos_t *dirinfo); + int Listdir(DirInfos_t *dirinfo); + int Mkdir(const char *dirpath, UserInfo_t *info); + int Rmdir(const char *dirpath, UserInfo_t *info); -void UnInit(); + void UnInit(); -int GetClusterId(char* buf = nullptr, int len = 0); + int GetClusterId(char *buf = nullptr, int len = 0); #ifdef __cplusplus } #endif -#endif // !CURVE_LIBCURVE_INTERFACE_H //NOLINT +#endif // !CURVE_LIBCURVE_INTERFACE_H //NOLINT diff --git a/nebd/src/common/name_lock.h b/nebd/src/common/name_lock.h index e179c4272d..eaebf6e806 100644 --- a/nebd/src/common/name_lock.h +++ b/nebd/src/common/name_lock.h @@ -25,80 +25,87 @@ #include #include -#include // NOLINT +#include // NOLINT #include #include #include #include "nebd/src/common/uncopyable.h" -namespace nebd { -namespace common { - -class NameLock : public Uncopyable { - public: - explicit NameLock(int bucketNum = 256); - - /** - * @brief locks the specified string - * - * @param lockStr locked string - */ - void Lock(const std::string& lockStr); - - /** - * @brief Attempt to specify sting lock - * - * @param lockStr locked string - * - * @retval succeeded - * @retval failed - */ - bool TryLock(const std::string& lockStr); - - /** - * @brief unlocks the specified string - * - * @param lockStr locked string - */ - void Unlock(const std::string& lockStr); - - private: - struct LockEntry { - std::atomic ref_; - std::mutex lock_; - }; - using LockEntryPtr = std::shared_ptr; - - struct LockBucket { - std::mutex mu; - std::unordered_map lockMap; - }; - using LockBucketPtr = std::shared_ptr; - - int GetBucketOffset(const std::string& lockStr); - - private: - std::vector locks_; -}; - -class NameLockGuard : public Uncopyable { - public: - NameLockGuard(NameLock& lock, const std::string& lockStr) - : // NOLINT - lock_(lock), - lockStr_(lockStr) { - lock_.Lock(lockStr_); - } - - ~NameLockGuard() { lock_.Unlock(lockStr_); } - - private: - NameLock& lock_; - std::string lockStr_; -}; - -} // namespace common -} // namespace nebd - -#endif // NEBD_SRC_COMMON_NAME_LOCK_H_ +namespace nebd +{ + namespace common + { + + class NameLock : public Uncopyable + { + public: + explicit NameLock(int bucketNum = 256); + + /** + * @brief locks the specified string + * + * @param lockStr locked string + */ + void Lock(const std::string &lockStr); + + /** + * @brief Attempt to specify sting lock + * + * @param lockStr locked string + * + * @retval succeeded + * @retval failed + */ + bool TryLock(const std::string &lockStr); + + /** + * @brief unlocks the specified string + * + * @param lockStr locked string + */ + void Unlock(const std::string &lockStr); + + private: + struct LockEntry + { + std::atomic ref_; + std::mutex lock_; + }; + using LockEntryPtr = std::shared_ptr; + + struct LockBucket + { + std::mutex mu; + std::unordered_map lockMap; + }; + using LockBucketPtr = std::shared_ptr; + + int GetBucketOffset(const std::string &lockStr); + + private: + std::vector locks_; + }; + + class NameLockGuard : public Uncopyable + { + public: + NameLockGuard(NameLock &lock, const std::string &lockStr) + : // NOLINT + lock_(lock), + lockStr_(lockStr) + { + lock_.Lock(lockStr_); + } + + ~NameLockGuard() { lock_.Unlock(lockStr_); } + + private: + NameLock &lock_; + std::string lockStr_; + }; + + } // namespace common +} // namespace nebd + +#endif // NEBD_SRC_COMMON_NAME_LOCK_H_ diff --git a/src/chunkserver/clone_core.cpp b/src/chunkserver/clone_core.cpp index 99eb260a95..422a5cce31 100644 --- a/src/chunkserver/clone_core.cpp +++ b/src/chunkserver/clone_core.cpp @@ -32,424 +32,467 @@ #include "src/common/bitmap.h" #include "src/common/timeutility.h" -namespace curve { -namespace chunkserver { - -using curve::common::Bitmap; -using curve::common::TimeUtility; - -static void ReadBufferDeleter(void* ptr) { delete[] static_cast(ptr); } - -DownloadClosure::DownloadClosure(std::shared_ptr readRequest, - std::shared_ptr cloneCore, - AsyncDownloadContext* downloadCtx, - Closure* done) - : isFailed_(false), - beginTime_(TimeUtility::GetTimeofDayUs()), - downloadCtx_(downloadCtx), - cloneCore_(cloneCore), - readRequest_(readRequest), - done_(done) { - // Record initial metric - if (readRequest_ != nullptr) { - const ChunkRequest* request = readRequest_->GetChunkRequest(); - ChunkServerMetric* csMetric = ChunkServerMetric::GetInstance(); - csMetric->OnRequest(request->logicpoolid(), request->copysetid(), - CSIOMetricType::DOWNLOAD); - } -} - -void DownloadClosure::Run() { - std::unique_ptr selfGuard(this); - std::unique_ptr contextGuard(downloadCtx_); - brpc::ClosureGuard doneGuard(done_); - butil::IOBuf copyData; - copyData.append_user_data(downloadCtx_->buf, downloadCtx_->size, - ReadBufferDeleter); - - CHECK(readRequest_ != nullptr) << "read request is nullptr."; - // Record End Metric - const ChunkRequest* request = readRequest_->GetChunkRequest(); - ChunkServerMetric* csMetric = ChunkServerMetric::GetInstance(); - uint64_t latencyUs = TimeUtility::GetTimeofDayUs() - beginTime_; - csMetric->OnResponse(request->logicpoolid(), request->copysetid(), - CSIOMetricType::DOWNLOAD, downloadCtx_->size, - latencyUs, isFailed_); - - // Copying data from the source failed - if (isFailed_) { - LOG(ERROR) << "download origin data failed: " - << " logic pool id: " << request->logicpoolid() - << " copyset id: " << request->copysetid() - << " chunkid: " << request->chunkid() - << " AsyncDownloadContext: " << *downloadCtx_; - cloneCore_->SetResponse( - readRequest_, CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - return; - } - - if (CHUNK_OP_TYPE::CHUNK_OP_RECOVER == request->optype()) { - // Release doneGuard, hand over the closure to the pass request for - // processing - cloneCore_->PasteCloneData(readRequest_, ©Data, - downloadCtx_->offset, downloadCtx_->size, - doneGuard.release()); - } else if (CHUNK_OP_TYPE::CHUNK_OP_READ == request->optype()) { - // Error or end of processing call closure returned to user - cloneCore_->SetReadChunkResponse(readRequest_, ©Data); - - // Paste clone data is an asynchronous operation that can be processed - // quickly - cloneCore_->PasteCloneData(readRequest_, ©Data, - downloadCtx_->offset, downloadCtx_->size, - nullptr); - } -} - -void CloneClosure::Run() { - // Release resources - std::unique_ptr selfGuard(this); - std::unique_ptr requestGuard(request_); - std::unique_ptr responseGuard(response_); - brpc::ClosureGuard doneGuard(done_); - // If userResponse is not empty, you need to set the response_ Assign the - // relevant content in to userResponse - if (userResponse_ != nullptr) { - if (response_->has_status()) { - userResponse_->set_status(response_->status()); +namespace curve +{ + namespace chunkserver + { + + using curve::common::Bitmap; + using curve::common::TimeUtility; + + static void ReadBufferDeleter(void *ptr) { delete[] static_cast(ptr); } + + DownloadClosure::DownloadClosure(std::shared_ptr readRequest, + std::shared_ptr cloneCore, + AsyncDownloadContext *downloadCtx, + Closure *done) + : isFailed_(false), + beginTime_(TimeUtility::GetTimeofDayUs()), + downloadCtx_(downloadCtx), + cloneCore_(cloneCore), + readRequest_(readRequest), + done_(done) + { + // Record initial metric + if (readRequest_ != nullptr) + { + const ChunkRequest *request = readRequest_->GetChunkRequest(); + ChunkServerMetric *csMetric = ChunkServerMetric::GetInstance(); + csMetric->OnRequest(request->logicpoolid(), request->copysetid(), + CSIOMetricType::DOWNLOAD); + } } - if (response_->has_redirect()) { - userResponse_->set_redirect(response_->redirect()); + + void DownloadClosure::Run() + { + std::unique_ptr selfGuard(this); + std::unique_ptr contextGuard(downloadCtx_); + brpc::ClosureGuard doneGuard(done_); + butil::IOBuf copyData; + copyData.append_user_data(downloadCtx_->buf, downloadCtx_->size, + ReadBufferDeleter); + + CHECK(readRequest_ != nullptr) << "read request is nullptr."; + // Record End Metric + const ChunkRequest *request = readRequest_->GetChunkRequest(); + ChunkServerMetric *csMetric = ChunkServerMetric::GetInstance(); + uint64_t latencyUs = TimeUtility::GetTimeofDayUs() - beginTime_; + csMetric->OnResponse(request->logicpoolid(), request->copysetid(), + CSIOMetricType::DOWNLOAD, downloadCtx_->size, + latencyUs, isFailed_); + + // Copying data from the source failed + if (isFailed_) + { + LOG(ERROR) << "download origin data failed: " + << " logic pool id: " << request->logicpoolid() + << " copyset id: " << request->copysetid() + << " chunkid: " << request->chunkid() + << " AsyncDownloadContext: " << *downloadCtx_; + cloneCore_->SetResponse( + readRequest_, CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + return; + } + + if (CHUNK_OP_TYPE::CHUNK_OP_RECOVER == request->optype()) + { + // Release doneGuard, hand over the closure to the pass request for + // processing + cloneCore_->PasteCloneData(readRequest_, ©Data, + downloadCtx_->offset, downloadCtx_->size, + doneGuard.release()); + } + else if (CHUNK_OP_TYPE::CHUNK_OP_READ == request->optype()) + { + // Error or end of processing call closure returned to user + cloneCore_->SetReadChunkResponse(readRequest_, ©Data); + + // Paste clone data is an asynchronous operation that can be processed + // quickly + cloneCore_->PasteCloneData(readRequest_, ©Data, + downloadCtx_->offset, downloadCtx_->size, + nullptr); + } } - if (response_->has_appliedindex()) { - userResponse_->set_appliedindex(response_->appliedindex()); + + void CloneClosure::Run() + { + // Release resources + std::unique_ptr selfGuard(this); + std::unique_ptr requestGuard(request_); + std::unique_ptr responseGuard(response_); + brpc::ClosureGuard doneGuard(done_); + // If userResponse is not empty, you need to set the response_ Assign the + // relevant content in to userResponse + if (userResponse_ != nullptr) + { + if (response_->has_status()) + { + userResponse_->set_status(response_->status()); + } + if (response_->has_redirect()) + { + userResponse_->set_redirect(response_->redirect()); + } + if (response_->has_appliedindex()) + { + userResponse_->set_appliedindex(response_->appliedindex()); + } + } } - } -} - -int CloneCore::CloneReadByLocalInfo( - std::shared_ptr readRequest, const CSChunkInfo& chunkInfo, - Closure* done) { - brpc::ClosureGuard doneGuard(done); - const ChunkRequest* request = readRequest->request_; - off_t offset = request->offset(); - size_t length = request->size(); - const uint32_t blockSize = chunkInfo.blockSize; - - // offset and length must be aligned with blockSize - if (offset % blockSize != 0 || length % blockSize != 0) { - LOG(ERROR) << "Invalid offset or length: " - << " logic pool id: " << request->logicpoolid() - << " copyset id: " << request->copysetid() - << " chunkid: " << request->chunkid() - << " offset: " << offset << " length: " << length - << " block size: " << blockSize; - SetResponse(readRequest, - CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - return -1; - } - - uint32_t beginIndex = offset / blockSize; - uint32_t endIndex = (offset + length - 1) / blockSize; - - // When submitting a request to CloneManager, the chunk must be a clone - // chunk However, due to other requests for the same chunk, it is possible - // that the chunk has already been overwritten at this time So here we need - // to first determine whether the chunk is a clone chunk, and then determine - // whether to copy the data if so - bool needClone = chunkInfo.isClone && - (chunkInfo.bitmap->NextClearBit(beginIndex, endIndex) != - Bitmap::NO_POS); - if (needClone) { - // The TODO(yyk) block can be optimized, but the optimization method may - // determine complex conditions Currently, the decision to trigger - // copying is only based on whether there are unwritten pages If the - // data within the requested read range in the chunk has a page that has - // not been written, it is necessary to copy the data from the source - // side - AsyncDownloadContext* downloadCtx = - new (std::nothrow) AsyncDownloadContext; - downloadCtx->location = chunkInfo.location; - downloadCtx->offset = offset; - downloadCtx->size = length; - downloadCtx->buf = new (std::nothrow) char[length]; - DownloadClosure* downloadClosure = new (std::nothrow) DownloadClosure( - readRequest, shared_from_this(), downloadCtx, doneGuard.release()); - copyer_->DownloadAsync(downloadClosure); - return 0; - } - - // Performing this step indicates that there is no need to copy data. If it - // is a recover request, it can directly return success If it is a ReadChunk - // request, read the chunk directly and return - if (CHUNK_OP_TYPE::CHUNK_OP_RECOVER == request->optype()) { - SetResponse(readRequest, CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - } else if (CHUNK_OP_TYPE::CHUNK_OP_READ == request->optype()) { - // Error or end of processing call closure returned to user - return ReadChunk(readRequest); - } - return 0; -} - -void CloneCore::CloneReadByRequestInfo( - std::shared_ptr readRequest, Closure* done) { - brpc::ClosureGuard doneGuard(done); - const ChunkRequest* chunkRequest = readRequest->request_; - - auto func = ::curve::common::LocationOperator::GenerateCurveLocation; - std::string location = - func(chunkRequest->clonefilesource(), chunkRequest->clonefileoffset()); - - AsyncDownloadContext* downloadCtx = new (std::nothrow) AsyncDownloadContext; - downloadCtx->location = location; - downloadCtx->offset = chunkRequest->offset(); - downloadCtx->size = chunkRequest->size(); - downloadCtx->buf = new (std::nothrow) char[chunkRequest->size()]; - DownloadClosure* downloadClosure = new (std::nothrow) DownloadClosure( - readRequest, shared_from_this(), downloadCtx, doneGuard.release()); - copyer_->DownloadAsync(downloadClosure); - return; -} - -int CloneCore::HandleReadRequest(std::shared_ptr readRequest, - Closure* done) { - brpc::ClosureGuard doneGuard(done); - const ChunkRequest* request = readRequest->request_; - - // Obtain chunk information - CSChunkInfo chunkInfo; - ChunkID id = readRequest->ChunkId(); - std::shared_ptr dataStore = readRequest->datastore_; - CSErrorCode errorCode = dataStore->GetChunkInfo(id, &chunkInfo); - - /* - * Chunk exists: Check and analyze Bitmap to determine if it can be read - * locally Chunk does not exist: if it contains clone information, it will be - * read from clonesource, otherwise an error will be returned Because the - * upper level ReadChunkRequest::OnApply has already processed NoExist And - * the situation where cloneinfo does not exist - */ - switch (errorCode) { - case CSErrorCode::Success: - return CloneReadByLocalInfo(readRequest, chunkInfo, - doneGuard.release()); - case CSErrorCode::ChunkNotExistError: - if (existCloneInfo(request)) { - CloneReadByRequestInfo(readRequest, doneGuard.release()); + + int CloneCore::CloneReadByLocalInfo( + std::shared_ptr readRequest, const CSChunkInfo &chunkInfo, + Closure *done) + { + brpc::ClosureGuard doneGuard(done); + const ChunkRequest *request = readRequest->request_; + off_t offset = request->offset(); + size_t length = request->size(); + const uint32_t blockSize = chunkInfo.blockSize; + + // offset and length must be aligned with blockSize + if (offset % blockSize != 0 || length % blockSize != 0) + { + LOG(ERROR) << "Invalid offset or length: " + << " logic pool id: " << request->logicpoolid() + << " copyset id: " << request->copysetid() + << " chunkid: " << request->chunkid() + << " offset: " << offset << " length: " << length + << " block size: " << blockSize; + SetResponse(readRequest, + CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + return -1; + } + + uint32_t beginIndex = offset / blockSize; + uint32_t endIndex = (offset + length - 1) / blockSize; + + // When submitting a request to CloneManager, the chunk must be a clone + // chunk However, due to other requests for the same chunk, it is possible + // that the chunk has already been overwritten at this time So here we need + // to first determine whether the chunk is a clone chunk, and then determine + // whether to copy the data if so + bool needClone = chunkInfo.isClone && + (chunkInfo.bitmap->NextClearBit(beginIndex, endIndex) != + Bitmap::NO_POS); + if (needClone) + { + // The TODO(yyk) block can be optimized, but the optimization method may + // determine complex conditions Currently, the decision to trigger + // copying is only based on whether there are unwritten pages If the + // data within the requested read range in the chunk has a page that has + // not been written, it is necessary to copy the data from the source + // side + AsyncDownloadContext *downloadCtx = + new (std::nothrow) AsyncDownloadContext; + downloadCtx->location = chunkInfo.location; + downloadCtx->offset = offset; + downloadCtx->size = length; + downloadCtx->buf = new (std::nothrow) char[length]; + DownloadClosure *downloadClosure = new (std::nothrow) DownloadClosure( + readRequest, shared_from_this(), downloadCtx, doneGuard.release()); + copyer_->DownloadAsync(downloadClosure); return 0; } - // Otherwise, fallthrough will directly return an error - FALLTHROUGH_INTENDED; - default: - LOG(ERROR) << "get chunkinfo failed: " - << " logic pool id: " << request->logicpoolid() - << " copyset id: " << request->copysetid() - << " chunkid: " << request->chunkid() - << " error code: " << errorCode; - SetResponse(readRequest, - CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - return -1; - } -} - -int CloneCore::ReadChunk(std::shared_ptr readRequest) { - const ChunkRequest* request = readRequest->request_; - off_t offset = request->offset(); - size_t length = request->size(); - std::unique_ptr chunkData(new char[length]); - std::shared_ptr dataStore = readRequest->datastore_; - CSErrorCode errorCode; - errorCode = dataStore->ReadChunk(request->chunkid(), request->sn(), - chunkData.get(), offset, length); - if (CSErrorCode::Success != errorCode) { - SetResponse(readRequest, - CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - LOG(ERROR) << "read chunk failed: " - << " logic pool id: " << request->logicpoolid() - << " copyset id: " << request->copysetid() - << " chunkid: " << request->chunkid() - << " read offset: " << offset << " read length: " << length - << " error code: " << errorCode; - return -1; - } - - // After successful reading, update the apply index - readRequest->node_->UpdateAppliedIndex(readRequest->applyIndex); - // After completing the data reading, Return can return the results to the - // user - readRequest->cntl_->response_attachment().append(chunkData.get(), length); - SetResponse(readRequest, CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - return 0; -} - -int CloneCore::SetReadChunkResponse( - std::shared_ptr readRequest, - const butil::IOBuf* cloneData) { - const ChunkRequest* request = readRequest->request_; - CSChunkInfo chunkInfo; - ChunkID id = readRequest->ChunkId(); - std::shared_ptr dataStore = readRequest->datastore_; - CSErrorCode errorCode = dataStore->GetChunkInfo(id, &chunkInfo); - - // If the chunk does not exist, it is necessary to determine whether the - // request contains information about the source chunk If the source chunk - // information is provided, it indicates that the lazy allocation chunk - // mechanism is used, and clone data can be directly returned There is a - // situation where the requested chunk is lazily allocated and the requested - // chunk exists locally, And the requested read area has already been - // written, and when copying data from the source, the chunk has been - // deleted again In this case, it will be returned as a normal request, but - // the returned data does not meet expectations Due to the current delayed - // deletion of our curve files, it is ensured that there is no user IO when - // the files are truly deleted If some changes are added later that trigger - // this issue, it needs to be fixed - // TODO(yyk) fix it - bool expect = errorCode == CSErrorCode::Success || - (errorCode == CSErrorCode::ChunkNotExistError && - existCloneInfo(request)); - if (!expect) { - LOG(ERROR) << "get chunkinfo failed: " - << " logic pool id: " << request->logicpoolid() - << " copyset id: " << request->copysetid() - << " chunkid: " << request->chunkid() - << " error code: " << errorCode; - SetResponse(readRequest, - CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - return -1; - } - - size_t length = request->size(); - butil::IOBuf responseData; - // If a chunk exists, read the regions that have already been written from - // the chunk and merge them back - if (errorCode == CSErrorCode::Success) { - char* chunkData = new (std::nothrow) char[length]; - int ret = ReadThenMerge(readRequest, chunkInfo, cloneData, chunkData); - responseData.append_user_data(chunkData, length, ReadBufferDeleter); - if (ret < 0) { - SetResponse(readRequest, - CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - return ret; + + // Performing this step indicates that there is no need to copy data. If it + // is a recover request, it can directly return success If it is a ReadChunk + // request, read the chunk directly and return + if (CHUNK_OP_TYPE::CHUNK_OP_RECOVER == request->optype()) + { + SetResponse(readRequest, CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + } + else if (CHUNK_OP_TYPE::CHUNK_OP_READ == request->optype()) + { + // Error or end of processing call closure returned to user + return ReadChunk(readRequest); + } + return 0; + } + + void CloneCore::CloneReadByRequestInfo( + std::shared_ptr readRequest, Closure *done) + { + brpc::ClosureGuard doneGuard(done); + const ChunkRequest *chunkRequest = readRequest->request_; + + auto func = ::curve::common::LocationOperator::GenerateCurveLocation; + std::string location = + func(chunkRequest->clonefilesource(), chunkRequest->clonefileoffset()); + + AsyncDownloadContext *downloadCtx = new (std::nothrow) AsyncDownloadContext; + downloadCtx->location = location; + downloadCtx->offset = chunkRequest->offset(); + downloadCtx->size = chunkRequest->size(); + downloadCtx->buf = new (std::nothrow) char[chunkRequest->size()]; + DownloadClosure *downloadClosure = new (std::nothrow) DownloadClosure( + readRequest, shared_from_this(), downloadCtx, doneGuard.release()); + copyer_->DownloadAsync(downloadClosure); + return; } - } else { - responseData = *cloneData; - } - readRequest->cntl_->response_attachment().append(responseData); - - // After successful reading, update the apply index - readRequest->node_->UpdateAppliedIndex(readRequest->applyIndex); - SetResponse(readRequest, CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - return 0; -} - -int CloneCore::ReadThenMerge(std::shared_ptr readRequest, - const CSChunkInfo& chunkInfo, - const butil::IOBuf* cloneData, char* chunkData) { - const ChunkRequest* request = readRequest->request_; - std::shared_ptr dataStore = readRequest->datastore_; - - off_t offset = request->offset(); - size_t length = request->size(); - uint32_t blockSize = chunkInfo.blockSize; - uint32_t beginIndex = offset / blockSize; - uint32_t endIndex = (offset + length - 1) / blockSize; - // Obtain the regions where the chunk file has been written and not written - std::vector copiedRanges; - std::vector uncopiedRanges; - if (chunkInfo.isClone) { - chunkInfo.bitmap->Divide(beginIndex, endIndex, &uncopiedRanges, - &copiedRanges); - } else { - BitRange range; - range.beginIndex = beginIndex; - range.endIndex = endIndex; - copiedRanges.push_back(range); - } - - // The offset of the starting position to be read in the chunk - off_t readOff; - // The relative offset of the read data to be copied into the buffer - off_t relativeOff; - // The length of data read from chunk each time - size_t readSize; - // 1. Read for regions that have already been written, read from the chunk - // file - CSErrorCode errorCode; - for (auto& range : copiedRanges) { - readOff = range.beginIndex * blockSize; - readSize = (range.endIndex - range.beginIndex + 1) * blockSize; - relativeOff = readOff - offset; - errorCode = - dataStore->ReadChunk(request->chunkid(), request->sn(), - chunkData + relativeOff, readOff, readSize); - if (CSErrorCode::Success != errorCode) { - LOG(ERROR) << "read chunk failed: " - << " logic pool id: " << request->logicpoolid() - << " copyset id: " << request->copysetid() - << " chunkid: " << request->chunkid() - << " read offset: " << readOff - << " read length: " << readSize - << " error code: " << errorCode; - return -1; + + int CloneCore::HandleReadRequest(std::shared_ptr readRequest, + Closure *done) + { + brpc::ClosureGuard doneGuard(done); + const ChunkRequest *request = readRequest->request_; + + // Obtain chunk information + CSChunkInfo chunkInfo; + ChunkID id = readRequest->ChunkId(); + std::shared_ptr dataStore = readRequest->datastore_; + CSErrorCode errorCode = dataStore->GetChunkInfo(id, &chunkInfo); + + /* + * Chunk exists: Check and analyze Bitmap to determine if it can be read + * locally Chunk does not exist: if it contains clone information, it will be + * read from clonesource, otherwise an error will be returned Because the + * upper level ReadChunkRequest::OnApply has already processed NoExist And + * the situation where cloneinfo does not exist + */ + switch (errorCode) + { + case CSErrorCode::Success: + return CloneReadByLocalInfo(readRequest, chunkInfo, + doneGuard.release()); + case CSErrorCode::ChunkNotExistError: + if (existCloneInfo(request)) + { + CloneReadByRequestInfo(readRequest, doneGuard.release()); + return 0; + } + // Otherwise, fallthrough will directly return an error + FALLTHROUGH_INTENDED; + default: + LOG(ERROR) << "get chunkinfo failed: " + << " logic pool id: " << request->logicpoolid() + << " copyset id: " << request->copysetid() + << " chunkid: " << request->chunkid() + << " error code: " << errorCode; + SetResponse(readRequest, + CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + return -1; + } } - } - - // 2. Merge: For areas that have not been written before, copy them from the - // downloaded area on the source side for merging - for (auto& range : uncopiedRanges) { - readOff = range.beginIndex * blockSize; - readSize = (range.endIndex - range.beginIndex + 1) * blockSize; - relativeOff = readOff - offset; - cloneData->copy_to(chunkData + relativeOff, readSize, relativeOff); - } - return 0; -} - -void CloneCore::PasteCloneData(std::shared_ptr readRequest, - const butil::IOBuf* cloneData, off_t offset, - size_t cloneDataSize, Closure* done) { - const ChunkRequest* request = readRequest->request_; - bool dontPaste = - CHUNK_OP_TYPE::CHUNK_OP_READ == request->optype() && !enablePaste_; - if (dontPaste) return; - - // After the data copy is completed, it is necessary to generate a - // PaseChunkRequest and paste the data to the chunk file - ChunkRequest* pasteRequest = new ChunkRequest(); - pasteRequest->set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_PASTE); - pasteRequest->set_logicpoolid(request->logicpoolid()); - pasteRequest->set_copysetid(request->copysetid()); - pasteRequest->set_chunkid(request->chunkid()); - pasteRequest->set_offset(offset); - pasteRequest->set_size(cloneDataSize); - std::shared_ptr req = nullptr; - - ChunkResponse* pasteResponse = new ChunkResponse(); - CloneClosure* closure = new CloneClosure(); - closure->SetRequest(pasteRequest); - closure->SetResponse(pasteResponse); - closure->SetClosure(done); - // If it is a request for a recover chunk, the result of the pass needs to - // be returned through rpc - if (CHUNK_OP_TYPE::CHUNK_OP_RECOVER == request->optype()) { - closure->SetUserResponse(readRequest->response_); - } - - ChunkServiceClosure* pasteClosure = new (std::nothrow) - ChunkServiceClosure(nullptr, pasteRequest, pasteResponse, closure); - - req = std::make_shared( - readRequest->node_, pasteRequest, pasteResponse, cloneData, - pasteClosure); - req->Process(); -} - -inline void CloneCore::SetResponse( - std::shared_ptr readRequest, CHUNK_OP_STATUS status) { - auto applyIndex = readRequest->node_->GetAppliedIndex(); - readRequest->response_->set_appliedindex(applyIndex); - readRequest->response_->set_status(status); -} - -} // namespace chunkserver -} // namespace curve + + int CloneCore::ReadChunk(std::shared_ptr readRequest) + { + const ChunkRequest *request = readRequest->request_; + off_t offset = request->offset(); + size_t length = request->size(); + std::unique_ptr chunkData(new char[length]); + std::shared_ptr dataStore = readRequest->datastore_; + CSErrorCode errorCode; + errorCode = dataStore->ReadChunk(request->chunkid(), request->sn(), + chunkData.get(), offset, length); + if (CSErrorCode::Success != errorCode) + { + SetResponse(readRequest, + CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + LOG(ERROR) << "read chunk failed: " + << " logic pool id: " << request->logicpoolid() + << " copyset id: " << request->copysetid() + << " chunkid: " << request->chunkid() + << " read offset: " << offset << " read length: " << length + << " error code: " << errorCode; + return -1; + } + + // After successful reading, update the apply index + readRequest->node_->UpdateAppliedIndex(readRequest->applyIndex); + // After completing the data reading, Return can return the results to the + // user + readRequest->cntl_->response_attachment().append(chunkData.get(), length); + SetResponse(readRequest, CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + return 0; + } + + int CloneCore::SetReadChunkResponse( + std::shared_ptr readRequest, + const butil::IOBuf *cloneData) + { + const ChunkRequest *request = readRequest->request_; + CSChunkInfo chunkInfo; + ChunkID id = readRequest->ChunkId(); + std::shared_ptr dataStore = readRequest->datastore_; + CSErrorCode errorCode = dataStore->GetChunkInfo(id, &chunkInfo); + + // If the chunk does not exist, it is necessary to determine whether the + // request contains information about the source chunk If the source chunk + // information is provided, it indicates that the lazy allocation chunk + // mechanism is used, and clone data can be directly returned There is a + // situation where the requested chunk is lazily allocated and the requested + // chunk exists locally, And the requested read area has already been + // written, and when copying data from the source, the chunk has been + // deleted again In this case, it will be returned as a normal request, but + // the returned data does not meet expectations Due to the current delayed + // deletion of our curve files, it is ensured that there is no user IO when + // the files are truly deleted If some changes are added later that trigger + // this issue, it needs to be fixed + // TODO(yyk) fix it + bool expect = errorCode == CSErrorCode::Success || + (errorCode == CSErrorCode::ChunkNotExistError && + existCloneInfo(request)); + if (!expect) + { + LOG(ERROR) << "get chunkinfo failed: " + << " logic pool id: " << request->logicpoolid() + << " copyset id: " << request->copysetid() + << " chunkid: " << request->chunkid() + << " error code: " << errorCode; + SetResponse(readRequest, + CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + return -1; + } + + size_t length = request->size(); + butil::IOBuf responseData; + // If a chunk exists, read the regions that have already been written from + // the chunk and merge them back + if (errorCode == CSErrorCode::Success) + { + char *chunkData = new (std::nothrow) char[length]; + int ret = ReadThenMerge(readRequest, chunkInfo, cloneData, chunkData); + responseData.append_user_data(chunkData, length, ReadBufferDeleter); + if (ret < 0) + { + SetResponse(readRequest, + CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + return ret; + } + } + else + { + responseData = *cloneData; + } + readRequest->cntl_->response_attachment().append(responseData); + + // After successful reading, update the apply index + readRequest->node_->UpdateAppliedIndex(readRequest->applyIndex); + SetResponse(readRequest, CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + return 0; + } + + int CloneCore::ReadThenMerge(std::shared_ptr readRequest, + const CSChunkInfo &chunkInfo, + const butil::IOBuf *cloneData, char *chunkData) + { + const ChunkRequest *request = readRequest->request_; + std::shared_ptr dataStore = readRequest->datastore_; + + off_t offset = request->offset(); + size_t length = request->size(); + uint32_t blockSize = chunkInfo.blockSize; + uint32_t beginIndex = offset / blockSize; + uint32_t endIndex = (offset + length - 1) / blockSize; + // Obtain the regions where the chunk file has been written and not written + std::vector copiedRanges; + std::vector uncopiedRanges; + if (chunkInfo.isClone) + { + chunkInfo.bitmap->Divide(beginIndex, endIndex, &uncopiedRanges, + &copiedRanges); + } + else + { + BitRange range; + range.beginIndex = beginIndex; + range.endIndex = endIndex; + copiedRanges.push_back(range); + } + + // The offset of the starting position to be read in the chunk + off_t readOff; + // The relative offset of the read data to be copied into the buffer + off_t relativeOff; + // The length of data read from chunk each time + size_t readSize; + // 1. Read for regions that have already been written, read from the chunk + // file + CSErrorCode errorCode; + for (auto &range : copiedRanges) + { + readOff = range.beginIndex * blockSize; + readSize = (range.endIndex - range.beginIndex + 1) * blockSize; + relativeOff = readOff - offset; + errorCode = + dataStore->ReadChunk(request->chunkid(), request->sn(), + chunkData + relativeOff, readOff, readSize); + if (CSErrorCode::Success != errorCode) + { + LOG(ERROR) << "read chunk failed: " + << " logic pool id: " << request->logicpoolid() + << " copyset id: " << request->copysetid() + << " chunkid: " << request->chunkid() + << " read offset: " << readOff + << " read length: " << readSize + << " error code: " << errorCode; + return -1; + } + } + + // 2. Merge: For areas that have not been written before, copy them from the + // downloaded area on the source side for merging + for (auto &range : uncopiedRanges) + { + readOff = range.beginIndex * blockSize; + readSize = (range.endIndex - range.beginIndex + 1) * blockSize; + relativeOff = readOff - offset; + cloneData->copy_to(chunkData + relativeOff, readSize, relativeOff); + } + return 0; + } + + void CloneCore::PasteCloneData(std::shared_ptr readRequest, + const butil::IOBuf *cloneData, off_t offset, + size_t cloneDataSize, Closure *done) + { + const ChunkRequest *request = readRequest->request_; + bool dontPaste = + CHUNK_OP_TYPE::CHUNK_OP_READ == request->optype() && !enablePaste_; + if (dontPaste) + return; + + // After the data copy is completed, it is necessary to generate a + // PaseChunkRequest and paste the data to the chunk file + ChunkRequest *pasteRequest = new ChunkRequest(); + pasteRequest->set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_PASTE); + pasteRequest->set_logicpoolid(request->logicpoolid()); + pasteRequest->set_copysetid(request->copysetid()); + pasteRequest->set_chunkid(request->chunkid()); + pasteRequest->set_offset(offset); + pasteRequest->set_size(cloneDataSize); + std::shared_ptr req = nullptr; + + ChunkResponse *pasteResponse = new ChunkResponse(); + CloneClosure *closure = new CloneClosure(); + closure->SetRequest(pasteRequest); + closure->SetResponse(pasteResponse); + closure->SetClosure(done); + // If it is a request for a recover chunk, the result of the pass needs to + // be returned through rpc + if (CHUNK_OP_TYPE::CHUNK_OP_RECOVER == request->optype()) + { + closure->SetUserResponse(readRequest->response_); + } + + ChunkServiceClosure *pasteClosure = new (std::nothrow) + ChunkServiceClosure(nullptr, pasteRequest, pasteResponse, closure); + + req = std::make_shared( + readRequest->node_, pasteRequest, pasteResponse, cloneData, + pasteClosure); + req->Process(); + } + + inline void CloneCore::SetResponse( + std::shared_ptr readRequest, CHUNK_OP_STATUS status) + { + auto applyIndex = readRequest->node_->GetAppliedIndex(); + readRequest->response_->set_appliedindex(applyIndex); + readRequest->response_->set_status(status); + } + + } // namespace chunkserver +} // namespace curve diff --git a/src/client/request_sender.h b/src/client/request_sender.h index 99bc94b2e3..a08be423d6 100644 --- a/src/client/request_sender.h +++ b/src/client/request_sender.h @@ -35,148 +35,151 @@ #include "src/client/client_config.h" #include "src/client/request_context.h" -namespace curve { -namespace client { - -/** - * A RequestSender is responsible for managing all aspects of a ChunkServer - * Connection, currently there is only one connection for a ChunkServer - */ -class RequestSender { - public: - RequestSender(ChunkServerID chunkServerId, butil::EndPoint serverEndPoint) - : chunkServerId_(chunkServerId), - serverEndPoint_(serverEndPoint), - channel_() {} - virtual ~RequestSender() {} - - int Init(const IOSenderOption& ioSenderOpt); - - /** - * Reading Chunk - * @param IDInfo is the ID information related to chunk - * @param sn: File version number - * @param offset: Read offset - * @param length: Read length - * @param sourceInfo Data source information - * @param done: closure of asynchronous callback on the previous layer - */ - int ReadChunk(const ChunkIDInfo& idinfo, uint64_t sn, off_t offset, - size_t length, const RequestSourceInfo& sourceInfo, - ClientClosure* done); - - /** - * Write Chunk - * @param IDInfo is the ID information related to chunk - * @param fileId: file id - * @param epoch: file epoch - * @param sn: File version number - * @param data The data to be written - * @param offset: write offset - * @param length: The length written - * @param sourceInfo Data source information - * @param done: closure of asynchronous callback on the previous layer - */ - int WriteChunk(const ChunkIDInfo& idinfo, uint64_t fileId, uint64_t epoch, - uint64_t sn, const butil::IOBuf& data, off_t offset, - size_t length, const RequestSourceInfo& sourceInfo, - ClientClosure* done); - - /** - * Reading Chunk snapshot files - * @param IDInfo is the ID information related to chunk - * @param sn: File version number - * @param offset: Read offset - * @param length: Read length - * @param done: closure of asynchronous callback on the previous layer - */ - int ReadChunkSnapshot(const ChunkIDInfo& idinfo, uint64_t sn, off_t offset, - size_t length, ClientClosure* done); - - /** - * Delete snapshots generated during this dump or left over from history - * If no snapshot is generated during the dump process, modify the - * correctedSn of the chunk - * @param IDInfo is the ID information related to chunk - * @param correctedSn: Chunk The version number that needs to be corrected - * @param done: closure of asynchronous callback on the previous layer - */ - int DeleteChunkSnapshotOrCorrectSn(const ChunkIDInfo& idinfo, - uint64_t correctedSn, - ClientClosure* done); - - /** - * Obtain information about chunk files - * @param IDInfo is the ID information related to chunk - * @param done: closure of asynchronous callback on the previous layer - * @param retriedTimes: Number of retries - */ - int GetChunkInfo(const ChunkIDInfo& idinfo, ClientClosure* done); - - /** - * @brief lazy Create clone chunk - * @detail - * - The format definition of a location is A@B The form of. - * - If the source data is on s3, the location format is uri@s3 Uri is the - * address of the actual chunk object; - * - If the source data is on curves, the location format - * is/filename/chunkindex@cs - * - * @param IDInfo is the ID information related to chunk - * @param done: closure of asynchronous callback on the previous layer - * @param: location, URL of the data source - * @param: sn chunk's serial number - * @param: correntSn used to modify the chunk when creating CloneChunk - * @param: chunkSize Chunk size - * @param retriedTimes: Number of retries - * - * @return error code - */ - int CreateCloneChunk(const ChunkIDInfo& idinfo, ClientClosure* done, - const std::string& location, uint64_t sn, - uint64_t correntSn, uint64_t chunkSize); - - /** - * @brief Actual recovery chunk data - * @param IDInfo is the ID information related to chunk - * @param done: closure of asynchronous callback on the previous layer - * @param: offset: offset - * @param: len: length - * @param retriedTimes: Number of retries - * - * @return error code - */ - int RecoverChunk(const ChunkIDInfo& idinfo, ClientClosure* done, - uint64_t offset, uint64_t len); - /** - * Reset Link to Chunk Server - * @param chunkServerId: Chunk Server unique identifier - * @param serverEndPoint: Chunk Server - * @return 0 succeeded, -1 failed - */ - int ResetSender(ChunkServerID chunkServerId, - butil::EndPoint serverEndPoint); - - bool IsSocketHealth() { return channel_.CheckHealth() == 0; } - - private: - void UpdateRpcRPS(ClientClosure* done, OpType type) const; - - void SetRpcStuff(ClientClosure* done, brpc::Controller* cntl, - google::protobuf::Message* rpcResponse) const; - - private: - // Rpc stub configuration - IOSenderOption iosenderopt_; - // The unique identification ID of ChunkServer - ChunkServerID chunkServerId_; - // Address of ChunkServer - butil::EndPoint serverEndPoint_; - brpc::Channel channel_; /* TODO(wudemiao): Multiple channels will be - maintained in the later stage */ -}; - -} // namespace client -} // namespace curve - -#endif // SRC_CLIENT_REQUEST_SENDER_H_ +namespace curve +{ + namespace client + { + + /** + * A RequestSender is responsible for managing all aspects of a ChunkServer + * Connection, currently there is only one connection for a ChunkServer + */ + class RequestSender + { + public: + RequestSender(ChunkServerID chunkServerId, butil::EndPoint serverEndPoint) + : chunkServerId_(chunkServerId), + serverEndPoint_(serverEndPoint), + channel_() {} + virtual ~RequestSender() {} + + int Init(const IOSenderOption &ioSenderOpt); + + /** + * Reading Chunk + * @param IDInfo is the ID information related to chunk + * @param sn: File version number + * @param offset: Read offset + * @param length: Read length + * @param sourceInfo Data source information + * @param done: closure of asynchronous callback on the previous layer + */ + int ReadChunk(const ChunkIDInfo &idinfo, uint64_t sn, off_t offset, + size_t length, const RequestSourceInfo &sourceInfo, + ClientClosure *done); + + /** + * Write Chunk + * @param IDInfo is the ID information related to chunk + * @param fileId: file id + * @param epoch: file epoch + * @param sn: File version number + * @param data The data to be written + * @param offset: write offset + * @param length: The length written + * @param sourceInfo Data source information + * @param done: closure of asynchronous callback on the previous layer + */ + int WriteChunk(const ChunkIDInfo &idinfo, uint64_t fileId, uint64_t epoch, + uint64_t sn, const butil::IOBuf &data, off_t offset, + size_t length, const RequestSourceInfo &sourceInfo, + ClientClosure *done); + + /** + * Reading Chunk snapshot files + * @param IDInfo is the ID information related to chunk + * @param sn: File version number + * @param offset: Read offset + * @param length: Read length + * @param done: closure of asynchronous callback on the previous layer + */ + int ReadChunkSnapshot(const ChunkIDInfo &idinfo, uint64_t sn, off_t offset, + size_t length, ClientClosure *done); + + /** + * Delete snapshots generated during this dump or left over from history + * If no snapshot is generated during the dump process, modify the + * correctedSn of the chunk + * @param IDInfo is the ID information related to chunk + * @param correctedSn: Chunk The version number that needs to be corrected + * @param done: closure of asynchronous callback on the previous layer + */ + int DeleteChunkSnapshotOrCorrectSn(const ChunkIDInfo &idinfo, + uint64_t correctedSn, + ClientClosure *done); + + /** + * Obtain information about chunk files + * @param IDInfo is the ID information related to chunk + * @param done: closure of asynchronous callback on the previous layer + * @param retriedTimes: Number of retries + */ + int GetChunkInfo(const ChunkIDInfo &idinfo, ClientClosure *done); + + /** + * @brief lazy Create clone chunk + * @detail + * - The format definition of a location is A@B The form of. + * - If the source data is on s3, the location format is uri@s3 Uri is the + * address of the actual chunk object; + * - If the source data is on curves, the location format + * is/filename/chunkindex@cs + * + * @param IDInfo is the ID information related to chunk + * @param done: closure of asynchronous callback on the previous layer + * @param: location, URL of the data source + * @param: sn chunk's serial number + * @param: correntSn used to modify the chunk when creating CloneChunk + * @param: chunkSize Chunk size + * @param retriedTimes: Number of retries + * + * @return error code + */ + int CreateCloneChunk(const ChunkIDInfo &idinfo, ClientClosure *done, + const std::string &location, uint64_t sn, + uint64_t correntSn, uint64_t chunkSize); + + /** + * @brief Actual recovery chunk data + * @param IDInfo is the ID information related to chunk + * @param done: closure of asynchronous callback on the previous layer + * @param: offset: offset + * @param: len: length + * @param retriedTimes: Number of retries + * + * @return error code + */ + int RecoverChunk(const ChunkIDInfo &idinfo, ClientClosure *done, + uint64_t offset, uint64_t len); + /** + * Reset Link to Chunk Server + * @param chunkServerId: Chunk Server unique identifier + * @param serverEndPoint: Chunk Server + * @return 0 succeeded, -1 failed + */ + int ResetSender(ChunkServerID chunkServerId, + butil::EndPoint serverEndPoint); + + bool IsSocketHealth() { return channel_.CheckHealth() == 0; } + + private: + void UpdateRpcRPS(ClientClosure *done, OpType type) const; + + void SetRpcStuff(ClientClosure *done, brpc::Controller *cntl, + google::protobuf::Message *rpcResponse) const; + + private: + // Rpc stub configuration + IOSenderOption iosenderopt_; + // The unique identification ID of ChunkServer + ChunkServerID chunkServerId_; + // Address of ChunkServer + butil::EndPoint serverEndPoint_; + brpc::Channel channel_; /* TODO(wudemiao): Multiple channels will be + maintained in the later stage */ + }; + + } // namespace client +} // namespace curve + +#endif // SRC_CLIENT_REQUEST_SENDER_H_ diff --git a/src/snapshotcloneserver/clone/clone_core.cpp b/src/snapshotcloneserver/clone/clone_core.cpp index 021da6b359..5620a4ff63 100644 --- a/src/snapshotcloneserver/clone/clone_core.cpp +++ b/src/snapshotcloneserver/clone/clone_core.cpp @@ -37,1605 +37,1903 @@ using ::curve::common::NameLock; using ::curve::common::NameLockGuard; using ::curve::common::UUIDGenerator; -namespace curve { -namespace snapshotcloneserver { - -int CloneCoreImpl::Init() { - int ret = client_->Mkdir(cloneTempDir_, mdsRootUser_); - if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::EXISTS) { - LOG(ERROR) << "Mkdir fail, ret = " << ret - << ", dirpath = " << cloneTempDir_; - return kErrCodeServerInitFail; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::CloneOrRecoverPre(const UUID& source, - const std::string& user, - const std::string& destination, - bool lazyFlag, CloneTaskType taskType, - std::string poolset, - CloneInfo* cloneInfo) { - // Check if there are tasks executing in the database - std::vector cloneInfoList; - metaStore_->GetCloneInfoByFileName(destination, &cloneInfoList); - bool needJudgeFileExist = false; - std::vector existCloneInfos; - for (auto& info : cloneInfoList) { - LOG(INFO) << "CloneOrRecoverPre find same clone task" - << ", source = " << source << ", user = " << user - << ", destination = " << destination - << ", poolset = " << poolset - << ", Exist CloneInfo : " << info; - // is clone - if (taskType == CloneTaskType::kClone) { - if (info.GetStatus() == CloneStatus::cloning || - info.GetStatus() == CloneStatus::retrying) { - if ((info.GetUser() == user) && (info.GetSrc() == source) && - (info.GetIsLazy() == lazyFlag) && - (info.GetTaskType() == taskType)) { - // Treat as the same clone - *cloneInfo = info; - return kErrCodeTaskExist; - } else { - // Treat it as a different clone, then the file is actually - // occupied and the return file already exists - return kErrCodeFileExist; - } - } else if (info.GetStatus() == CloneStatus::done || - info.GetStatus() == CloneStatus::error || - info.GetStatus() == CloneStatus::metaInstalled) { - // It may have been deleted, and it is necessary to determine - // whether the file exists again, Allowing further cloning under - // deleted conditions - existCloneInfos.push_back(info); - needJudgeFileExist = true; - } else { - // At this point, the same clone task is being deleted and the - // return file is occupied - return kErrCodeFileExist; - } - } else { // is recover - if (info.GetStatus() == CloneStatus::recovering || - info.GetStatus() == CloneStatus::retrying) { - if ((info.GetUser() == user) && (info.GetSrc() == source) && - (info.GetIsLazy() == lazyFlag) && - (info.GetTaskType() == taskType)) { - // Treat as the same clone, return task already exists - *cloneInfo = info; - return kErrCodeTaskExist; - } else { - // Treat it as a different clone, then the file is actually - // occupied and the return file already exists - return kErrCodeFileExist; - } - } else if (info.GetStatus() == CloneStatus::done || - info.GetStatus() == CloneStatus::error || - info.GetStatus() == CloneStatus::metaInstalled) { - // nothing - } else { - // At this point, the same task is being deleted and the return - // file is occupied - return kErrCodeFileExist; +namespace curve +{ + namespace snapshotcloneserver + { + + int CloneCoreImpl::Init() + { + int ret = client_->Mkdir(cloneTempDir_, mdsRootUser_); + if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::EXISTS) + { + LOG(ERROR) << "Mkdir fail, ret = " << ret + << ", dirpath = " << cloneTempDir_; + return kErrCodeServerInitFail; } + return kErrCodeSuccess; } - } - - // The target file already exists and cannot be cloned or recovered if it - // does not exist - FInfo destFInfo; - int ret = client_->GetFileInfo(destination, mdsRootUser_, &destFInfo); - switch (ret) { - case LIBCURVE_ERROR::OK: - if (CloneTaskType::kClone == taskType) { - if (needJudgeFileExist) { - bool match = false; - // Find the cloneInfo that matches the inodeid - for (auto& existInfo : existCloneInfos) { - if (destFInfo.id == existInfo.GetDestId()) { - *cloneInfo = existInfo; - match = true; - break; + + int CloneCoreImpl::CloneOrRecoverPre(const UUID &source, + const std::string &user, + const std::string &destination, + bool lazyFlag, CloneTaskType taskType, + std::string poolset, + CloneInfo *cloneInfo) + { + // Check if there are tasks executing in the database + std::vector cloneInfoList; + metaStore_->GetCloneInfoByFileName(destination, &cloneInfoList); + bool needJudgeFileExist = false; + std::vector existCloneInfos; + for (auto &info : cloneInfoList) + { + LOG(INFO) << "CloneOrRecoverPre find same clone task" + << ", source = " << source << ", user = " << user + << ", destination = " << destination + << ", poolset = " << poolset + << ", Exist CloneInfo : " << info; + // is clone + if (taskType == CloneTaskType::kClone) + { + if (info.GetStatus() == CloneStatus::cloning || + info.GetStatus() == CloneStatus::retrying) + { + if ((info.GetUser() == user) && (info.GetSrc() == source) && + (info.GetIsLazy() == lazyFlag) && + (info.GetTaskType() == taskType)) + { + // Treat as the same clone + *cloneInfo = info; + return kErrCodeTaskExist; + } + else + { + // Treat it as a different clone, then the file is actually + // occupied and the return file already exists + return kErrCodeFileExist; } } - if (match) { - return kErrCodeTaskExist; - } else { - // If not found, then none of the dest files were - // created by these clone tasks, It means the file has a - // duplicate name - LOG(ERROR) - << "Clone dest file exist, " - << "but task not match! " - << "source = " << source << ", user = " << user - << ", destination = " << destination - << ", poolset = " << poolset; + else if (info.GetStatus() == CloneStatus::done || + info.GetStatus() == CloneStatus::error || + info.GetStatus() == CloneStatus::metaInstalled) + { + // It may have been deleted, and it is necessary to determine + // whether the file exists again, Allowing further cloning under + // deleted conditions + existCloneInfos.push_back(info); + needJudgeFileExist = true; + } + else + { + // At this point, the same clone task is being deleted and the + // return file is occupied + return kErrCodeFileExist; + } + } + else + { // is recover + if (info.GetStatus() == CloneStatus::recovering || + info.GetStatus() == CloneStatus::retrying) + { + if ((info.GetUser() == user) && (info.GetSrc() == source) && + (info.GetIsLazy() == lazyFlag) && + (info.GetTaskType() == taskType)) + { + // Treat as the same clone, return task already exists + *cloneInfo = info; + return kErrCodeTaskExist; + } + else + { + // Treat it as a different clone, then the file is actually + // occupied and the return file already exists + return kErrCodeFileExist; + } + } + else if (info.GetStatus() == CloneStatus::done || + info.GetStatus() == CloneStatus::error || + info.GetStatus() == CloneStatus::metaInstalled) + { + // nothing + } + else + { + // At this point, the same task is being deleted and the return + // file is occupied return kErrCodeFileExist; } - } else { - // There is no corresponding cloneInfo, which means the file - // has a duplicate name - LOG(ERROR) << "Clone dest file must not exist" - << ", source = " << source << ", user = " << user - << ", destination = " << destination - << ", poolset = " << poolset; - return kErrCodeFileExist; } - } else if (CloneTaskType::kRecover == taskType) { - // The recover task keeps the poolset information of the volume - // unchanged - poolset = destFInfo.poolset; - } else { - assert(false); - } - break; - case -LIBCURVE_ERROR::NOTEXIST: - if (CloneTaskType::kRecover == taskType) { - LOG(ERROR) << "Recover dest file must exist" - << ", source = " << source << ", user = " << user - << ", destination = " << destination; - return kErrCodeFileNotExist; } - break; - default: - LOG(ERROR) << "GetFileInfo encounter an error" - << ", ret = " << ret << ", source = " << source - << ", user = " << user; - return kErrCodeInternalError; - } - // Is it a snapshot - SnapshotInfo snapInfo; - CloneFileType fileType; - - { - NameLockGuard lockSnapGuard(snapshotRef_->GetSnapshotLock(), source); - ret = metaStore_->GetSnapshotInfo(source, &snapInfo); - if (0 == ret) { - if (CloneTaskType::kRecover == taskType && - destination != snapInfo.GetFileName()) { - LOG(ERROR) << "Can not recover from the snapshot " - << "which is not belong to the destination volume."; - return kErrCodeInvalidSnapshot; - } - if (snapInfo.GetStatus() != Status::done) { - LOG(ERROR) << "Can not clone by snapshot has status:" - << static_cast(snapInfo.GetStatus()); - return kErrCodeInvalidSnapshot; - } - if (snapInfo.GetUser() != user) { - LOG(ERROR) << "Clone snapshot by invalid user" - << ", source = " << source << ", user = " << user - << ", destination = " << destination - << ", poolset = " << poolset - << ", snapshot.user = " << snapInfo.GetUser(); - return kErrCodeInvalidUser; - } - fileType = CloneFileType::kSnapshot; - snapshotRef_->IncrementSnapshotRef(source); - } - } - if (ret < 0) { - FInfo fInfo; - ret = client_->GetFileInfo(source, mdsRootUser_, &fInfo); - switch (ret) { + // The target file already exists and cannot be cloned or recovered if it + // does not exist + FInfo destFInfo; + int ret = client_->GetFileInfo(destination, mdsRootUser_, &destFInfo); + switch (ret) + { case LIBCURVE_ERROR::OK: - fileType = CloneFileType::kFile; + if (CloneTaskType::kClone == taskType) + { + if (needJudgeFileExist) + { + bool match = false; + // Find the cloneInfo that matches the inodeid + for (auto &existInfo : existCloneInfos) + { + if (destFInfo.id == existInfo.GetDestId()) + { + *cloneInfo = existInfo; + match = true; + break; + } + } + if (match) + { + return kErrCodeTaskExist; + } + else + { + // If not found, then none of the dest files were + // created by these clone tasks, It means the file has a + // duplicate name + LOG(ERROR) + << "Clone dest file exist, " + << "but task not match! " + << "source = " << source << ", user = " << user + << ", destination = " << destination + << ", poolset = " << poolset; + return kErrCodeFileExist; + } + } + else + { + // There is no corresponding cloneInfo, which means the file + // has a duplicate name + LOG(ERROR) << "Clone dest file must not exist" + << ", source = " << source << ", user = " << user + << ", destination = " << destination + << ", poolset = " << poolset; + return kErrCodeFileExist; + } + } + else if (CloneTaskType::kRecover == taskType) + { + // The recover task keeps the poolset information of the volume + // unchanged + poolset = destFInfo.poolset; + } + else + { + assert(false); + } break; case -LIBCURVE_ERROR::NOTEXIST: - case -LIBCURVE_ERROR::PARAM_ERROR: - LOG(ERROR) << "Clone source file not exist" - << ", source = " << source << ", user = " << user - << ", destination = " << destination - << ", poolset = " << poolset; - return kErrCodeFileNotExist; + if (CloneTaskType::kRecover == taskType) + { + LOG(ERROR) << "Recover dest file must exist" + << ", source = " << source << ", user = " << user + << ", destination = " << destination; + return kErrCodeFileNotExist; + } + break; default: LOG(ERROR) << "GetFileInfo encounter an error" << ", ret = " << ret << ", source = " << source << ", user = " << user; return kErrCodeInternalError; - } - if (fInfo.filestatus != FileStatus::Created && - fInfo.filestatus != FileStatus::Cloned && - fInfo.filestatus != FileStatus::BeingCloned) { - LOG(ERROR) << "Can not clone when file status = " - << static_cast(fInfo.filestatus); - return kErrCodeFileStatusInvalid; - } + } - // TODO (User authentication for mirror cloning to be improved) - } - - UUID uuid = UUIDGenerator().GenerateUUID(); - CloneInfo info(uuid, user, taskType, source, destination, poolset, fileType, - lazyFlag); - if (CloneTaskType::kClone == taskType) { - info.SetStatus(CloneStatus::cloning); - } else { - info.SetStatus(CloneStatus::recovering); - } - // Here, you must first AddCloneInfo because if you first set - // CloneFileStatus and then AddCloneInfo, If AddCloneInfo fails and - // unexpectedly restarts, no one will know that SetCloneFileStatus has been - // called, causing Mirror cannot be deleted - ret = metaStore_->AddCloneInfo(info); - if (ret < 0) { - LOG(ERROR) << "AddCloneInfo error" - << ", ret = " << ret << ", taskId = " << uuid - << ", user = " << user << ", source = " << source - << ", destination = " << destination - << ", poolset = " << poolset; - if (CloneFileType::kSnapshot == fileType) { - snapshotRef_->DecrementSnapshotRef(source); - } - return ret; - } - if (CloneFileType::kFile == fileType) { - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - ret = client_->SetCloneFileStatus(source, FileStatus::BeingCloned, - mdsRootUser_); - if (ret < 0) { - // The SetCloneFileStatus error is not handled here, - // Because all results of SetCloneFileStatus failure are acceptable, - // Compared to handling SetCloneFileStatus failure, it is more - // direct: For example, calling DeleteCloneInfo to delete a task, - // Once DeleteCloneInfo fails and an error is returned to the user, - // Restarting the service will cause Clone to continue, - // Inconsistency with the results returned by the user, causing - // confusion for the user - LOG(WARNING) << "SetCloneFileStatus encounter an error" - << ", ret = " << ret << ", source = " << source - << ", user = " << user; - } - cloneRef_->IncrementRef(source); - } - - *cloneInfo = info; - return kErrCodeSuccess; -} - -int CloneCoreImpl::FlattenPre(const std::string& user, const TaskIdType& taskId, - CloneInfo* cloneInfo) { - (void)user; - int ret = metaStore_->GetCloneInfo(taskId, cloneInfo); - if (ret < 0) { - return kErrCodeFileNotExist; - } - switch (cloneInfo->GetStatus()) { - case CloneStatus::done: - case CloneStatus::cloning: - case CloneStatus::recovering: { - // A task exists is returned for completed or in progress, - // indicating that it does not need to be processed - return kErrCodeTaskExist; - } - case CloneStatus::metaInstalled: { - if (CloneTaskType::kClone == cloneInfo->GetTaskType()) { - cloneInfo->SetStatus(CloneStatus::cloning); - } else { - cloneInfo->SetStatus(CloneStatus::recovering); + // Is it a snapshot + SnapshotInfo snapInfo; + CloneFileType fileType; + + { + NameLockGuard lockSnapGuard(snapshotRef_->GetSnapshotLock(), source); + ret = metaStore_->GetSnapshotInfo(source, &snapInfo); + if (0 == ret) + { + if (CloneTaskType::kRecover == taskType && + destination != snapInfo.GetFileName()) + { + LOG(ERROR) << "Can not recover from the snapshot " + << "which is not belong to the destination volume."; + return kErrCodeInvalidSnapshot; + } + if (snapInfo.GetStatus() != Status::done) + { + LOG(ERROR) << "Can not clone by snapshot has status:" + << static_cast(snapInfo.GetStatus()); + return kErrCodeInvalidSnapshot; + } + if (snapInfo.GetUser() != user) + { + LOG(ERROR) << "Clone snapshot by invalid user" + << ", source = " << source << ", user = " << user + << ", destination = " << destination + << ", poolset = " << poolset + << ", snapshot.user = " << snapInfo.GetUser(); + return kErrCodeInvalidUser; + } + fileType = CloneFileType::kSnapshot; + snapshotRef_->IncrementSnapshotRef(source); + } } - break; - } - case CloneStatus::cleaning: - case CloneStatus::errorCleaning: - case CloneStatus::error: - default: { - LOG(ERROR) << "FlattenPre find clone task status Invalid" - << ", status = " - << static_cast(cloneInfo->GetStatus()); - return kErrCodeFileStatusInvalid; - } - } - ret = metaStore_->UpdateCloneInfo(*cloneInfo); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo fail" - << ", ret = " << ret - << ", taskId = " << cloneInfo->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -void CloneCoreImpl::HandleCloneOrRecoverTask( - std::shared_ptr task) { - brpc::ClosureGuard doneGuard(task->GetClosure().get()); - int ret = kErrCodeSuccess; - FInfo newFileInfo; - CloneSegmentMap segInfos; - if (IsSnapshot(task)) { - ret = BuildFileInfoFromSnapshot(task, &newFileInfo, &segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; - } - } else { - ret = BuildFileInfoFromFile(task, &newFileInfo, &segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; - } - } - - // In the steps after kCreateCloneMeta, it is necessary to update the - // chunkIdInfo in the CloneChunkInfo information - if (NeedUpdateCloneMeta(task)) { - ret = CreateOrUpdateCloneMeta(task, &newFileInfo, &segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; - } - } - - CloneStep step = task->GetCloneInfo().GetNextStep(); - while (step != CloneStep::kEnd) { - switch (step) { - case CloneStep::kCreateCloneFile: - ret = CreateCloneFile(task, newFileInfo); - if (ret < 0) { - HandleCloneError(task, ret); - return; + if (ret < 0) + { + FInfo fInfo; + ret = client_->GetFileInfo(source, mdsRootUser_, &fInfo); + switch (ret) + { + case LIBCURVE_ERROR::OK: + fileType = CloneFileType::kFile; + break; + case -LIBCURVE_ERROR::NOTEXIST: + case -LIBCURVE_ERROR::PARAM_ERROR: + LOG(ERROR) << "Clone source file not exist" + << ", source = " << source << ", user = " << user + << ", destination = " << destination + << ", poolset = " << poolset; + return kErrCodeFileNotExist; + default: + LOG(ERROR) << "GetFileInfo encounter an error" + << ", ret = " << ret << ", source = " << source + << ", user = " << user; + return kErrCodeInternalError; } - task->SetProgress(kProgressCreateCloneFile); - break; - case CloneStep::kCreateCloneMeta: - ret = CreateCloneMeta(task, &newFileInfo, &segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; + if (fInfo.filestatus != FileStatus::Created && + fInfo.filestatus != FileStatus::Cloned && + fInfo.filestatus != FileStatus::BeingCloned) + { + LOG(ERROR) << "Can not clone when file status = " + << static_cast(fInfo.filestatus); + return kErrCodeFileStatusInvalid; } - task->SetProgress(kProgressCreateCloneMeta); - break; - case CloneStep::kCreateCloneChunk: - ret = CreateCloneChunk(task, newFileInfo, &segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; + + // TODO (User authentication for mirror cloning to be improved) + } + + UUID uuid = UUIDGenerator().GenerateUUID(); + CloneInfo info(uuid, user, taskType, source, destination, poolset, fileType, + lazyFlag); + if (CloneTaskType::kClone == taskType) + { + info.SetStatus(CloneStatus::cloning); + } + else + { + info.SetStatus(CloneStatus::recovering); + } + // Here, you must first AddCloneInfo because if you first set + // CloneFileStatus and then AddCloneInfo, If AddCloneInfo fails and + // unexpectedly restarts, no one will know that SetCloneFileStatus has been + // called, causing Mirror cannot be deleted + ret = metaStore_->AddCloneInfo(info); + if (ret < 0) + { + LOG(ERROR) << "AddCloneInfo error" + << ", ret = " << ret << ", taskId = " << uuid + << ", user = " << user << ", source = " << source + << ", destination = " << destination + << ", poolset = " << poolset; + if (CloneFileType::kSnapshot == fileType) + { + snapshotRef_->DecrementSnapshotRef(source); } - break; - case CloneStep::kCompleteCloneMeta: - ret = CompleteCloneMeta(task, newFileInfo, segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; + return ret; + } + if (CloneFileType::kFile == fileType) + { + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + ret = client_->SetCloneFileStatus(source, FileStatus::BeingCloned, + mdsRootUser_); + if (ret < 0) + { + // The SetCloneFileStatus error is not handled here, + // Because all results of SetCloneFileStatus failure are acceptable, + // Compared to handling SetCloneFileStatus failure, it is more + // direct: For example, calling DeleteCloneInfo to delete a task, + // Once DeleteCloneInfo fails and an error is returned to the user, + // Restarting the service will cause Clone to continue, + // Inconsistency with the results returned by the user, causing + // confusion for the user + LOG(WARNING) << "SetCloneFileStatus encounter an error" + << ", ret = " << ret << ", source = " << source + << ", user = " << user; } - task->SetProgress(kProgressMetaInstalled); - break; - case CloneStep::kRecoverChunk: - ret = RecoverChunk(task, newFileInfo, segInfos); - if (ret < 0) { - HandleCloneError(task, ret); - return; + cloneRef_->IncrementRef(source); + } + + *cloneInfo = info; + return kErrCodeSuccess; + } + + int CloneCoreImpl::FlattenPre(const std::string &user, const TaskIdType &taskId, + CloneInfo *cloneInfo) + { + (void)user; + int ret = metaStore_->GetCloneInfo(taskId, cloneInfo); + if (ret < 0) + { + return kErrCodeFileNotExist; + } + switch (cloneInfo->GetStatus()) + { + case CloneStatus::done: + case CloneStatus::cloning: + case CloneStatus::recovering: + { + // A task exists is returned for completed or in progress, + // indicating that it does not need to be processed + return kErrCodeTaskExist; + } + case CloneStatus::metaInstalled: + { + if (CloneTaskType::kClone == cloneInfo->GetTaskType()) + { + cloneInfo->SetStatus(CloneStatus::cloning); + } + else + { + cloneInfo->SetStatus(CloneStatus::recovering); } break; - case CloneStep::kChangeOwner: - ret = ChangeOwner(task, newFileInfo); - if (ret < 0) { + } + case CloneStatus::cleaning: + case CloneStatus::errorCleaning: + case CloneStatus::error: + default: + { + LOG(ERROR) << "FlattenPre find clone task status Invalid" + << ", status = " + << static_cast(cloneInfo->GetStatus()); + return kErrCodeFileStatusInvalid; + } + } + ret = metaStore_->UpdateCloneInfo(*cloneInfo); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo fail" + << ", ret = " << ret + << ", taskId = " << cloneInfo->GetTaskId(); + return ret; + } + return kErrCodeSuccess; + } + + void CloneCoreImpl::HandleCloneOrRecoverTask( + std::shared_ptr task) + { + brpc::ClosureGuard doneGuard(task->GetClosure().get()); + int ret = kErrCodeSuccess; + FInfo newFileInfo; + CloneSegmentMap segInfos; + if (IsSnapshot(task)) + { + ret = BuildFileInfoFromSnapshot(task, &newFileInfo, &segInfos); + if (ret < 0) + { HandleCloneError(task, ret); return; } - break; - case CloneStep::kRenameCloneFile: - ret = RenameCloneFile(task, newFileInfo); - if (ret < 0) { + } + else + { + ret = BuildFileInfoFromFile(task, &newFileInfo, &segInfos); + if (ret < 0) + { HandleCloneError(task, ret); return; } - if (IsLazy(task)) { - HandleLazyCloneStage1Finish(task); - doneGuard.release(); + } + + // In the steps after kCreateCloneMeta, it is necessary to update the + // chunkIdInfo in the CloneChunkInfo information + if (NeedUpdateCloneMeta(task)) + { + ret = CreateOrUpdateCloneMeta(task, &newFileInfo, &segInfos); + if (ret < 0) + { + HandleCloneError(task, ret); return; } - break; - case CloneStep::kCompleteCloneFile: - ret = CompleteCloneFile(task, newFileInfo, segInfos); - if (ret < 0) { + } + + CloneStep step = task->GetCloneInfo().GetNextStep(); + while (step != CloneStep::kEnd) + { + switch (step) + { + case CloneStep::kCreateCloneFile: + ret = CreateCloneFile(task, newFileInfo); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + task->SetProgress(kProgressCreateCloneFile); + break; + case CloneStep::kCreateCloneMeta: + ret = CreateCloneMeta(task, &newFileInfo, &segInfos); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + task->SetProgress(kProgressCreateCloneMeta); + break; + case CloneStep::kCreateCloneChunk: + ret = CreateCloneChunk(task, newFileInfo, &segInfos); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + break; + case CloneStep::kCompleteCloneMeta: + ret = CompleteCloneMeta(task, newFileInfo, segInfos); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + task->SetProgress(kProgressMetaInstalled); + break; + case CloneStep::kRecoverChunk: + ret = RecoverChunk(task, newFileInfo, segInfos); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + break; + case CloneStep::kChangeOwner: + ret = ChangeOwner(task, newFileInfo); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + break; + case CloneStep::kRenameCloneFile: + ret = RenameCloneFile(task, newFileInfo); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + if (IsLazy(task)) + { + HandleLazyCloneStage1Finish(task); + doneGuard.release(); + return; + } + break; + case CloneStep::kCompleteCloneFile: + ret = CompleteCloneFile(task, newFileInfo, segInfos); + if (ret < 0) + { + HandleCloneError(task, ret); + return; + } + break; + default: + LOG(ERROR) << "can not reach here" + << ", taskid = " << task->GetTaskId(); HandleCloneError(task, ret); return; } - break; - default: - LOG(ERROR) << "can not reach here" - << ", taskid = " << task->GetTaskId(); - HandleCloneError(task, ret); - return; + task->UpdateMetric(); + step = task->GetCloneInfo().GetNextStep(); + } + HandleCloneSuccess(task); } - task->UpdateMetric(); - step = task->GetCloneInfo().GetNextStep(); - } - HandleCloneSuccess(task); -} - -int CloneCoreImpl::BuildFileInfoFromSnapshot( - std::shared_ptr task, FInfo* newFileInfo, - CloneSegmentMap* segInfos) { - segInfos->clear(); - UUID source = task->GetCloneInfo().GetSrc(); - - SnapshotInfo snapInfo; - int ret = metaStore_->GetSnapshotInfo(source, &snapInfo); - if (ret < 0) { - LOG(ERROR) << "GetSnapshotInfo error" - << ", source = " << source - << ", taskid = " << task->GetTaskId(); - return kErrCodeFileNotExist; - } - newFileInfo->chunksize = snapInfo.GetChunkSize(); - newFileInfo->segmentsize = snapInfo.GetSegmentSize(); - newFileInfo->length = snapInfo.GetFileLength(); - newFileInfo->stripeUnit = snapInfo.GetStripeUnit(); - newFileInfo->stripeCount = snapInfo.GetStripeCount(); - - if (task->GetCloneInfo().GetTaskType() == CloneTaskType::kRecover && - task->GetCloneInfo().GetPoolset().empty()) { - LOG(ERROR) << "Recover task's poolset should not be empty"; - return kErrCodeInternalError; - } - newFileInfo->poolset = !task->GetCloneInfo().GetPoolset().empty() - ? task->GetCloneInfo().GetPoolset() - : snapInfo.GetPoolset(); - - if (IsRecover(task)) { - FInfo fInfo; - std::string destination = task->GetCloneInfo().GetDest(); - std::string user = task->GetCloneInfo().GetUser(); - ret = client_->GetFileInfo(destination, mdsRootUser_, &fInfo); - switch (ret) { - case LIBCURVE_ERROR::OK: - break; - case -LIBCURVE_ERROR::NOTEXIST: - LOG(ERROR) << "BuildFileInfoFromSnapshot " - << "find dest file not exist, maybe deleted" - << ", ret = " << ret - << ", destination = " << destination - << ", user = " << user + + int CloneCoreImpl::BuildFileInfoFromSnapshot( + std::shared_ptr task, FInfo *newFileInfo, + CloneSegmentMap *segInfos) + { + segInfos->clear(); + UUID source = task->GetCloneInfo().GetSrc(); + + SnapshotInfo snapInfo; + int ret = metaStore_->GetSnapshotInfo(source, &snapInfo); + if (ret < 0) + { + LOG(ERROR) << "GetSnapshotInfo error" + << ", source = " << source << ", taskid = " << task->GetTaskId(); return kErrCodeFileNotExist; - default: - LOG(ERROR) << "GetFileInfo fail" - << ", ret = " << ret - << ", destination = " << destination - << ", user = " << user - << ", taskid = " << task->GetTaskId(); + } + newFileInfo->chunksize = snapInfo.GetChunkSize(); + newFileInfo->segmentsize = snapInfo.GetSegmentSize(); + newFileInfo->length = snapInfo.GetFileLength(); + newFileInfo->stripeUnit = snapInfo.GetStripeUnit(); + newFileInfo->stripeCount = snapInfo.GetStripeCount(); + + if (task->GetCloneInfo().GetTaskType() == CloneTaskType::kRecover && + task->GetCloneInfo().GetPoolset().empty()) + { + LOG(ERROR) << "Recover task's poolset should not be empty"; return kErrCodeInternalError; - } - // The destinationId recovered from the snapshot is the ID of the target - // file - task->GetCloneInfo().SetDestId(fInfo.id); - // Restore seqnum+1 from snapshot - newFileInfo->seqnum = fInfo.seqnum + 1; - } else { - newFileInfo->seqnum = kInitializeSeqNum; - } - newFileInfo->owner = task->GetCloneInfo().GetUser(); - - ChunkIndexDataName indexName(snapInfo.GetFileName(), snapInfo.GetSeqNum()); - ChunkIndexData snapMeta; - ret = dataStore_->GetChunkIndexData(indexName, &snapMeta); - if (ret < 0) { - LOG(ERROR) << "GetChunkIndexData error" - << ", fileName = " << snapInfo.GetFileName() - << ", seqNum = " << snapInfo.GetSeqNum() - << ", taskid = " << task->GetTaskId(); - return ret; - } - - uint64_t segmentSize = snapInfo.GetSegmentSize(); - uint64_t chunkSize = snapInfo.GetChunkSize(); - uint64_t chunkPerSegment = segmentSize / chunkSize; - - std::vector chunkIndexs = snapMeta.GetAllChunkIndex(); - for (auto& chunkIndex : chunkIndexs) { - ChunkDataName chunkDataName; - snapMeta.GetChunkDataName(chunkIndex, &chunkDataName); - uint64_t segmentIndex = chunkIndex / chunkPerSegment; - CloneChunkInfo info; - info.location = chunkDataName.ToDataChunkKey(); - info.needRecover = true; - if (IsRecover(task)) { - info.seqNum = chunkDataName.chunkSeqNum_; - } else { - info.seqNum = kInitializeSeqNum; - } + } + newFileInfo->poolset = !task->GetCloneInfo().GetPoolset().empty() + ? task->GetCloneInfo().GetPoolset() + : snapInfo.GetPoolset(); + + if (IsRecover(task)) + { + FInfo fInfo; + std::string destination = task->GetCloneInfo().GetDest(); + std::string user = task->GetCloneInfo().GetUser(); + ret = client_->GetFileInfo(destination, mdsRootUser_, &fInfo); + switch (ret) + { + case LIBCURVE_ERROR::OK: + break; + case -LIBCURVE_ERROR::NOTEXIST: + LOG(ERROR) << "BuildFileInfoFromSnapshot " + << "find dest file not exist, maybe deleted" + << ", ret = " << ret + << ", destination = " << destination + << ", user = " << user + << ", taskid = " << task->GetTaskId(); + return kErrCodeFileNotExist; + default: + LOG(ERROR) << "GetFileInfo fail" + << ", ret = " << ret + << ", destination = " << destination + << ", user = " << user + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + // The destinationId recovered from the snapshot is the ID of the target + // file + task->GetCloneInfo().SetDestId(fInfo.id); + // Restore seqnum+1 from snapshot + newFileInfo->seqnum = fInfo.seqnum + 1; + } + else + { + newFileInfo->seqnum = kInitializeSeqNum; + } + newFileInfo->owner = task->GetCloneInfo().GetUser(); + + ChunkIndexDataName indexName(snapInfo.GetFileName(), snapInfo.GetSeqNum()); + ChunkIndexData snapMeta; + ret = dataStore_->GetChunkIndexData(indexName, &snapMeta); + if (ret < 0) + { + LOG(ERROR) << "GetChunkIndexData error" + << ", fileName = " << snapInfo.GetFileName() + << ", seqNum = " << snapInfo.GetSeqNum() + << ", taskid = " << task->GetTaskId(); + return ret; + } - auto it = segInfos->find(segmentIndex); - if (it == segInfos->end()) { - CloneSegmentInfo segInfo; - segInfo.emplace(chunkIndex % chunkPerSegment, info); - segInfos->emplace(segmentIndex, segInfo); - } else { - it->second.emplace(chunkIndex % chunkPerSegment, info); - } - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::BuildFileInfoFromFile(std::shared_ptr task, - FInfo* newFileInfo, - CloneSegmentMap* segInfos) { - segInfos->clear(); - UUID source = task->GetCloneInfo().GetSrc(); - std::string user = task->GetCloneInfo().GetUser(); - - FInfo fInfo; - int ret = client_->GetFileInfo(source, mdsRootUser_, &fInfo); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "GetFileInfo fail" - << ", ret = " << ret << ", source = " << source - << ", user = " << user << ", taskid = " << task->GetTaskId(); - return kErrCodeFileNotExist; - } - // GetOrAllocateSegment depends on fullPathName - fInfo.fullPathName = source; - - newFileInfo->chunksize = fInfo.chunksize; - newFileInfo->segmentsize = fInfo.segmentsize; - newFileInfo->length = fInfo.length; - newFileInfo->seqnum = kInitializeSeqNum; - newFileInfo->owner = task->GetCloneInfo().GetUser(); - newFileInfo->stripeUnit = fInfo.stripeUnit; - newFileInfo->stripeCount = fInfo.stripeCount; - - if (task->GetCloneInfo().GetTaskType() == CloneTaskType::kRecover && - task->GetCloneInfo().GetPoolset().empty()) { - LOG(ERROR) << "Recover task's poolset should not be empty"; - return kErrCodeInternalError; - } - newFileInfo->poolset = !task->GetCloneInfo().GetPoolset().empty() - ? task->GetCloneInfo().GetPoolset() - : fInfo.poolset; - - uint64_t fileLength = fInfo.length; - uint64_t segmentSize = fInfo.segmentsize; - uint64_t chunkSize = fInfo.chunksize; - - if (0 == segmentSize) { - LOG(ERROR) << "GetFileInfo return invalid fileInfo, segmentSize == 0" - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - if (fileLength % segmentSize != 0) { - LOG(ERROR) << "GetFileInfo return invalid fileInfo, " - << "fileLength is not align to SegmentSize" - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - - for (uint64_t i = 0; i < fileLength / segmentSize; i++) { - uint64_t offset = i * segmentSize; - SegmentInfo segInfoOut; - ret = client_->GetOrAllocateSegmentInfo(false, offset, &fInfo, - mdsRootUser_, &segInfoOut); - if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::NOT_ALLOCATE) { - LOG(ERROR) << "GetOrAllocateSegmentInfo fail" - << ", ret = " << ret << ", filename = " << source - << ", user = " << user << ", offset = " << offset - << ", allocateIfNotExist = " - << "false" - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - if (segInfoOut.chunkvec.size() != 0) { - CloneSegmentInfo segInfo; - for (std::vector::size_type j = 0; - j < segInfoOut.chunkvec.size(); j++) { + uint64_t segmentSize = snapInfo.GetSegmentSize(); + uint64_t chunkSize = snapInfo.GetChunkSize(); + uint64_t chunkPerSegment = segmentSize / chunkSize; + + std::vector chunkIndexs = snapMeta.GetAllChunkIndex(); + for (auto &chunkIndex : chunkIndexs) + { + ChunkDataName chunkDataName; + snapMeta.GetChunkDataName(chunkIndex, &chunkDataName); + uint64_t segmentIndex = chunkIndex / chunkPerSegment; CloneChunkInfo info; - info.location = std::to_string(offset + j * chunkSize); - info.seqNum = kInitializeSeqNum; + info.location = chunkDataName.ToDataChunkKey(); info.needRecover = true; - segInfo.emplace(j, info); + if (IsRecover(task)) + { + info.seqNum = chunkDataName.chunkSeqNum_; + } + else + { + info.seqNum = kInitializeSeqNum; + } + + auto it = segInfos->find(segmentIndex); + if (it == segInfos->end()) + { + CloneSegmentInfo segInfo; + segInfo.emplace(chunkIndex % chunkPerSegment, info); + segInfos->emplace(segmentIndex, segInfo); + } + else + { + it->second.emplace(chunkIndex % chunkPerSegment, info); + } } - segInfos->emplace(i, segInfo); - } - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::CreateCloneFile(std::shared_ptr task, - const FInfo& fInfo) { - std::string fileName = - cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - std::string user = fInfo.owner; - uint64_t fileLength = fInfo.length; - uint64_t seqNum = fInfo.seqnum; - uint32_t chunkSize = fInfo.chunksize; - uint64_t stripeUnit = fInfo.stripeUnit; - uint64_t stripeCount = fInfo.stripeCount; - const auto& poolset = fInfo.poolset; - - std::string source = ""; - // Clone source is only available when cloning from a file - if (CloneFileType::kFile == task->GetCloneInfo().GetFileType()) { - source = task->GetCloneInfo().GetSrc(); - } - - FInfo fInfoOut; - int ret = client_->CreateCloneFile( - source, fileName, mdsRootUser_, fileLength, seqNum, chunkSize, - stripeUnit, stripeCount, poolset, &fInfoOut); - if (ret == LIBCURVE_ERROR::OK) { - // nothing - } else if (ret == -LIBCURVE_ERROR::EXISTS) { - ret = client_->GetFileInfo(fileName, mdsRootUser_, &fInfoOut); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "GetFileInfo fail" - << ", ret = " << ret << ", fileName = " << fileName - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; + return kErrCodeSuccess; } - } else { - LOG(ERROR) << "CreateCloneFile file" - << ", ret = " << ret << ", destination = " << fileName - << ", user = " << user << ", fileLength = " << fileLength - << ", seqNum = " << seqNum << ", chunkSize = " << chunkSize - << ", return fileId = " << fInfoOut.id - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - task->GetCloneInfo().SetOriginId(fInfoOut.id); - if (IsClone(task)) { - // In the case of cloning, destinationId = originId; - task->GetCloneInfo().SetDestId(fInfoOut.id); - } - task->GetCloneInfo().SetTime(fInfoOut.ctime); - // If it is a lazy&non snapshot, do not createCloneMeta or createCloneChunk - // yet Wait until stage 2 recoveryChunk, go to createCloneMeta, - // createCloneChunk - if (IsLazy(task) && IsFile(task)) { - task->GetCloneInfo().SetNextStep(CloneStep::kCompleteCloneMeta); - } else { - task->GetCloneInfo().SetNextStep(CloneStep::kCreateCloneMeta); - } - - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after CreateCloneFile error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::CreateCloneMeta(std::shared_ptr task, - FInfo* fInfo, CloneSegmentMap* segInfos) { - int ret = CreateOrUpdateCloneMeta(task, fInfo, segInfos); - if (ret < 0) { - return ret; - } - - task->GetCloneInfo().SetNextStep(CloneStep::kCreateCloneChunk); - - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after CreateCloneMeta error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::CreateCloneChunk(std::shared_ptr task, - const FInfo& fInfo, - CloneSegmentMap* segInfos) { - int ret = kErrCodeSuccess; - uint32_t chunkSize = fInfo.chunksize; - uint32_t correctSn = 0; - // When cloning, correctSn is 0, and when restoring, it is the newly - // generated file version - if (IsClone(task)) { - correctSn = 0; - } else { - correctSn = fInfo.seqnum; - } - auto tracker = std::make_shared(); - for (auto& cloneSegmentInfo : *segInfos) { - for (auto& cloneChunkInfo : cloneSegmentInfo.second) { - std::string location; - if (IsSnapshot(task)) { - location = LocationOperator::GenerateS3Location( - cloneChunkInfo.second.location); - } else { - location = LocationOperator::GenerateCurveLocation( - task->GetCloneInfo().GetSrc(), - std::stoull(cloneChunkInfo.second.location)); - } - ChunkIDInfo cidInfo = cloneChunkInfo.second.chunkIdInfo; - - auto context = std::make_shared(); - context->location = location; - context->cidInfo = cidInfo; - context->cloneChunkInfo = &cloneChunkInfo.second; - context->sn = cloneChunkInfo.second.seqNum; - context->csn = correctSn; - context->chunkSize = chunkSize; - context->taskid = task->GetTaskId(); - context->startTime = TimeUtility::GetTimeofDaySec(); - context->clientAsyncMethodRetryTimeSec = - clientAsyncMethodRetryTimeSec_; - - ret = StartAsyncCreateCloneChunk(task, tracker, context); - if (ret < 0) { + + int CloneCoreImpl::BuildFileInfoFromFile(std::shared_ptr task, + FInfo *newFileInfo, + CloneSegmentMap *segInfos) + { + segInfos->clear(); + UUID source = task->GetCloneInfo().GetSrc(); + std::string user = task->GetCloneInfo().GetUser(); + + FInfo fInfo; + int ret = client_->GetFileInfo(source, mdsRootUser_, &fInfo); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "GetFileInfo fail" + << ", ret = " << ret << ", source = " << source + << ", user = " << user << ", taskid = " << task->GetTaskId(); + return kErrCodeFileNotExist; + } + // GetOrAllocateSegment depends on fullPathName + fInfo.fullPathName = source; + + newFileInfo->chunksize = fInfo.chunksize; + newFileInfo->segmentsize = fInfo.segmentsize; + newFileInfo->length = fInfo.length; + newFileInfo->seqnum = kInitializeSeqNum; + newFileInfo->owner = task->GetCloneInfo().GetUser(); + newFileInfo->stripeUnit = fInfo.stripeUnit; + newFileInfo->stripeCount = fInfo.stripeCount; + + if (task->GetCloneInfo().GetTaskType() == CloneTaskType::kRecover && + task->GetCloneInfo().GetPoolset().empty()) + { + LOG(ERROR) << "Recover task's poolset should not be empty"; return kErrCodeInternalError; } + newFileInfo->poolset = !task->GetCloneInfo().GetPoolset().empty() + ? task->GetCloneInfo().GetPoolset() + : fInfo.poolset; - if (tracker->GetTaskNum() >= createCloneChunkConcurrency_) { - tracker->WaitSome(1); + uint64_t fileLength = fInfo.length; + uint64_t segmentSize = fInfo.segmentsize; + uint64_t chunkSize = fInfo.chunksize; + + if (0 == segmentSize) + { + LOG(ERROR) << "GetFileInfo return invalid fileInfo, segmentSize == 0" + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; } - std::list results = - tracker->PopResultContexts(); - ret = HandleCreateCloneChunkResultsAndRetry(task, tracker, results); - if (ret < 0) { + if (fileLength % segmentSize != 0) + { + LOG(ERROR) << "GetFileInfo return invalid fileInfo, " + << "fileLength is not align to SegmentSize" + << ", taskid = " << task->GetTaskId(); return kErrCodeInternalError; } + + for (uint64_t i = 0; i < fileLength / segmentSize; i++) + { + uint64_t offset = i * segmentSize; + SegmentInfo segInfoOut; + ret = client_->GetOrAllocateSegmentInfo(false, offset, &fInfo, + mdsRootUser_, &segInfoOut); + if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::NOT_ALLOCATE) + { + LOG(ERROR) << "GetOrAllocateSegmentInfo fail" + << ", ret = " << ret << ", filename = " << source + << ", user = " << user << ", offset = " << offset + << ", allocateIfNotExist = " + << "false" + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + if (segInfoOut.chunkvec.size() != 0) + { + CloneSegmentInfo segInfo; + for (std::vector::size_type j = 0; + j < segInfoOut.chunkvec.size(); j++) + { + CloneChunkInfo info; + info.location = std::to_string(offset + j * chunkSize); + info.seqNum = kInitializeSeqNum; + info.needRecover = true; + segInfo.emplace(j, info); + } + segInfos->emplace(i, segInfo); + } + } + return kErrCodeSuccess; } - } - // Tasks with insufficient remaining quantity in the end - do { - tracker->WaitSome(1); - std::list results = - tracker->PopResultContexts(); - if (0 == results.size()) { - // Completed, no new results - break; + + int CloneCoreImpl::CreateCloneFile(std::shared_ptr task, + const FInfo &fInfo) + { + std::string fileName = + cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + std::string user = fInfo.owner; + uint64_t fileLength = fInfo.length; + uint64_t seqNum = fInfo.seqnum; + uint32_t chunkSize = fInfo.chunksize; + uint64_t stripeUnit = fInfo.stripeUnit; + uint64_t stripeCount = fInfo.stripeCount; + const auto &poolset = fInfo.poolset; + + std::string source = ""; + // Clone source is only available when cloning from a file + if (CloneFileType::kFile == task->GetCloneInfo().GetFileType()) + { + source = task->GetCloneInfo().GetSrc(); + } + + FInfo fInfoOut; + int ret = client_->CreateCloneFile( + source, fileName, mdsRootUser_, fileLength, seqNum, chunkSize, + stripeUnit, stripeCount, poolset, &fInfoOut); + if (ret == LIBCURVE_ERROR::OK) + { + // nothing + } + else if (ret == -LIBCURVE_ERROR::EXISTS) + { + ret = client_->GetFileInfo(fileName, mdsRootUser_, &fInfoOut); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "GetFileInfo fail" + << ", ret = " << ret << ", fileName = " << fileName + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + } + else + { + LOG(ERROR) << "CreateCloneFile file" + << ", ret = " << ret << ", destination = " << fileName + << ", user = " << user << ", fileLength = " << fileLength + << ", seqNum = " << seqNum << ", chunkSize = " << chunkSize + << ", return fileId = " << fInfoOut.id + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + task->GetCloneInfo().SetOriginId(fInfoOut.id); + if (IsClone(task)) + { + // In the case of cloning, destinationId = originId; + task->GetCloneInfo().SetDestId(fInfoOut.id); + } + task->GetCloneInfo().SetTime(fInfoOut.ctime); + // If it is a lazy&non snapshot, do not createCloneMeta or createCloneChunk + // yet Wait until stage 2 recoveryChunk, go to createCloneMeta, + // createCloneChunk + if (IsLazy(task) && IsFile(task)) + { + task->GetCloneInfo().SetNextStep(CloneStep::kCompleteCloneMeta); + } + else + { + task->GetCloneInfo().SetNextStep(CloneStep::kCreateCloneMeta); + } + + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after CreateCloneFile error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return ret; + } + return kErrCodeSuccess; } - ret = HandleCreateCloneChunkResultsAndRetry(task, tracker, results); - if (ret < 0) { - return kErrCodeInternalError; + + int CloneCoreImpl::CreateCloneMeta(std::shared_ptr task, + FInfo *fInfo, CloneSegmentMap *segInfos) + { + int ret = CreateOrUpdateCloneMeta(task, fInfo, segInfos); + if (ret < 0) + { + return ret; + } + + task->GetCloneInfo().SetNextStep(CloneStep::kCreateCloneChunk); + + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after CreateCloneMeta error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return ret; + } + return kErrCodeSuccess; } - } while (true); - - if (IsLazy(task) && IsFile(task)) { - task->GetCloneInfo().SetNextStep(CloneStep::kRecoverChunk); - } else { - task->GetCloneInfo().SetNextStep(CloneStep::kCompleteCloneMeta); - } - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after CreateCloneChunk error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::StartAsyncCreateCloneChunk( - std::shared_ptr task, - std::shared_ptr tracker, - std::shared_ptr context) { - CreateCloneChunkClosure* cb = new CreateCloneChunkClosure(tracker, context); - tracker->AddOneTrace(); - LOG(INFO) << "Doing CreateCloneChunk" - << ", location = " << context->location - << ", logicalPoolId = " << context->cidInfo.lpid_ - << ", copysetId = " << context->cidInfo.cpid_ - << ", chunkId = " << context->cidInfo.cid_ - << ", seqNum = " << context->sn << ", csn = " << context->csn - << ", taskid = " << task->GetTaskId(); - int ret = client_->CreateCloneChunk(context->location, context->cidInfo, - context->sn, context->csn, - context->chunkSize, cb); - - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "CreateCloneChunk fail" - << ", ret = " << ret << ", location = " << context->location - << ", logicalPoolId = " << context->cidInfo.lpid_ - << ", copysetId = " << context->cidInfo.cpid_ - << ", chunkId = " << context->cidInfo.cid_ - << ", seqNum = " << context->sn << ", csn = " << context->csn - << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::HandleCreateCloneChunkResultsAndRetry( - std::shared_ptr task, - std::shared_ptr tracker, - const std::list& results) { - int ret = kErrCodeSuccess; - for (auto context : results) { - if (context->retCode == -LIBCURVE_ERROR::EXISTS) { - LOG(INFO) << "CreateCloneChunk chunk exist" + + int CloneCoreImpl::CreateCloneChunk(std::shared_ptr task, + const FInfo &fInfo, + CloneSegmentMap *segInfos) + { + int ret = kErrCodeSuccess; + uint32_t chunkSize = fInfo.chunksize; + uint32_t correctSn = 0; + // When cloning, correctSn is 0, and when restoring, it is the newly + // generated file version + if (IsClone(task)) + { + correctSn = 0; + } + else + { + correctSn = fInfo.seqnum; + } + auto tracker = std::make_shared(); + for (auto &cloneSegmentInfo : *segInfos) + { + for (auto &cloneChunkInfo : cloneSegmentInfo.second) + { + std::string location; + if (IsSnapshot(task)) + { + location = LocationOperator::GenerateS3Location( + cloneChunkInfo.second.location); + } + else + { + location = LocationOperator::GenerateCurveLocation( + task->GetCloneInfo().GetSrc(), + std::stoull(cloneChunkInfo.second.location)); + } + ChunkIDInfo cidInfo = cloneChunkInfo.second.chunkIdInfo; + + auto context = std::make_shared(); + context->location = location; + context->cidInfo = cidInfo; + context->cloneChunkInfo = &cloneChunkInfo.second; + context->sn = cloneChunkInfo.second.seqNum; + context->csn = correctSn; + context->chunkSize = chunkSize; + context->taskid = task->GetTaskId(); + context->startTime = TimeUtility::GetTimeofDaySec(); + context->clientAsyncMethodRetryTimeSec = + clientAsyncMethodRetryTimeSec_; + + ret = StartAsyncCreateCloneChunk(task, tracker, context); + if (ret < 0) + { + return kErrCodeInternalError; + } + + if (tracker->GetTaskNum() >= createCloneChunkConcurrency_) + { + tracker->WaitSome(1); + } + std::list results = + tracker->PopResultContexts(); + ret = HandleCreateCloneChunkResultsAndRetry(task, tracker, results); + if (ret < 0) + { + return kErrCodeInternalError; + } + } + } + // Tasks with insufficient remaining quantity in the end + do + { + tracker->WaitSome(1); + std::list results = + tracker->PopResultContexts(); + if (0 == results.size()) + { + // Completed, no new results + break; + } + ret = HandleCreateCloneChunkResultsAndRetry(task, tracker, results); + if (ret < 0) + { + return kErrCodeInternalError; + } + } while (true); + + if (IsLazy(task) && IsFile(task)) + { + task->GetCloneInfo().SetNextStep(CloneStep::kRecoverChunk); + } + else + { + task->GetCloneInfo().SetNextStep(CloneStep::kCompleteCloneMeta); + } + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after CreateCloneChunk error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + return kErrCodeSuccess; + } + + int CloneCoreImpl::StartAsyncCreateCloneChunk( + std::shared_ptr task, + std::shared_ptr tracker, + std::shared_ptr context) + { + CreateCloneChunkClosure *cb = new CreateCloneChunkClosure(tracker, context); + tracker->AddOneTrace(); + LOG(INFO) << "Doing CreateCloneChunk" << ", location = " << context->location << ", logicalPoolId = " << context->cidInfo.lpid_ << ", copysetId = " << context->cidInfo.cpid_ << ", chunkId = " << context->cidInfo.cid_ - << ", seqNum = " << context->sn - << ", csn = " << context->csn + << ", seqNum = " << context->sn << ", csn = " << context->csn << ", taskid = " << task->GetTaskId(); - context->cloneChunkInfo->needRecover = false; - } else if (context->retCode != LIBCURVE_ERROR::OK) { - uint64_t nowTime = TimeUtility::GetTimeofDaySec(); - if (nowTime - context->startTime < - context->clientAsyncMethodRetryTimeSec) { - // retry - std::this_thread::sleep_for(std::chrono::milliseconds( - clientAsyncMethodRetryIntervalMs_)); - ret = StartAsyncCreateCloneChunk(task, tracker, context); - if (ret < 0) { - return kErrCodeInternalError; - } - } else { - LOG(ERROR) << "CreateCloneChunk tracker GetResult fail" - << ", ret = " << ret + int ret = client_->CreateCloneChunk(context->location, context->cidInfo, + context->sn, context->csn, + context->chunkSize, cb); + + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "CreateCloneChunk fail" + << ", ret = " << ret << ", location = " << context->location + << ", logicalPoolId = " << context->cidInfo.lpid_ + << ", copysetId = " << context->cidInfo.cpid_ + << ", chunkId = " << context->cidInfo.cid_ + << ", seqNum = " << context->sn << ", csn = " << context->csn << ", taskid = " << task->GetTaskId(); + return ret; + } + return kErrCodeSuccess; + } + + int CloneCoreImpl::HandleCreateCloneChunkResultsAndRetry( + std::shared_ptr task, + std::shared_ptr tracker, + const std::list &results) + { + int ret = kErrCodeSuccess; + for (auto context : results) + { + if (context->retCode == -LIBCURVE_ERROR::EXISTS) + { + LOG(INFO) << "CreateCloneChunk chunk exist" + << ", location = " << context->location + << ", logicalPoolId = " << context->cidInfo.lpid_ + << ", copysetId = " << context->cidInfo.cpid_ + << ", chunkId = " << context->cidInfo.cid_ + << ", seqNum = " << context->sn + << ", csn = " << context->csn + << ", taskid = " << task->GetTaskId(); + context->cloneChunkInfo->needRecover = false; + } + else if (context->retCode != LIBCURVE_ERROR::OK) + { + uint64_t nowTime = TimeUtility::GetTimeofDaySec(); + if (nowTime - context->startTime < + context->clientAsyncMethodRetryTimeSec) + { + // retry + std::this_thread::sleep_for(std::chrono::milliseconds( + clientAsyncMethodRetryIntervalMs_)); + ret = StartAsyncCreateCloneChunk(task, tracker, context); + if (ret < 0) + { + return kErrCodeInternalError; + } + } + else + { + LOG(ERROR) << "CreateCloneChunk tracker GetResult fail" + << ", ret = " << ret + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + } + } + return ret; + } + + int CloneCoreImpl::CompleteCloneMeta(std::shared_ptr task, + const FInfo &fInfo, + const CloneSegmentMap &segInfos) + { + (void)fInfo; + (void)segInfos; + std::string origin = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + std::string user = task->GetCloneInfo().GetUser(); + int ret = client_->CompleteCloneMeta(origin, mdsRootUser_); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "CompleteCloneMeta fail" + << ", ret = " << ret << ", filename = " << origin + << ", user = " << user << ", taskid = " << task->GetTaskId(); return kErrCodeInternalError; } + if (IsLazy(task)) + { + task->GetCloneInfo().SetNextStep(CloneStep::kChangeOwner); + } + else + { + task->GetCloneInfo().SetNextStep(CloneStep::kRecoverChunk); + } + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after CompleteCloneMeta error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return ret; + } + return kErrCodeSuccess; } - } - return ret; -} - -int CloneCoreImpl::CompleteCloneMeta(std::shared_ptr task, - const FInfo& fInfo, - const CloneSegmentMap& segInfos) { - (void)fInfo; - (void)segInfos; - std::string origin = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - std::string user = task->GetCloneInfo().GetUser(); - int ret = client_->CompleteCloneMeta(origin, mdsRootUser_); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "CompleteCloneMeta fail" - << ", ret = " << ret << ", filename = " << origin - << ", user = " << user << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - if (IsLazy(task)) { - task->GetCloneInfo().SetNextStep(CloneStep::kChangeOwner); - } else { - task->GetCloneInfo().SetNextStep(CloneStep::kRecoverChunk); - } - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after CompleteCloneMeta error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::RecoverChunk(std::shared_ptr task, - const FInfo& fInfo, - const CloneSegmentMap& segInfos) { - int ret = kErrCodeSuccess; - uint32_t chunkSize = fInfo.chunksize; - - uint32_t totalProgress = - kProgressRecoverChunkEnd - kProgressRecoverChunkBegin; - uint32_t segNum = segInfos.size(); - double progressPerData = static_cast(totalProgress) / segNum; - uint32_t index = 0; - - if (0 == cloneChunkSplitSize_ || chunkSize % cloneChunkSplitSize_ != 0) { - LOG(ERROR) << "chunk is not align to cloneChunkSplitSize" - << ", taskid = " << task->GetTaskId(); - return kErrCodeChunkSizeNotAligned; - } - - auto tracker = std::make_shared(); - uint64_t workingChunkNum = 0; - // To avoid collisions with the same chunk, asynchronous requests for - // different chunks - for (auto& cloneSegmentInfo : segInfos) { - for (auto& cloneChunkInfo : cloneSegmentInfo.second) { - if (!cloneChunkInfo.second.needRecover) { - continue; - } - // When the current number of chunks for concurrent work exceeds the - // required number of concurrent tasks, digest a portion first - while (workingChunkNum >= recoverChunkConcurrency_) { + + int CloneCoreImpl::RecoverChunk(std::shared_ptr task, + const FInfo &fInfo, + const CloneSegmentMap &segInfos) + { + int ret = kErrCodeSuccess; + uint32_t chunkSize = fInfo.chunksize; + + uint32_t totalProgress = + kProgressRecoverChunkEnd - kProgressRecoverChunkBegin; + uint32_t segNum = segInfos.size(); + double progressPerData = static_cast(totalProgress) / segNum; + uint32_t index = 0; + + if (0 == cloneChunkSplitSize_ || chunkSize % cloneChunkSplitSize_ != 0) + { + LOG(ERROR) << "chunk is not align to cloneChunkSplitSize" + << ", taskid = " << task->GetTaskId(); + return kErrCodeChunkSizeNotAligned; + } + + auto tracker = std::make_shared(); + uint64_t workingChunkNum = 0; + // To avoid collisions with the same chunk, asynchronous requests for + // different chunks + for (auto &cloneSegmentInfo : segInfos) + { + for (auto &cloneChunkInfo : cloneSegmentInfo.second) + { + if (!cloneChunkInfo.second.needRecover) + { + continue; + } + // When the current number of chunks for concurrent work exceeds the + // required number of concurrent tasks, digest a portion first + while (workingChunkNum >= recoverChunkConcurrency_) + { + uint64_t completeChunkNum = 0; + ret = ContinueAsyncRecoverChunkPartAndWaitSomeChunkEnd( + task, tracker, &completeChunkNum); + if (ret < 0) + { + return kErrCodeInternalError; + } + workingChunkNum -= completeChunkNum; + } + // Chunk joining a new job + workingChunkNum++; + auto context = std::make_shared(); + context->cidInfo = cloneChunkInfo.second.chunkIdInfo; + context->totalPartNum = chunkSize / cloneChunkSplitSize_; + context->partIndex = 0; + context->partSize = cloneChunkSplitSize_; + context->taskid = task->GetTaskId(); + context->startTime = TimeUtility::GetTimeofDaySec(); + context->clientAsyncMethodRetryTimeSec = + clientAsyncMethodRetryTimeSec_; + + LOG(INFO) << "RecoverChunk start" + << ", logicalPoolId = " << context->cidInfo.lpid_ + << ", copysetId = " << context->cidInfo.cpid_ + << ", chunkId = " << context->cidInfo.cid_ + << ", len = " << context->partSize + << ", taskid = " << task->GetTaskId(); + + ret = StartAsyncRecoverChunkPart(task, tracker, context); + if (ret < 0) + { + return kErrCodeInternalError; + } + } + task->SetProgress(static_cast(kProgressRecoverChunkBegin + + index * progressPerData)); + task->UpdateMetric(); + index++; + } + + while (workingChunkNum > 0) + { uint64_t completeChunkNum = 0; ret = ContinueAsyncRecoverChunkPartAndWaitSomeChunkEnd( task, tracker, &completeChunkNum); - if (ret < 0) { + if (ret < 0) + { return kErrCodeInternalError; } workingChunkNum -= completeChunkNum; } - // Chunk joining a new job - workingChunkNum++; - auto context = std::make_shared(); - context->cidInfo = cloneChunkInfo.second.chunkIdInfo; - context->totalPartNum = chunkSize / cloneChunkSplitSize_; - context->partIndex = 0; - context->partSize = cloneChunkSplitSize_; - context->taskid = task->GetTaskId(); - context->startTime = TimeUtility::GetTimeofDaySec(); - context->clientAsyncMethodRetryTimeSec = - clientAsyncMethodRetryTimeSec_; - - LOG(INFO) << "RecoverChunk start" - << ", logicalPoolId = " << context->cidInfo.lpid_ - << ", copysetId = " << context->cidInfo.cpid_ - << ", chunkId = " << context->cidInfo.cid_ - << ", len = " << context->partSize - << ", taskid = " << task->GetTaskId(); - ret = StartAsyncRecoverChunkPart(task, tracker, context); - if (ret < 0) { + task->GetCloneInfo().SetNextStep(CloneStep::kCompleteCloneFile); + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after RecoverChunk error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); return kErrCodeInternalError; } + return kErrCodeSuccess; } - task->SetProgress(static_cast(kProgressRecoverChunkBegin + - index * progressPerData)); - task->UpdateMetric(); - index++; - } - - while (workingChunkNum > 0) { - uint64_t completeChunkNum = 0; - ret = ContinueAsyncRecoverChunkPartAndWaitSomeChunkEnd( - task, tracker, &completeChunkNum); - if (ret < 0) { - return kErrCodeInternalError; - } - workingChunkNum -= completeChunkNum; - } - - task->GetCloneInfo().SetNextStep(CloneStep::kCompleteCloneFile); - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after RecoverChunk error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::StartAsyncRecoverChunkPart( - std::shared_ptr task, - std::shared_ptr tracker, - std::shared_ptr context) { - RecoverChunkClosure* cb = new RecoverChunkClosure(tracker, context); - tracker->AddOneTrace(); - uint64_t offset = context->partIndex * context->partSize; - LOG_EVERY_SECOND(INFO) << "Doing RecoverChunk" + + int CloneCoreImpl::StartAsyncRecoverChunkPart( + std::shared_ptr task, + std::shared_ptr tracker, + std::shared_ptr context) + { + RecoverChunkClosure *cb = new RecoverChunkClosure(tracker, context); + tracker->AddOneTrace(); + uint64_t offset = context->partIndex * context->partSize; + LOG_EVERY_SECOND(INFO) << "Doing RecoverChunk" + << ", logicalPoolId = " << context->cidInfo.lpid_ + << ", copysetId = " << context->cidInfo.cpid_ + << ", chunkId = " << context->cidInfo.cid_ + << ", offset = " << offset + << ", len = " << context->partSize + << ", taskid = " << task->GetTaskId(); + int ret = + client_->RecoverChunk(context->cidInfo, offset, context->partSize, cb); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "RecoverChunk fail" + << ", ret = " << ret << ", logicalPoolId = " << context->cidInfo.lpid_ << ", copysetId = " << context->cidInfo.cpid_ << ", chunkId = " << context->cidInfo.cid_ - << ", offset = " << offset - << ", len = " << context->partSize + << ", offset = " << offset << ", len = " << context->partSize << ", taskid = " << task->GetTaskId(); - int ret = - client_->RecoverChunk(context->cidInfo, offset, context->partSize, cb); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "RecoverChunk fail" - << ", ret = " << ret - << ", logicalPoolId = " << context->cidInfo.lpid_ - << ", copysetId = " << context->cidInfo.cpid_ - << ", chunkId = " << context->cidInfo.cid_ - << ", offset = " << offset << ", len = " << context->partSize - << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::ContinueAsyncRecoverChunkPartAndWaitSomeChunkEnd( - std::shared_ptr task, - std::shared_ptr tracker, - uint64_t* completeChunkNum) { - *completeChunkNum = 0; - tracker->WaitSome(1); - std::list results = tracker->PopResultContexts(); - for (auto context : results) { - if (context->retCode != LIBCURVE_ERROR::OK) { - uint64_t nowTime = TimeUtility::GetTimeofDaySec(); - if (nowTime - context->startTime < - context->clientAsyncMethodRetryTimeSec) { - // retry - std::this_thread::sleep_for(std::chrono::milliseconds( - clientAsyncMethodRetryIntervalMs_)); - int ret = StartAsyncRecoverChunkPart(task, tracker, context); - if (ret < 0) { - return ret; + return ret; + } + return kErrCodeSuccess; + } + + int CloneCoreImpl::ContinueAsyncRecoverChunkPartAndWaitSomeChunkEnd( + std::shared_ptr task, + std::shared_ptr tracker, + uint64_t *completeChunkNum) + { + *completeChunkNum = 0; + tracker->WaitSome(1); + std::list results = tracker->PopResultContexts(); + for (auto context : results) + { + if (context->retCode != LIBCURVE_ERROR::OK) + { + uint64_t nowTime = TimeUtility::GetTimeofDaySec(); + if (nowTime - context->startTime < + context->clientAsyncMethodRetryTimeSec) + { + // retry + std::this_thread::sleep_for(std::chrono::milliseconds( + clientAsyncMethodRetryIntervalMs_)); + int ret = StartAsyncRecoverChunkPart(task, tracker, context); + if (ret < 0) + { + return ret; + } + } + else + { + LOG(ERROR) << "RecoverChunk tracker GetResult fail" + << ", ret = " << context->retCode + << ", taskid = " << task->GetTaskId(); + return context->retCode; + } } - } else { - LOG(ERROR) << "RecoverChunk tracker GetResult fail" - << ", ret = " << context->retCode - << ", taskid = " << task->GetTaskId(); - return context->retCode; - } - } else { - // Start a new shard, index++, and reset the start time - context->partIndex++; - context->startTime = TimeUtility::GetTimeofDaySec(); - if (context->partIndex < context->totalPartNum) { - int ret = StartAsyncRecoverChunkPart(task, tracker, context); - if (ret < 0) { - return ret; + else + { + // Start a new shard, index++, and reset the start time + context->partIndex++; + context->startTime = TimeUtility::GetTimeofDaySec(); + if (context->partIndex < context->totalPartNum) + { + int ret = StartAsyncRecoverChunkPart(task, tracker, context); + if (ret < 0) + { + return ret; + } + } + else + { + LOG(INFO) << "RecoverChunk Complete" + << ", logicalPoolId = " << context->cidInfo.lpid_ + << ", copysetId = " << context->cidInfo.cpid_ + << ", chunkId = " << context->cidInfo.cid_ + << ", len = " << context->partSize + << ", taskid = " << task->GetTaskId(); + (*completeChunkNum)++; + } } - } else { - LOG(INFO) << "RecoverChunk Complete" - << ", logicalPoolId = " << context->cidInfo.lpid_ - << ", copysetId = " << context->cidInfo.cpid_ - << ", chunkId = " << context->cidInfo.cid_ - << ", len = " << context->partSize - << ", taskid = " << task->GetTaskId(); - (*completeChunkNum)++; } + return kErrCodeSuccess; } - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::ChangeOwner(std::shared_ptr task, - const FInfo& fInfo) { - (void)fInfo; - std::string user = task->GetCloneInfo().GetUser(); - std::string origin = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - - int ret = client_->ChangeOwner(origin, user); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "ChangeOwner fail, ret = " << ret - << ", fileName = " << origin << ", newOwner = " << user - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - - task->GetCloneInfo().SetNextStep(CloneStep::kRenameCloneFile); - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after ChangeOwner error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::RenameCloneFile(std::shared_ptr task, - const FInfo& fInfo) { - std::string user = fInfo.owner; - uint64_t originId = task->GetCloneInfo().GetOriginId(); - uint64_t destinationId = task->GetCloneInfo().GetDestId(); - std::string origin = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - std::string destination = task->GetCloneInfo().GetDest(); - - // Rename first - int ret = client_->RenameCloneFile(mdsRootUser_, originId, destinationId, - origin, destination); - if (-LIBCURVE_ERROR::NOTEXIST == ret) { - // It is possible that it has already been renamed - FInfo destFInfo; - ret = client_->GetFileInfo(destination, mdsRootUser_, &destFInfo); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "RenameCloneFile return NOTEXIST," - << "And get dest fileInfo fail, ret = " << ret - << ", destination filename = " << destination - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; + + int CloneCoreImpl::ChangeOwner(std::shared_ptr task, + const FInfo &fInfo) + { + (void)fInfo; + std::string user = task->GetCloneInfo().GetUser(); + std::string origin = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + + int ret = client_->ChangeOwner(origin, user); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "ChangeOwner fail, ret = " << ret + << ", fileName = " << origin << ", newOwner = " << user + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + + task->GetCloneInfo().SetNextStep(CloneStep::kRenameCloneFile); + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after ChangeOwner error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + return kErrCodeSuccess; } - if (destFInfo.id != originId) { - LOG(ERROR) << "RenameCloneFile return NOTEXIST," - << "And get dest file id not equal, ret = " << ret - << "originId = " << originId - << "destFInfo.id = " << destFInfo.id - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; + + int CloneCoreImpl::RenameCloneFile(std::shared_ptr task, + const FInfo &fInfo) + { + std::string user = fInfo.owner; + uint64_t originId = task->GetCloneInfo().GetOriginId(); + uint64_t destinationId = task->GetCloneInfo().GetDestId(); + std::string origin = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + std::string destination = task->GetCloneInfo().GetDest(); + + // Rename first + int ret = client_->RenameCloneFile(mdsRootUser_, originId, destinationId, + origin, destination); + if (-LIBCURVE_ERROR::NOTEXIST == ret) + { + // It is possible that it has already been renamed + FInfo destFInfo; + ret = client_->GetFileInfo(destination, mdsRootUser_, &destFInfo); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "RenameCloneFile return NOTEXIST," + << "And get dest fileInfo fail, ret = " << ret + << ", destination filename = " << destination + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + if (destFInfo.id != originId) + { + LOG(ERROR) << "RenameCloneFile return NOTEXIST," + << "And get dest file id not equal, ret = " << ret + << "originId = " << originId + << "destFInfo.id = " << destFInfo.id + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + } + else if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "RenameCloneFile fail" + << ", ret = " << ret << ", user = " << user + << ", originId = " << originId << ", origin = " << origin + << ", destination = " << destination + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + + if (IsLazy(task)) + { + if (IsFile(task)) + { + task->GetCloneInfo().SetNextStep(CloneStep::kCreateCloneMeta); + } + else + { + task->GetCloneInfo().SetNextStep(CloneStep::kRecoverChunk); + } + task->GetCloneInfo().SetStatus(CloneStatus::metaInstalled); + } + else + { + task->GetCloneInfo().SetNextStep(CloneStep::kEnd); + } + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after RenameCloneFile error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return ret; + } + return kErrCodeSuccess; + } + + int CloneCoreImpl::CompleteCloneFile(std::shared_ptr task, + const FInfo &fInfo, + const CloneSegmentMap &segInfos) + { + (void)fInfo; + (void)segInfos; + std::string fileName; + if (IsLazy(task)) + { + fileName = task->GetCloneInfo().GetDest(); + } + else + { + fileName = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + } + std::string user = task->GetCloneInfo().GetUser(); + int ret = client_->CompleteCloneFile(fileName, mdsRootUser_); + switch (ret) + { + case LIBCURVE_ERROR::OK: + break; + case -LIBCURVE_ERROR::NOTEXIST: + LOG(ERROR) << "CompleteCloneFile " + << "find dest file not exist, maybe deleted" + << ", ret = " << ret << ", destination = " << fileName + << ", user = " << user + << ", taskid = " << task->GetTaskId(); + return kErrCodeFileNotExist; + default: + LOG(ERROR) << "CompleteCloneFile fail" + << ", ret = " << ret << ", fileName = " << fileName + << ", user = " << user + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + if (IsLazy(task)) + { + task->GetCloneInfo().SetNextStep(CloneStep::kEnd); + } + else + { + task->GetCloneInfo().SetNextStep(CloneStep::kChangeOwner); + } + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo after CompleteCloneFile error." + << " ret = " << ret << ", taskid = " << task->GetTaskId(); + return ret; + } + return kErrCodeSuccess; } - } else if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "RenameCloneFile fail" - << ", ret = " << ret << ", user = " << user - << ", originId = " << originId << ", origin = " << origin - << ", destination = " << destination - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - - if (IsLazy(task)) { - if (IsFile(task)) { - task->GetCloneInfo().SetNextStep(CloneStep::kCreateCloneMeta); - } else { - task->GetCloneInfo().SetNextStep(CloneStep::kRecoverChunk); + + void CloneCoreImpl::HandleLazyCloneStage1Finish( + std::shared_ptr task) + { + LOG(INFO) << "Task Lazy Stage1 Success" + << ", TaskInfo : " << *task; + task->GetClosure()->SetErrCode(kErrCodeSuccess); + task->Finish(); + task->GetClosure()->Run(); + return; } - task->GetCloneInfo().SetStatus(CloneStatus::metaInstalled); - } else { - task->GetCloneInfo().SetNextStep(CloneStep::kEnd); - } - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after RenameCloneFile error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::CompleteCloneFile(std::shared_ptr task, - const FInfo& fInfo, - const CloneSegmentMap& segInfos) { - (void)fInfo; - (void)segInfos; - std::string fileName; - if (IsLazy(task)) { - fileName = task->GetCloneInfo().GetDest(); - } else { - fileName = cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - } - std::string user = task->GetCloneInfo().GetUser(); - int ret = client_->CompleteCloneFile(fileName, mdsRootUser_); - switch (ret) { - case LIBCURVE_ERROR::OK: - break; - case -LIBCURVE_ERROR::NOTEXIST: - LOG(ERROR) << "CompleteCloneFile " - << "find dest file not exist, maybe deleted" - << ", ret = " << ret << ", destination = " << fileName - << ", user = " << user - << ", taskid = " << task->GetTaskId(); - return kErrCodeFileNotExist; - default: - LOG(ERROR) << "CompleteCloneFile fail" - << ", ret = " << ret << ", fileName = " << fileName - << ", user = " << user - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - if (IsLazy(task)) { - task->GetCloneInfo().SetNextStep(CloneStep::kEnd); - } else { - task->GetCloneInfo().SetNextStep(CloneStep::kChangeOwner); - } - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo after CompleteCloneFile error." - << " ret = " << ret << ", taskid = " << task->GetTaskId(); - return ret; - } - return kErrCodeSuccess; -} - -void CloneCoreImpl::HandleLazyCloneStage1Finish( - std::shared_ptr task) { - LOG(INFO) << "Task Lazy Stage1 Success" - << ", TaskInfo : " << *task; - task->GetClosure()->SetErrCode(kErrCodeSuccess); - task->Finish(); - task->GetClosure()->Run(); - return; -} - -void CloneCoreImpl::HandleCloneSuccess(std::shared_ptr task) { - int ret = kErrCodeSuccess; - if (IsSnapshot(task)) { - snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); - } else { - std::string source = task->GetCloneInfo().GetSrc(); - cloneRef_->DecrementRef(source); - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - if (cloneRef_->GetRef(source) == 0) { - int ret = client_->SetCloneFileStatus(source, FileStatus::Created, - mdsRootUser_); - if (ret < 0) { - task->GetCloneInfo().SetStatus(CloneStatus::error); - int ret2 = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret2 < 0) { - LOG(ERROR) << "UpdateCloneInfo Task error Fail!" - << " ret = " << ret2 - << ", uuid = " << task->GetTaskId(); + + void CloneCoreImpl::HandleCloneSuccess(std::shared_ptr task) + { + int ret = kErrCodeSuccess; + if (IsSnapshot(task)) + { + snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); + } + else + { + std::string source = task->GetCloneInfo().GetSrc(); + cloneRef_->DecrementRef(source); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) + { + int ret = client_->SetCloneFileStatus(source, FileStatus::Created, + mdsRootUser_); + if (ret < 0) + { + task->GetCloneInfo().SetStatus(CloneStatus::error); + int ret2 = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret2 < 0) + { + LOG(ERROR) << "UpdateCloneInfo Task error Fail!" + << " ret = " << ret2 + << ", uuid = " << task->GetTaskId(); + } + LOG(ERROR) << "Task Fail cause by SetCloneFileStatus fail" + << ", ret = " << ret << ", TaskInfo : " << *task; + task->Finish(); + return; + } } - LOG(ERROR) << "Task Fail cause by SetCloneFileStatus fail" - << ", ret = " << ret << ", TaskInfo : " << *task; - task->Finish(); + } + task->GetCloneInfo().SetStatus(CloneStatus::done); + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo Task Success Fail!" + << " ret = " << ret << ", uuid = " << task->GetTaskId(); + } + task->SetProgress(kProgressCloneComplete); + + LOG(INFO) << "Task Success" + << ", TaskInfo : " << *task; + task->Finish(); + return; + } + + void CloneCoreImpl::HandleCloneError(std::shared_ptr task, + int retCode) + { + int ret = kErrCodeSuccess; + if (NeedRetry(task, retCode)) + { + HandleCloneToRetry(task); return; } + + if (IsLazy(task)) + { + task->GetClosure()->SetErrCode(retCode); + } + if (IsSnapshot(task)) + { + snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); + } + else + { + std::string source = task->GetCloneInfo().GetSrc(); + cloneRef_->DecrementRef(source); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) + { + ret = client_->SetCloneFileStatus(source, FileStatus::Created, + mdsRootUser_); + if (ret < 0) + { + LOG(ERROR) << "SetCloneFileStatus fail, ret = " << ret + << ", taskid = " << task->GetTaskId(); + } + } + } + task->GetCloneInfo().SetStatus(CloneStatus::error); + ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo Task error Fail!" + << " ret = " << ret << ", uuid = " << task->GetTaskId(); + } + LOG(ERROR) << "Task Fail" + << ", TaskInfo : " << *task; + task->Finish(); + return; } - } - task->GetCloneInfo().SetStatus(CloneStatus::done); - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo Task Success Fail!" - << " ret = " << ret << ", uuid = " << task->GetTaskId(); - } - task->SetProgress(kProgressCloneComplete); - - LOG(INFO) << "Task Success" - << ", TaskInfo : " << *task; - task->Finish(); - return; -} - -void CloneCoreImpl::HandleCloneError(std::shared_ptr task, - int retCode) { - int ret = kErrCodeSuccess; - if (NeedRetry(task, retCode)) { - HandleCloneToRetry(task); - return; - } - - if (IsLazy(task)) { - task->GetClosure()->SetErrCode(retCode); - } - if (IsSnapshot(task)) { - snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); - } else { - std::string source = task->GetCloneInfo().GetSrc(); - cloneRef_->DecrementRef(source); - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - if (cloneRef_->GetRef(source) == 0) { - ret = client_->SetCloneFileStatus(source, FileStatus::Created, - mdsRootUser_); - if (ret < 0) { - LOG(ERROR) << "SetCloneFileStatus fail, ret = " << ret - << ", taskid = " << task->GetTaskId(); + + void CloneCoreImpl::HandleCloneToRetry(std::shared_ptr task) + { + task->GetCloneInfo().SetStatus(CloneStatus::retrying); + int ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo Task retrying Fail!" + << " ret = " << ret << ", uuid = " << task->GetTaskId(); + } + LOG(WARNING) << "Task Fail, Retrying" + << ", TaskInfo : " << *task; + task->Finish(); + return; + } + + void CloneCoreImpl::HandleCleanSuccess(std::shared_ptr task) + { + TaskIdType taskId = task->GetCloneInfo().GetTaskId(); + int ret = metaStore_->DeleteCloneInfo(taskId); + if (ret < 0) + { + LOG(ERROR) << "DeleteCloneInfo failed" + << ", ret = " << ret << ", taskId = " << taskId; + } + else + { + LOG(INFO) << "Clean Task Success" + << ", TaskInfo : " << *task; } + task->SetProgress(kProgressCloneComplete); + task->GetCloneInfo().SetStatus(CloneStatus::done); + + task->Finish(); + return; } - } - task->GetCloneInfo().SetStatus(CloneStatus::error); - ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo Task error Fail!" - << " ret = " << ret << ", uuid = " << task->GetTaskId(); - } - LOG(ERROR) << "Task Fail" - << ", TaskInfo : " << *task; - task->Finish(); - return; -} - -void CloneCoreImpl::HandleCloneToRetry(std::shared_ptr task) { - task->GetCloneInfo().SetStatus(CloneStatus::retrying); - int ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo Task retrying Fail!" - << " ret = " << ret << ", uuid = " << task->GetTaskId(); - } - LOG(WARNING) << "Task Fail, Retrying" - << ", TaskInfo : " << *task; - task->Finish(); - return; -} - -void CloneCoreImpl::HandleCleanSuccess(std::shared_ptr task) { - TaskIdType taskId = task->GetCloneInfo().GetTaskId(); - int ret = metaStore_->DeleteCloneInfo(taskId); - if (ret < 0) { - LOG(ERROR) << "DeleteCloneInfo failed" - << ", ret = " << ret << ", taskId = " << taskId; - } else { - LOG(INFO) << "Clean Task Success" - << ", TaskInfo : " << *task; - } - task->SetProgress(kProgressCloneComplete); - task->GetCloneInfo().SetStatus(CloneStatus::done); - - task->Finish(); - return; -} - -void CloneCoreImpl::HandleCleanError(std::shared_ptr task) { - task->GetCloneInfo().SetStatus(CloneStatus::error); - int ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo Task error Fail!" - << " ret = " << ret << ", uuid = " << task->GetTaskId(); - } - LOG(ERROR) << "Clean Task Fail" - << ", TaskInfo : " << *task; - task->Finish(); - return; -} - -int CloneCoreImpl::GetCloneInfoList(std::vector* taskList) { - metaStore_->GetCloneInfoList(taskList); - return kErrCodeSuccess; -} - -int CloneCoreImpl::GetCloneInfo(TaskIdType taskId, CloneInfo* cloneInfo) { - return metaStore_->GetCloneInfo(taskId, cloneInfo); -} - -int CloneCoreImpl::GetCloneInfoByFileName(const std::string& fileName, - std::vector* list) { - return metaStore_->GetCloneInfoByFileName(fileName, list); -} - -inline bool CloneCoreImpl::IsLazy(std::shared_ptr task) { - return task->GetCloneInfo().GetIsLazy(); -} - -inline bool CloneCoreImpl::IsSnapshot(std::shared_ptr task) { - return CloneFileType::kSnapshot == task->GetCloneInfo().GetFileType(); -} - -inline bool CloneCoreImpl::IsFile(std::shared_ptr task) { - return CloneFileType::kFile == task->GetCloneInfo().GetFileType(); -} - -inline bool CloneCoreImpl::IsRecover(std::shared_ptr task) { - return CloneTaskType::kRecover == task->GetCloneInfo().GetTaskType(); -} - -inline bool CloneCoreImpl::IsClone(std::shared_ptr task) { - return CloneTaskType::kClone == task->GetCloneInfo().GetTaskType(); -} - -bool CloneCoreImpl::NeedUpdateCloneMeta(std::shared_ptr task) { - bool ret = true; - CloneStep step = task->GetCloneInfo().GetNextStep(); - if (CloneStep::kCreateCloneFile == step || - CloneStep::kCreateCloneMeta == step || CloneStep::kEnd == step) { - ret = false; - } - return ret; -} - -bool CloneCoreImpl::NeedRetry(std::shared_ptr task, - int retCode) { - if (IsLazy(task)) { - CloneStep step = task->GetCloneInfo().GetNextStep(); - if (CloneStep::kRecoverChunk == step || - CloneStep::kCompleteCloneFile == step || CloneStep::kEnd == step) { - // In scenarios where the file does not exist, there is no need to - // retry as it may have been deleted - if (retCode != kErrCodeFileNotExist) { - return true; + + void CloneCoreImpl::HandleCleanError(std::shared_ptr task) + { + task->GetCloneInfo().SetStatus(CloneStatus::error); + int ret = metaStore_->UpdateCloneInfo(task->GetCloneInfo()); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo Task error Fail!" + << " ret = " << ret << ", uuid = " << task->GetTaskId(); } + LOG(ERROR) << "Clean Task Fail" + << ", TaskInfo : " << *task; + task->Finish(); + return; } - } - return false; -} - -int CloneCoreImpl::CreateOrUpdateCloneMeta(std::shared_ptr task, - FInfo* fInfo, - CloneSegmentMap* segInfos) { - std::string newFileName = - cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - std::string user = fInfo->owner; - FInfo fInfoOut; - int ret = client_->GetFileInfo(newFileName, mdsRootUser_, &fInfoOut); - if (LIBCURVE_ERROR::OK == ret) { - // nothing - } else if (-LIBCURVE_ERROR::NOTEXIST == ret) { - // Perhaps it has already been renamed - newFileName = task->GetCloneInfo().GetDest(); - ret = client_->GetFileInfo(newFileName, mdsRootUser_, &fInfoOut); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "File is missing, " - << "when CreateOrUpdateCloneMeta, " - << "GetFileInfo fail, ret = " << ret - << ", filename = " << newFileName - << ", taskid = " << task->GetTaskId(); - return kErrCodeFileNotExist; + + int CloneCoreImpl::GetCloneInfoList(std::vector *taskList) + { + metaStore_->GetCloneInfoList(taskList); + return kErrCodeSuccess; } - // If it has already been renamed, then the id should be consistent - uint64_t originId = task->GetCloneInfo().GetOriginId(); - if (fInfoOut.id != originId) { - LOG(ERROR) << "File is missing, fileId not equal, " - << "when CreateOrUpdateCloneMeta" - << ", fileId = " << fInfoOut.id - << ", originId = " << originId - << ", filename = " << newFileName - << ", taskid = " << task->GetTaskId(); - return kErrCodeFileNotExist; + + int CloneCoreImpl::GetCloneInfo(TaskIdType taskId, CloneInfo *cloneInfo) + { + return metaStore_->GetCloneInfo(taskId, cloneInfo); } - } else { - LOG(ERROR) << "GetFileInfo fail" - << ", ret = " << ret << ", filename = " << newFileName - << ", user = " << user << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; - } - // Update fInfo - *fInfo = fInfoOut; - // GetOrAllocateSegment depends on fullPathName and needs to be updated here - fInfo->fullPathName = newFileName; - - uint32_t segmentSize = fInfo->segmentsize; - for (auto& segInfo : *segInfos) { - SegmentInfo segInfoOut; - uint64_t offset = segInfo.first * segmentSize; - ret = client_->GetOrAllocateSegmentInfo(true, offset, fInfo, - mdsRootUser_, &segInfoOut); - if (ret != LIBCURVE_ERROR::OK) { - LOG(ERROR) << "GetOrAllocateSegmentInfo fail" - << ", newFileName = " << newFileName - << ", user = " << user << ", offset = " << offset - << ", allocateIfNotExist = " - << "true" - << ", taskid = " << task->GetTaskId(); - return kErrCodeInternalError; + + int CloneCoreImpl::GetCloneInfoByFileName(const std::string &fileName, + std::vector *list) + { + return metaStore_->GetCloneInfoByFileName(fileName, list); } - for (auto& cloneChunkInfo : segInfo.second) { - if (cloneChunkInfo.first > segInfoOut.chunkvec.size()) { - LOG(ERROR) << "can not find chunkIndexInSeg = " - << cloneChunkInfo.first - << ", segmentIndex = " << segInfo.first - << ", logicalPoolId = " - << cloneChunkInfo.second.chunkIdInfo.lpid_ - << ", copysetId = " - << cloneChunkInfo.second.chunkIdInfo.cpid_ - << ", chunkId = " - << cloneChunkInfo.second.chunkIdInfo.cid_ - << ", taskid = " << task->GetTaskId(); + inline bool CloneCoreImpl::IsLazy(std::shared_ptr task) + { + return task->GetCloneInfo().GetIsLazy(); + } + + inline bool CloneCoreImpl::IsSnapshot(std::shared_ptr task) + { + return CloneFileType::kSnapshot == task->GetCloneInfo().GetFileType(); + } + + inline bool CloneCoreImpl::IsFile(std::shared_ptr task) + { + return CloneFileType::kFile == task->GetCloneInfo().GetFileType(); + } + + inline bool CloneCoreImpl::IsRecover(std::shared_ptr task) + { + return CloneTaskType::kRecover == task->GetCloneInfo().GetTaskType(); + } + + inline bool CloneCoreImpl::IsClone(std::shared_ptr task) + { + return CloneTaskType::kClone == task->GetCloneInfo().GetTaskType(); + } + + bool CloneCoreImpl::NeedUpdateCloneMeta(std::shared_ptr task) + { + bool ret = true; + CloneStep step = task->GetCloneInfo().GetNextStep(); + if (CloneStep::kCreateCloneFile == step || + CloneStep::kCreateCloneMeta == step || CloneStep::kEnd == step) + { + ret = false; + } + return ret; + } + + bool CloneCoreImpl::NeedRetry(std::shared_ptr task, + int retCode) + { + if (IsLazy(task)) + { + CloneStep step = task->GetCloneInfo().GetNextStep(); + if (CloneStep::kRecoverChunk == step || + CloneStep::kCompleteCloneFile == step || CloneStep::kEnd == step) + { + // In scenarios where the file does not exist, there is no need to + // retry as it may have been deleted + if (retCode != kErrCodeFileNotExist) + { + return true; + } + } + } + return false; + } + + int CloneCoreImpl::CreateOrUpdateCloneMeta(std::shared_ptr task, + FInfo *fInfo, + CloneSegmentMap *segInfos) + { + std::string newFileName = + cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + std::string user = fInfo->owner; + FInfo fInfoOut; + int ret = client_->GetFileInfo(newFileName, mdsRootUser_, &fInfoOut); + if (LIBCURVE_ERROR::OK == ret) + { + // nothing + } + else if (-LIBCURVE_ERROR::NOTEXIST == ret) + { + // Perhaps it has already been renamed + newFileName = task->GetCloneInfo().GetDest(); + ret = client_->GetFileInfo(newFileName, mdsRootUser_, &fInfoOut); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "File is missing, " + << "when CreateOrUpdateCloneMeta, " + << "GetFileInfo fail, ret = " << ret + << ", filename = " << newFileName + << ", taskid = " << task->GetTaskId(); + return kErrCodeFileNotExist; + } + // If it has already been renamed, then the id should be consistent + uint64_t originId = task->GetCloneInfo().GetOriginId(); + if (fInfoOut.id != originId) + { + LOG(ERROR) << "File is missing, fileId not equal, " + << "when CreateOrUpdateCloneMeta" + << ", fileId = " << fInfoOut.id + << ", originId = " << originId + << ", filename = " << newFileName + << ", taskid = " << task->GetTaskId(); + return kErrCodeFileNotExist; + } + } + else + { + LOG(ERROR) << "GetFileInfo fail" + << ", ret = " << ret << ", filename = " << newFileName + << ", user = " << user << ", taskid = " << task->GetTaskId(); return kErrCodeInternalError; } - cloneChunkInfo.second.chunkIdInfo = - segInfoOut.chunkvec[cloneChunkInfo.first]; + // Update fInfo + *fInfo = fInfoOut; + // GetOrAllocateSegment depends on fullPathName and needs to be updated here + fInfo->fullPathName = newFileName; + + uint32_t segmentSize = fInfo->segmentsize; + for (auto &segInfo : *segInfos) + { + SegmentInfo segInfoOut; + uint64_t offset = segInfo.first * segmentSize; + ret = client_->GetOrAllocateSegmentInfo(true, offset, fInfo, + mdsRootUser_, &segInfoOut); + if (ret != LIBCURVE_ERROR::OK) + { + LOG(ERROR) << "GetOrAllocateSegmentInfo fail" + << ", newFileName = " << newFileName + << ", user = " << user << ", offset = " << offset + << ", allocateIfNotExist = " + << "true" + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + + for (auto &cloneChunkInfo : segInfo.second) + { + if (cloneChunkInfo.first > segInfoOut.chunkvec.size()) + { + LOG(ERROR) << "can not find chunkIndexInSeg = " + << cloneChunkInfo.first + << ", segmentIndex = " << segInfo.first + << ", logicalPoolId = " + << cloneChunkInfo.second.chunkIdInfo.lpid_ + << ", copysetId = " + << cloneChunkInfo.second.chunkIdInfo.cpid_ + << ", chunkId = " + << cloneChunkInfo.second.chunkIdInfo.cid_ + << ", taskid = " << task->GetTaskId(); + return kErrCodeInternalError; + } + cloneChunkInfo.second.chunkIdInfo = + segInfoOut.chunkvec[cloneChunkInfo.first]; + } + } + return kErrCodeSuccess; + } + + int CloneCoreImpl::CleanCloneOrRecoverTaskPre(const std::string &user, + const TaskIdType &taskId, + CloneInfo *cloneInfo) + { + int ret = metaStore_->GetCloneInfo(taskId, cloneInfo); + if (ret < 0) + { + // Directly returns success when it does not exist, making the interface + // idempotent + return kErrCodeSuccess; + } + if (cloneInfo->GetUser() != user) + { + LOG(ERROR) << "CleanCloneOrRecoverTaskPre by Invalid user"; + return kErrCodeInvalidUser; + } + switch (cloneInfo->GetStatus()) + { + case CloneStatus::done: + cloneInfo->SetStatus(CloneStatus::cleaning); + break; + case CloneStatus::error: + cloneInfo->SetStatus(CloneStatus::errorCleaning); + break; + case CloneStatus::cleaning: + case CloneStatus::errorCleaning: + return kErrCodeTaskExist; + break; + default: + LOG(ERROR) << "Can not clean clone/recover task unfinished."; + return kErrCodeCannotCleanCloneUnfinished; + break; + } + + ret = metaStore_->UpdateCloneInfo(*cloneInfo); + if (ret < 0) + { + LOG(ERROR) << "UpdateCloneInfo fail" + << ", ret = " << ret << ", taskId = " << taskId; + return ret; + } + return kErrCodeSuccess; } - } - return kErrCodeSuccess; -} - -int CloneCoreImpl::CleanCloneOrRecoverTaskPre(const std::string& user, - const TaskIdType& taskId, - CloneInfo* cloneInfo) { - int ret = metaStore_->GetCloneInfo(taskId, cloneInfo); - if (ret < 0) { - // Directly returns success when it does not exist, making the interface - // idempotent - return kErrCodeSuccess; - } - if (cloneInfo->GetUser() != user) { - LOG(ERROR) << "CleanCloneOrRecoverTaskPre by Invalid user"; - return kErrCodeInvalidUser; - } - switch (cloneInfo->GetStatus()) { - case CloneStatus::done: - cloneInfo->SetStatus(CloneStatus::cleaning); - break; - case CloneStatus::error: - cloneInfo->SetStatus(CloneStatus::errorCleaning); - break; - case CloneStatus::cleaning: - case CloneStatus::errorCleaning: - return kErrCodeTaskExist; - break; - default: - LOG(ERROR) << "Can not clean clone/recover task unfinished."; - return kErrCodeCannotCleanCloneUnfinished; - break; - } - - ret = metaStore_->UpdateCloneInfo(*cloneInfo); - if (ret < 0) { - LOG(ERROR) << "UpdateCloneInfo fail" - << ", ret = " << ret << ", taskId = " << taskId; - return ret; - } - return kErrCodeSuccess; -} - -void CloneCoreImpl::HandleCleanCloneOrRecoverTask( - std::shared_ptr task) { - // Only the wrong clone/recover task cleans up temporary files - if (CloneStatus::errorCleaning == task->GetCloneInfo().GetStatus()) { - // In the event of an error, the mirror being cloned flag may not be - // cleared - if (IsFile(task)) { - // Resend - std::string source = task->GetCloneInfo().GetSrc(); - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - if (cloneRef_->GetRef(source) == 0) { - int ret = client_->SetCloneFileStatus( - source, FileStatus::Created, mdsRootUser_); - if (ret != LIBCURVE_ERROR::OK && - ret != -LIBCURVE_ERROR::NOTEXIST) { - LOG(ERROR) << "SetCloneFileStatus fail, ret = " << ret + + void CloneCoreImpl::HandleCleanCloneOrRecoverTask( + std::shared_ptr task) + { + // Only the wrong clone/recover task cleans up temporary files + if (CloneStatus::errorCleaning == task->GetCloneInfo().GetStatus()) + { + // In the event of an error, the mirror being cloned flag may not be + // cleared + if (IsFile(task)) + { + // Resend + std::string source = task->GetCloneInfo().GetSrc(); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) + { + int ret = client_->SetCloneFileStatus( + source, FileStatus::Created, mdsRootUser_); + if (ret != LIBCURVE_ERROR::OK && + ret != -LIBCURVE_ERROR::NOTEXIST) + { + LOG(ERROR) << "SetCloneFileStatus fail, ret = " << ret + << ", taskid = " << task->GetTaskId(); + HandleCleanError(task); + return; + } + } + } + std::string tempFileName = + cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); + uint64_t fileId = task->GetCloneInfo().GetOriginId(); + std::string user = task->GetCloneInfo().GetUser(); + int ret = client_->DeleteFile(tempFileName, mdsRootUser_, fileId); + if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::NOTEXIST) + { + LOG(ERROR) << "DeleteFile failed" + << ", ret = " << ret << ", fileName = " << tempFileName + << ", user = " << user << ", fileId = " << fileId << ", taskid = " << task->GetTaskId(); HandleCleanError(task); return; } } - } - std::string tempFileName = - cloneTempDir_ + "/" + task->GetCloneInfo().GetTaskId(); - uint64_t fileId = task->GetCloneInfo().GetOriginId(); - std::string user = task->GetCloneInfo().GetUser(); - int ret = client_->DeleteFile(tempFileName, mdsRootUser_, fileId); - if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::NOTEXIST) { - LOG(ERROR) << "DeleteFile failed" - << ", ret = " << ret << ", fileName = " << tempFileName - << ", user = " << user << ", fileId = " << fileId - << ", taskid = " << task->GetTaskId(); - HandleCleanError(task); + HandleCleanSuccess(task); return; } - } - HandleCleanSuccess(task); - return; -} - -int CloneCoreImpl::HandleRemoveCloneOrRecoverTask( - std::shared_ptr task) { - TaskIdType taskId = task->GetCloneInfo().GetTaskId(); - int ret = metaStore_->DeleteCloneInfo(taskId); - if (ret < 0) { - LOG(ERROR) << "DeleteCloneInfo failed" - << ", ret = " << ret << ", taskId = " << taskId; - return kErrCodeInternalError; - } - - if (IsSnapshot(task)) { - snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); - } else { - std::string source = task->GetCloneInfo().GetSrc(); - cloneRef_->DecrementRef(source); - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - if (cloneRef_->GetRef(source) == 0) { - int ret = client_->SetCloneFileStatus(source, FileStatus::Created, - mdsRootUser_); - if (ret < 0) { - LOG(ERROR) << "Task Fail cause by SetCloneFileStatus fail" - << ", ret = " << ret << ", TaskInfo : " << *task; + + int CloneCoreImpl::HandleRemoveCloneOrRecoverTask( + std::shared_ptr task) + { + TaskIdType taskId = task->GetCloneInfo().GetTaskId(); + int ret = metaStore_->DeleteCloneInfo(taskId); + if (ret < 0) + { + LOG(ERROR) << "DeleteCloneInfo failed" + << ", ret = " << ret << ", taskId = " << taskId; return kErrCodeInternalError; } + + if (IsSnapshot(task)) + { + snapshotRef_->DecrementSnapshotRef(task->GetCloneInfo().GetSrc()); + } + else + { + std::string source = task->GetCloneInfo().GetSrc(); + cloneRef_->DecrementRef(source); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) + { + int ret = client_->SetCloneFileStatus(source, FileStatus::Created, + mdsRootUser_); + if (ret < 0) + { + LOG(ERROR) << "Task Fail cause by SetCloneFileStatus fail" + << ", ret = " << ret << ", TaskInfo : " << *task; + return kErrCodeInternalError; + } + } + } + + return kErrCodeSuccess; } - } - - return kErrCodeSuccess; -} - -int CloneCoreImpl::CheckFileExists(const std::string& filename, - uint64_t inodeId) { - FInfo destFInfo; - int ret = client_->GetFileInfo(filename, mdsRootUser_, &destFInfo); - if (ret == LIBCURVE_ERROR::OK) { - if (destFInfo.id == inodeId) { - return kErrCodeFileExist; - } else { - return kErrCodeFileNotExist; + + int CloneCoreImpl::CheckFileExists(const std::string &filename, + uint64_t inodeId) + { + FInfo destFInfo; + int ret = client_->GetFileInfo(filename, mdsRootUser_, &destFInfo); + if (ret == LIBCURVE_ERROR::OK) + { + if (destFInfo.id == inodeId) + { + return kErrCodeFileExist; + } + else + { + return kErrCodeFileNotExist; + } + } + + if (ret == -LIBCURVE_ERROR::NOTEXIST) + { + return kErrCodeFileNotExist; + } + + return kErrCodeInternalError; } - } - - if (ret == -LIBCURVE_ERROR::NOTEXIST) { - return kErrCodeFileNotExist; - } - - return kErrCodeInternalError; -} - -// When adding or subtracting reference counts, the interface will lock the -// reference count map; When adding a reference count and reducing the reference -// count to 0, an additional lock needs to be added to the modified record. -int CloneCoreImpl::HandleDeleteCloneInfo(const CloneInfo& cloneInfo) { - // First, reduce the reference count. If you are cloning from a mirror and - // the reference count is reduced to 0, you need to modify the status of the - // source mirror to 'created' - std::string source = cloneInfo.GetSrc(); - if (cloneInfo.GetFileType() == CloneFileType::kSnapshot) { - snapshotRef_->DecrementSnapshotRef(source); - } else { - cloneRef_->DecrementRef(source); - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - if (cloneRef_->GetRef(source) == 0) { - int ret = client_->SetCloneFileStatus(source, FileStatus::Created, - mdsRootUser_); - if (ret == -LIBCURVE_ERROR::NOTEXIST) { - LOG(WARNING) << "SetCloneFileStatus, file not exist, filename: " - << source; - } else if (ret != LIBCURVE_ERROR::OK) { - cloneRef_->IncrementRef(source); - LOG(ERROR) << "SetCloneFileStatus fail" - << ", ret = " << ret - << ", cloneInfo : " << cloneInfo; + + // When adding or subtracting reference counts, the interface will lock the + // reference count map; When adding a reference count and reducing the reference + // count to 0, an additional lock needs to be added to the modified record. + int CloneCoreImpl::HandleDeleteCloneInfo(const CloneInfo &cloneInfo) + { + // First, reduce the reference count. If you are cloning from a mirror and + // the reference count is reduced to 0, you need to modify the status of the + // source mirror to 'created' + std::string source = cloneInfo.GetSrc(); + if (cloneInfo.GetFileType() == CloneFileType::kSnapshot) + { + snapshotRef_->DecrementSnapshotRef(source); + } + else + { + cloneRef_->DecrementRef(source); + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + if (cloneRef_->GetRef(source) == 0) + { + int ret = client_->SetCloneFileStatus(source, FileStatus::Created, + mdsRootUser_); + if (ret == -LIBCURVE_ERROR::NOTEXIST) + { + LOG(WARNING) << "SetCloneFileStatus, file not exist, filename: " + << source; + } + else if (ret != LIBCURVE_ERROR::OK) + { + cloneRef_->IncrementRef(source); + LOG(ERROR) << "SetCloneFileStatus fail" + << ", ret = " << ret + << ", cloneInfo : " << cloneInfo; + return kErrCodeInternalError; + } + } + } + + // Delete this record. If the deletion fails, add back the previously + // subtracted reference count + int ret = metaStore_->DeleteCloneInfo(cloneInfo.GetTaskId()); + if (ret != 0) + { + if (cloneInfo.GetFileType() == CloneFileType::kSnapshot) + { + NameLockGuard lockSnapGuard(snapshotRef_->GetSnapshotLock(), + source); + snapshotRef_->IncrementSnapshotRef(source); + } + else + { + NameLockGuard lockGuard(cloneRef_->GetLock(), source); + cloneRef_->IncrementRef(source); + } + LOG(ERROR) << "DeleteCloneInfo failed" + << ", ret = " << ret << ", CloneInfo = " << cloneInfo; return kErrCodeInternalError; } - } - } - - // Delete this record. If the deletion fails, add back the previously - // subtracted reference count - int ret = metaStore_->DeleteCloneInfo(cloneInfo.GetTaskId()); - if (ret != 0) { - if (cloneInfo.GetFileType() == CloneFileType::kSnapshot) { - NameLockGuard lockSnapGuard(snapshotRef_->GetSnapshotLock(), - source); - snapshotRef_->IncrementSnapshotRef(source); - } else { - NameLockGuard lockGuard(cloneRef_->GetLock(), source); - cloneRef_->IncrementRef(source); - } - LOG(ERROR) << "DeleteCloneInfo failed" - << ", ret = " << ret << ", CloneInfo = " << cloneInfo; - return kErrCodeInternalError; - } - LOG(INFO) << "HandleDeleteCloneInfo success" - << ", cloneInfo = " << cloneInfo; + LOG(INFO) << "HandleDeleteCloneInfo success" + << ", cloneInfo = " << cloneInfo; - return kErrCodeSuccess; -} + return kErrCodeSuccess; + } -} // namespace snapshotcloneserver -} // namespace curve + } // namespace snapshotcloneserver +} // namespace curve diff --git a/test/chunkserver/copyset_node_test.cpp b/test/chunkserver/copyset_node_test.cpp index c81a4b9358..4a30fae926 100644 --- a/test/chunkserver/copyset_node_test.cpp +++ b/test/chunkserver/copyset_node_test.cpp @@ -42,1059 +42,1074 @@ #include "test/chunkserver/mock_node.h" #include "test/fs/mock_local_filesystem.h" -namespace curve { -namespace chunkserver { - -using ::testing::_; -using ::testing::AnyNumber; -using ::testing::AtLeast; -using ::testing::DoAll; -using ::testing::InSequence; -using ::testing::Invoke; -using ::testing::Matcher; -using ::testing::Return; -using ::testing::SaveArgPointee; -using ::testing::SetArgPointee; -using ::testing::SetArgReferee; - -using curve::chunkserver::concurrent::ConcurrentApplyOption; -using curve::fs::FileSystemType; -using curve::fs::MockLocalFileSystem; - -const char copysetUri[] = "local://./copyset_node_test"; -const int port = 9044; - -class FakeSnapshotReader : public braft::SnapshotReader { - public: - std::string get_path() { - /*Returns a non-existent path*/ - return std::string("/1002093939/temp/238408034"); - } - void list_files(std::vector* files) { return; } - int load_meta(braft::SnapshotMeta* meta) { return 1; } - std::string generate_uri_for_copy() { return std::string(""); } -}; - -class FakeSnapshotWriter : public braft::SnapshotWriter { - public: - std::string get_path() { - /*Returns a non-existent path*/ - return std::string("."); - } - void list_files(std::vector* files) { return; } - virtual int save_meta(const braft::SnapshotMeta& meta) { return 0; } - - virtual int add_file(const std::string& filename) { return 0; } - - virtual int add_file(const std::string& filename, - const ::google::protobuf::Message* file_meta) { - return 0; - } - - virtual int remove_file(const std::string& filename) { return 0; } -}; - -class FakeClosure : public braft::Closure { - public: - void Run() { std::cerr << "FakeClosure run" << std::endl; } -}; - -class CopysetNodeTest : public ::testing::Test { - protected: - void SetUp() { - defaultOptions_.ip = "127.0.0.1"; - defaultOptions_.port = port; - defaultOptions_.electionTimeoutMs = 1000; - defaultOptions_.snapshotIntervalS = 30; - defaultOptions_.catchupMargin = 50; - defaultOptions_.chunkDataUri = copysetUri; - defaultOptions_.chunkSnapshotUri = copysetUri; - defaultOptions_.logUri = copysetUri; - defaultOptions_.raftMetaUri = copysetUri; - defaultOptions_.raftSnapshotUri = copysetUri; - defaultOptions_.loadConcurrency = 5; - defaultOptions_.syncConcurrency = 20; - defaultOptions_.checkRetryTimes = 3; - defaultOptions_.finishLoadMargin = 1000; - - defaultOptions_.concurrentapply = &concurrentModule_; - ConcurrentApplyOption opt{2, 1, 2, 1}; - defaultOptions_.concurrentapply->Init(opt); - std::shared_ptr fs = - LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); - ASSERT_TRUE(nullptr != fs); - defaultOptions_.localFileSystem = fs; - defaultOptions_.chunkFilePool = std::make_shared(fs); - defaultOptions_.trash = std::make_shared(); - defaultOptions_.enableOdsyncWhenOpenChunkFile = true; - } - - void TearDown() { ::system("rm -rf copyset_node_test"); } - - protected: - CopysetNodeOptions defaultOptions_; - ConcurrentApplyModule concurrentModule_; -}; - -TEST_F(CopysetNodeTest, error_test) { - std::shared_ptr fs( - LocalFsFactory::CreateFs(FileSystemType::EXT4, "")); // NOLINT - std::string rmCmd("rm -f "); - rmCmd += kCurveConfEpochFilename; - - // on_snapshot_save: List failed +namespace curve +{ + namespace chunkserver { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - std::vector files; - files.push_back("test-1.txt"); - files.push_back("test-2.txt"); - - const char* json = - "{\"logicPoolId\":123,\"copysetId\":1345,\"epoch\":0,\"checksum\":" - "774340440}"; // NOLINT - std::string jsonStr(json); - - CopysetNode copysetNode(logicPoolID, copysetID, conf); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - FakeClosure closure; - FakeSnapshotWriter writer; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(10)); - EXPECT_CALL(*mockfs, Write(_, Matcher(_), _, _)) - .Times(1) - .WillOnce(Return(jsonStr.size())); - EXPECT_CALL(*mockfs, Fsync(_)).Times(1).WillOnce(Return(0)); - EXPECT_CALL(*mockfs, Close(_)).Times(1).WillOnce(Return(0)); - EXPECT_CALL(*mockfs, List(_, _)).Times(1).WillOnce(Return(-1)); - - copysetNode.on_snapshot_save(&writer, &closure); - copysetNode.WaitSnapshotDone(); - LOG(INFO) << closure.status().error_cstr(); - } - - // on_snapshot_save: save conf open failed - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - std::vector files; - files.push_back("test-1.txt"); - files.push_back("test-2.txt"); - - CopysetNode copysetNode(logicPoolID, copysetID, conf); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - FakeClosure closure; - FakeSnapshotWriter writer; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); - - copysetNode.on_snapshot_save(&writer, &closure); - copysetNode.WaitSnapshotDone(); - LOG(INFO) << closure.status().error_cstr(); - } - // on_snapshot_save: success - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - std::vector files; - files.push_back("test-1.txt"); - files.push_back("test-2.txt"); - - const char* json = - "{\"logicPoolId\":123,\"copysetId\":1345,\"epoch\":0,\"checksum\":" - "774340440}"; // NOLINT - std::string jsonStr(json); - - CopysetNode copysetNode(logicPoolID, copysetID, conf); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - FakeClosure closure; - FakeSnapshotWriter writer; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(10)); - EXPECT_CALL(*mockfs, Write(_, Matcher(_), _, _)) - .Times(1) - .WillOnce(Return(jsonStr.size())); - EXPECT_CALL(*mockfs, Fsync(_)).Times(1).WillOnce(Return(0)); - EXPECT_CALL(*mockfs, Close(_)).Times(1).WillOnce(Return(0)); - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - - copysetNode.on_snapshot_save(&writer, &closure); - copysetNode.WaitSnapshotDone(); - } - - // on_snapshot_save: success, enableOdsyncWhenOpenChunkFile_ = false - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - std::vector files; - files.push_back("test-1.txt"); - files.push_back("test-2.txt"); - - const char* json = - "{\"logicPoolId\":123,\"copysetId\":1345,\"epoch\":0,\"checksum\":" - "774340440}"; // NOLINT - std::string jsonStr(json); - - CopysetNode copysetNode(logicPoolID, copysetID, conf); - defaultOptions_.enableOdsyncWhenOpenChunkFile = false; - defaultOptions_.syncConcurrency = 20; - defaultOptions_.syncChunkLimit = 2 * 1024 * 1024; - defaultOptions_.syncThreshold = 65536; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - FakeClosure closure; - FakeSnapshotWriter writer; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(10)); - EXPECT_CALL(*mockfs, Write(_, Matcher(_), _, _)) - .Times(1) - .WillOnce(Return(jsonStr.size())); - EXPECT_CALL(*mockfs, Fsync(_)).Times(1).WillOnce(Return(0)); - EXPECT_CALL(*mockfs, Close(_)).Times(1).WillOnce(Return(0)); - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - - copysetNode.on_snapshot_save(&writer, &closure); - copysetNode.WaitSnapshotDone(); - } - // ShipToSync & handle sync time out - { - CopysetNode::copysetSyncPool_ = - std::make_shared>(); - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - - defaultOptions_.enableOdsyncWhenOpenChunkFile = false; - defaultOptions_.syncConcurrency = 20; - defaultOptions_.syncChunkLimit = 2 * 1024 * 1024; - defaultOptions_.syncThreshold = 65536; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - - ChunkID id1 = 100; - ChunkID id2 = 200; - ChunkID id3 = 100; - copysetNode.ShipToSync(id1); - copysetNode.ShipToSync(id2); - copysetNode.ShipToSync(id3); - copysetNode.HandleSyncTimerOut(); - } - - // on_snapshot_load: Dir not exist, File not exist, data init success - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - FakeClosure closure; - FakeSnapshotReader reader; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - DataStoreOptions options; - options.baseDir = "./test-temp"; - options.chunkSize = 16 * 1024 * 1024; - options.metaPageSize = 4 * 1024; - options.blockSize = 4 * 1024; - std::shared_ptr dataStore = - std::make_shared(options, fs); - copysetNode.SetCSDateStore(dataStore); - - EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(false)); - EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(false)); - - ASSERT_EQ(0, copysetNode.on_snapshot_load(&reader)); - LOG(INFO) << "OK"; - } - // on_snapshot_load: Dir not exist, File not exist, data init failed - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - FakeClosure closure; - FakeSnapshotReader reader; - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - DataStoreOptions options; - options.baseDir = "./test-temp"; - options.chunkSize = 16 * 1024 * 1024; - options.metaPageSize = 4 * 1024; - options.blockSize = 4 * 1024; - std::shared_ptr dataStore = - std::make_shared(options, fs); - copysetNode.SetCSDateStore(dataStore); - dataStore->InjectError(); - - EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(false)); - EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(false)); - - ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); - LOG(INFO) << "OK"; - } - // on_snapshot_load: Dir not exist, File exist, load conf.epoch failed - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - FakeClosure closure; - FakeSnapshotReader reader; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - - EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(false)); - EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); - - ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); - } - - // on_snapshot_load: Dir exist, delete failed - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - FakeClosure closure; - FakeSnapshotReader reader; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - MockCurveFilesystemAdaptor* cfa = new MockCurveFilesystemAdaptor(); - auto sfs = new scoped_refptr(cfa); - copysetNode.SetSnapshotFileSystem(sfs); - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*cfa, delete_file(_, _)).Times(1).WillOnce(Return(false)); - - ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); - } - - // on_snapshot_load: Dir exist, delete success, rename failed - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - FakeClosure closure; - FakeSnapshotReader reader; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - defaultOptions_.localFileSystem = mockfs; - MockCurveFilesystemAdaptor* cfa = new MockCurveFilesystemAdaptor(); - auto sfs = new scoped_refptr(cfa); - copysetNode.SetSnapshotFileSystem(sfs); - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*cfa, delete_file(_, _)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*cfa, rename(_, _)).Times(1).WillOnce(Return(false)); - - ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); - } - - // on_snapshot_load: Dir exist, rename success - // file exist, open failed - { - LogicPoolID logicPoolID = 1; - CopysetID copysetID = 1; - Configuration conf; - std::vector files; - files.push_back("test-1.txt"); - - CopysetNode copysetNode(logicPoolID, copysetID, conf); - FakeClosure closure; - FakeSnapshotReader reader; - std::shared_ptr mockfs = - std::make_shared(); - std::unique_ptr epochFile(new ConfEpochFile(mockfs)); - ; - defaultOptions_.localFileSystem = mockfs; - MockCurveFilesystemAdaptor* cfa = new MockCurveFilesystemAdaptor(); - auto sfs = new scoped_refptr(cfa); - copysetNode.SetSnapshotFileSystem(sfs); - copysetNode.SetLocalFileSystem(mockfs); - copysetNode.SetConfEpochFile(std::move(epochFile)); - EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*cfa, delete_file(_, _)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*cfa, rename(_, _)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(true)); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); - - ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); - LOG(INFO) << "OK"; - } - /* on_error */ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - braft::Error error; - ASSERT_DEATH(copysetNode.on_error(error), ".*raft error.*"); - } - /* Fini, raftNode is null */ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - copysetNode.Fini(); - } - /* Fini, raftNode is not null */ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - std::vector files; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - defaultOptions_.localFileSystem = fs; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - copysetNode.Fini(); - } - /* Load/SaveConfEpoch */ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - defaultOptions_.localFileSystem = fs; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - ASSERT_EQ(0, copysetNode.SaveConfEpoch(kCurveConfEpochFilename)); - ASSERT_EQ(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); - ASSERT_EQ(0, copysetNode.GetConfEpoch()); - copysetNode.Fini(); - ::system(rmCmd.c_str()); - } - /* load: ConfEpochFile load failed*/ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - defaultOptions_.localFileSystem = fs; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - ASSERT_NE(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); - copysetNode.Fini(); - ::system(rmCmd.c_str()); - } - /* Load: logic pool id error */ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - uint64_t epoch = 12; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - auto fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); - ConfEpochFile confEpochFile(fs); - ASSERT_EQ(0, confEpochFile.Save(kCurveConfEpochFilename, - logicPoolID + 1, copysetID, epoch)); - defaultOptions_.localFileSystem = fs; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - ASSERT_NE(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); - copysetNode.Fini(); - ::system(rmCmd.c_str()); - } - /* Load: copyset id error */ - { - LogicPoolID logicPoolID = 123; - CopysetID copysetID = 1345; - uint64_t epoch = 12; - Configuration conf; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - auto fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); - ConfEpochFile confEpochFile(fs); - ASSERT_EQ(0, confEpochFile.Save(kCurveConfEpochFilename, logicPoolID, - copysetID + 1, epoch)); - defaultOptions_.localFileSystem = fs; - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - ASSERT_NE(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); - copysetNode.Fini(); - ::system(rmCmd.c_str()); - } -} - -TEST_F(CopysetNodeTest, get_conf_change) { - std::shared_ptr fs( - LocalFsFactory::CreateFs(FileSystemType::EXT4, "")); // NOLINT - std::string rmCmd("rm -f "); - rmCmd += kCurveConfEpochFilename; - - LogicPoolID logicPoolID = 1; - CopysetID copysetID = 1; - Configuration conf; - Configuration conf1; - Configuration conf2; - PeerId peer("127.0.0.1:3200:0"); - PeerId peer1("127.0.0.1:3201:0"); - PeerId emptyPeer; - conf.add_peer(peer); - conf1.add_peer(peer); - conf1.add_peer(peer1); - conf2.add_peer(peer1); - - // There are currently no configuration changes in progress - { - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - copysetNode.SetCopysetNode(mockNode); - - ConfigChangeType type; - Configuration oldConf; - Peer alterPeer; - - copysetNode.on_leader_start(8); - NodeStatus status; - status.state = braft::State::STATE_LEADER; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); - EXPECT_EQ(ConfigChangeType::NONE, type); - } - // Currently adding Peer - { - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - copysetNode.SetCopysetNode(mockNode); - - ConfigChangeType type; - Configuration oldConf; - Peer alterPeer; - - copysetNode.on_leader_start(8); - - EXPECT_CALL(*mockNode, add_peer(_, _)).Times(1); - EXPECT_CALL(*mockNode, remove_peer(_, _)) - .WillOnce(Invoke([](const PeerId& peer, braft::Closure* done) { - done->status().set_error(-1, - "another config change is ongoing"); - })); - Peer addPeer; - addPeer.set_address("127.0.0.1:3202:0"); - Peer removePeer; - removePeer.set_address("127.0.0.1:3200:0"); - copysetNode.AddPeer(addPeer); - copysetNode.RemovePeer(removePeer); - - NodeStatus status; - status.state = braft::State::STATE_LEADER; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); - EXPECT_EQ(ConfigChangeType::ADD_PEER, type); - EXPECT_EQ(addPeer.address(), alterPeer.address()); - } - // Currently removing Peer - { - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - copysetNode.SetCopysetNode(mockNode); - - ConfigChangeType type; - Configuration oldConf; - Peer alterPeer; - - copysetNode.on_leader_start(8); - - EXPECT_CALL(*mockNode, remove_peer(_, _)).Times(1); - EXPECT_CALL(*mockNode, add_peer(_, _)) - .WillOnce( - Invoke([](const braft::PeerId& peer, braft::Closure* done) { - done->status().set_error( - -1, "another config change is ongoing"); - })); - Peer addPeer1; - addPeer1.set_address("127.0.0.1:3202:0"); - Peer removePeer; - removePeer.set_address("127.0.0.1:3200:0"); - copysetNode.RemovePeer(removePeer); - copysetNode.AddPeer(addPeer1); - - NodeStatus status; - status.state = braft::State::STATE_LEADER; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); - EXPECT_EQ(ConfigChangeType::REMOVE_PEER, type); - EXPECT_EQ(removePeer.address(), alterPeer.address()); - } - // Currently transferring leader - { - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - copysetNode.SetCopysetNode(mockNode); - - ConfigChangeType type; - Configuration oldConf; - Peer alterPeer; - - copysetNode.on_leader_start(8); - - Peer transferee1; - transferee1.set_address("127.0.0.1:3201:0"); - Peer transferee2; - transferee2.set_address("127.0.0.1:3200:0"); - EXPECT_CALL(*mockNode, transfer_leadership_to(_)) - .WillOnce(Return(0)) - .WillOnce(Return(-1)); - EXPECT_CALL(*mockNode, leader_id()) - .WillOnce(Return(peer)) - .WillOnce(Return(peer1)) - .WillOnce(Return(peer)); - copysetNode.TransferLeader(transferee1); - copysetNode.TransferLeader(transferee2); - copysetNode.TransferLeader(transferee2); - - NodeStatus status; - status.state = braft::State::STATE_TRANSFERRING; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); - EXPECT_EQ(ConfigChangeType::TRANSFER_LEADER, type); - EXPECT_EQ(transferee1.address(), alterPeer.address()); - } - // Currently changing Peer - { - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - copysetNode.SetCopysetNode(mockNode); - - ConfigChangeType type; - Configuration oldConf; - Peer alterPeer; - - copysetNode.on_leader_start(8); - - EXPECT_CALL(*mockNode, change_peers(_, _)).Times(1); - - Peer addPeer1; - addPeer1.set_address("127.0.0.1:3201:0"); - std::vector peers; - peers.emplace_back(addPeer1); - copysetNode.ChangePeer(peers); - Peer addPeer2; - addPeer2.set_address("127.0.0.1:3202:0"); - peers.emplace_back(addPeer2); - copysetNode.ChangePeer(peers); - - NodeStatus status; - status.state = braft::State::STATE_LEADER; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); - EXPECT_EQ(ConfigChangeType::CHANGE_PEER, type); - EXPECT_EQ(addPeer1.address(), alterPeer.address()); - } - // leader term is less than 0 - { - CopysetNode copysetNode(logicPoolID, copysetID, conf); - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - ConfigChangeType type; - Configuration oldConf; - Peer alterPeer; - copysetNode.on_leader_start(-1); - EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); - EXPECT_EQ(ConfigChangeType::NONE, type); - } -} - -TEST_F(CopysetNodeTest, get_hash) { - std::shared_ptr fs( - LocalFsFactory::CreateFs(FileSystemType::EXT4, "")); // NOLINT - std::string rmCmd("rm -f "); - rmCmd += kCurveConfEpochFilename; - - LogicPoolID logicPoolID = 1 + 1; - CopysetID copysetID = 1 + 1; - Configuration conf; - Configuration conf1; - PeerId peer("127.0.0.1:3200:0"); - PeerId peer1("127.0.0.1:3201:0"); - PeerId emptyPeer; - conf.add_peer(peer); - conf1.add_peer(peer); - conf1.add_peer(peer1); - - std::string hashValue = std::to_string(1355371765); - // get hash - { - std::string hash; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - - // Generate multiple files with data - ::system( - "echo \"abcddddddddd333\" >" - "copyset_node_test/8589934594/data/test-2.txt"); - ::system( - "echo \"mmmmmmmm\" >" - "copyset_node_test/8589934594/data/test-4.txt"); - ::system( - "dd if=/dev/zero of=" - "copyset_node_test/8589934594/data/test-3.txt bs=512 count=15"); // NOLINT - ::system( - "echo \"eeeeeeeeeee\" > " - "copyset_node_test/8589934594/data/test-5.txt"); - - ::system("touch copyset_node_test/8589934594/data/test-1.txt"); - ::system( - "echo \"wwwww\" > " - "copyset_node_test/8589934594/data/test-1.txt"); - - // Get hash - ASSERT_EQ(0, copysetNode.GetHash(&hash)); - ASSERT_STREQ(hashValue.c_str(), hash.c_str()); - ::system("rm -fr copyset_node_test/8589934594"); - } - - { - std::string hash; - // Using different copyset IDs to make the directory different - CopysetNode copysetNode(logicPoolID, copysetID + 1, conf); - - ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); - - // Generate multiple files with data and exchange the order of generated - // files - ::system("touch copyset_node_test/8589934595/data/test-1.txt"); - ::system( - "echo \"wwwww\" > " - "copyset_node_test/8589934595/data/test-1.txt"); - - ::system( - "echo \"mmmmmmmm\" > " - "copyset_node_test/8589934595/data/test-4.txt"); - ::system( - "echo \"eeeeeeeeeee\" > " - "copyset_node_test/8589934595/data/test-5.txt"); - ::system( - "dd if=/dev/zero of=" - "copyset_node_test/8589934595/data/test-3.txt bs=512 count=15"); // NOLINT - ::system( - "echo \"abcddddddddd333\" > " - "copyset_node_test/8589934595/data/test-2.txt"); - - // Get hash - ASSERT_EQ(0, copysetNode.GetHash(&hash)); - ASSERT_STREQ(hashValue.c_str(), hash.c_str()); - ::system("rm -fr copyset_node_test/8589934595"); - } - - // List failed - { - std::string hash; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - - std::shared_ptr mockfs = - std::make_shared(); - copysetNode.SetLocalFileSystem(mockfs); - std::vector files; - files.push_back("test-1.txt"); - - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); - - ASSERT_EQ(-1, copysetNode.GetHash(&hash)); - } - - // List success - { - std::string hash; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockfs = - std::make_shared(); - copysetNode.SetLocalFileSystem(mockfs); - - std::vector files; - - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - - ASSERT_EQ(0, copysetNode.GetHash(&hash)); - ASSERT_EQ(hash, "0"); - } - - // List success, open failed - { - std::string hash; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockfs = - std::make_shared(); - copysetNode.SetLocalFileSystem(mockfs); - - std::vector files; - files.push_back("test-1.txt"); - - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); - - ASSERT_EQ(-1, copysetNode.GetHash(&hash)); - } - - // List success, open success，fstat failed - { - std::string hash; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockfs = - std::make_shared(); - copysetNode.SetLocalFileSystem(mockfs); - - std::vector files; - files.push_back("test-1.txt"); + using ::testing::_; + using ::testing::AnyNumber; + using ::testing::AtLeast; + using ::testing::DoAll; + using ::testing::InSequence; + using ::testing::Invoke; + using ::testing::Matcher; + using ::testing::Return; + using ::testing::SaveArgPointee; + using ::testing::SetArgPointee; + using ::testing::SetArgReferee; + + using curve::chunkserver::concurrent::ConcurrentApplyOption; + using curve::fs::FileSystemType; + using curve::fs::MockLocalFileSystem; + + const char copysetUri[] = "local://./copyset_node_test"; + const int port = 9044; + + class FakeSnapshotReader : public braft::SnapshotReader + { + public: + std::string get_path() + { + /*Returns a non-existent path*/ + return std::string("/1002093939/temp/238408034"); + } + void list_files(std::vector *files) { return; } + int load_meta(braft::SnapshotMeta *meta) { return 1; } + std::string generate_uri_for_copy() { return std::string(""); } + }; + + class FakeSnapshotWriter : public braft::SnapshotWriter + { + public: + std::string get_path() + { + /*Returns a non-existent path*/ + return std::string("."); + } + void list_files(std::vector *files) { return; } + virtual int save_meta(const braft::SnapshotMeta &meta) { return 0; } + + virtual int add_file(const std::string &filename) { return 0; } + + virtual int add_file(const std::string &filename, + const ::google::protobuf::Message *file_meta) + { + return 0; + } + + virtual int remove_file(const std::string &filename) { return 0; } + }; + + class FakeClosure : public braft::Closure + { + public: + void Run() { std::cerr << "FakeClosure run" << std::endl; } + }; + + class CopysetNodeTest : public ::testing::Test + { + protected: + void SetUp() + { + defaultOptions_.ip = "127.0.0.1"; + defaultOptions_.port = port; + defaultOptions_.electionTimeoutMs = 1000; + defaultOptions_.snapshotIntervalS = 30; + defaultOptions_.catchupMargin = 50; + defaultOptions_.chunkDataUri = copysetUri; + defaultOptions_.chunkSnapshotUri = copysetUri; + defaultOptions_.logUri = copysetUri; + defaultOptions_.raftMetaUri = copysetUri; + defaultOptions_.raftSnapshotUri = copysetUri; + defaultOptions_.loadConcurrency = 5; + defaultOptions_.syncConcurrency = 20; + defaultOptions_.checkRetryTimes = 3; + defaultOptions_.finishLoadMargin = 1000; + + defaultOptions_.concurrentapply = &concurrentModule_; + ConcurrentApplyOption opt{2, 1, 2, 1}; + defaultOptions_.concurrentapply->Init(opt); + std::shared_ptr fs = + LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); + ASSERT_TRUE(nullptr != fs); + defaultOptions_.localFileSystem = fs; + defaultOptions_.chunkFilePool = std::make_shared(fs); + defaultOptions_.trash = std::make_shared(); + defaultOptions_.enableOdsyncWhenOpenChunkFile = true; + } + + void TearDown() { ::system("rm -rf copyset_node_test"); } + + protected: + CopysetNodeOptions defaultOptions_; + ConcurrentApplyModule concurrentModule_; + }; + + TEST_F(CopysetNodeTest, error_test) + { + std::shared_ptr fs( + LocalFsFactory::CreateFs(FileSystemType::EXT4, "")); // NOLINT + std::string rmCmd("rm -f "); + rmCmd += kCurveConfEpochFilename; + + // on_snapshot_save: List failed + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + std::vector files; + files.push_back("test-1.txt"); + files.push_back("test-2.txt"); + + const char *json = + "{\"logicPoolId\":123,\"copysetId\":1345,\"epoch\":0,\"checksum\":" + "774340440}"; // NOLINT + std::string jsonStr(json); + + CopysetNode copysetNode(logicPoolID, copysetID, conf); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + FakeClosure closure; + FakeSnapshotWriter writer; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(10)); + EXPECT_CALL(*mockfs, Write(_, Matcher(_), _, _)) + .Times(1) + .WillOnce(Return(jsonStr.size())); + EXPECT_CALL(*mockfs, Fsync(_)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*mockfs, Close(_)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*mockfs, List(_, _)).Times(1).WillOnce(Return(-1)); + + copysetNode.on_snapshot_save(&writer, &closure); + copysetNode.WaitSnapshotDone(); + LOG(INFO) << closure.status().error_cstr(); + } + + // on_snapshot_save: save conf open failed + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + std::vector files; + files.push_back("test-1.txt"); + files.push_back("test-2.txt"); + + CopysetNode copysetNode(logicPoolID, copysetID, conf); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + FakeClosure closure; + FakeSnapshotWriter writer; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); + + copysetNode.on_snapshot_save(&writer, &closure); + copysetNode.WaitSnapshotDone(); + LOG(INFO) << closure.status().error_cstr(); + } + // on_snapshot_save: success + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + std::vector files; + files.push_back("test-1.txt"); + files.push_back("test-2.txt"); + + const char *json = + "{\"logicPoolId\":123,\"copysetId\":1345,\"epoch\":0,\"checksum\":" + "774340440}"; // NOLINT + std::string jsonStr(json); + + CopysetNode copysetNode(logicPoolID, copysetID, conf); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + FakeClosure closure; + FakeSnapshotWriter writer; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(10)); + EXPECT_CALL(*mockfs, Write(_, Matcher(_), _, _)) + .Times(1) + .WillOnce(Return(jsonStr.size())); + EXPECT_CALL(*mockfs, Fsync(_)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*mockfs, Close(_)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + + copysetNode.on_snapshot_save(&writer, &closure); + copysetNode.WaitSnapshotDone(); + } + + // on_snapshot_save: success, enableOdsyncWhenOpenChunkFile_ = false + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + std::vector files; + files.push_back("test-1.txt"); + files.push_back("test-2.txt"); + + const char *json = + "{\"logicPoolId\":123,\"copysetId\":1345,\"epoch\":0,\"checksum\":" + "774340440}"; // NOLINT + std::string jsonStr(json); + + CopysetNode copysetNode(logicPoolID, copysetID, conf); + defaultOptions_.enableOdsyncWhenOpenChunkFile = false; + defaultOptions_.syncConcurrency = 20; + defaultOptions_.syncChunkLimit = 2 * 1024 * 1024; + defaultOptions_.syncThreshold = 65536; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + FakeClosure closure; + FakeSnapshotWriter writer; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(10)); + EXPECT_CALL(*mockfs, Write(_, Matcher(_), _, _)) + .Times(1) + .WillOnce(Return(jsonStr.size())); + EXPECT_CALL(*mockfs, Fsync(_)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*mockfs, Close(_)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + + copysetNode.on_snapshot_save(&writer, &closure); + copysetNode.WaitSnapshotDone(); + } + // ShipToSync & handle sync time out + { + CopysetNode::copysetSyncPool_ = + std::make_shared>(); + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + + defaultOptions_.enableOdsyncWhenOpenChunkFile = false; + defaultOptions_.syncConcurrency = 20; + defaultOptions_.syncChunkLimit = 2 * 1024 * 1024; + defaultOptions_.syncThreshold = 65536; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + + ChunkID id1 = 100; + ChunkID id2 = 200; + ChunkID id3 = 100; + copysetNode.ShipToSync(id1); + copysetNode.ShipToSync(id2); + copysetNode.ShipToSync(id3); + copysetNode.HandleSyncTimerOut(); + } + + // on_snapshot_load: Dir not exist, File not exist, data init success + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + FakeClosure closure; + FakeSnapshotReader reader; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + DataStoreOptions options; + options.baseDir = "./test-temp"; + options.chunkSize = 16 * 1024 * 1024; + options.metaPageSize = 4 * 1024; + options.blockSize = 4 * 1024; + std::shared_ptr dataStore = + std::make_shared(options, fs); + copysetNode.SetCSDateStore(dataStore); + + EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(false)); + EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(false)); + + ASSERT_EQ(0, copysetNode.on_snapshot_load(&reader)); + LOG(INFO) << "OK"; + } + // on_snapshot_load: Dir not exist, File not exist, data init failed + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + FakeClosure closure; + FakeSnapshotReader reader; + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + DataStoreOptions options; + options.baseDir = "./test-temp"; + options.chunkSize = 16 * 1024 * 1024; + options.metaPageSize = 4 * 1024; + options.blockSize = 4 * 1024; + std::shared_ptr dataStore = + std::make_shared(options, fs); + copysetNode.SetCSDateStore(dataStore); + dataStore->InjectError(); + + EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(false)); + EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(false)); + + ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); + LOG(INFO) << "OK"; + } + // on_snapshot_load: Dir not exist, File exist, load conf.epoch failed + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + FakeClosure closure; + FakeSnapshotReader reader; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + + EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(false)); + EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); + + ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); + } + + // on_snapshot_load: Dir exist, delete failed + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + FakeClosure closure; + FakeSnapshotReader reader; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + MockCurveFilesystemAdaptor *cfa = new MockCurveFilesystemAdaptor(); + auto sfs = new scoped_refptr(cfa); + copysetNode.SetSnapshotFileSystem(sfs); + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*cfa, delete_file(_, _)).Times(1).WillOnce(Return(false)); + + ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); + } + + // on_snapshot_load: Dir exist, delete success, rename failed + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + FakeClosure closure; + FakeSnapshotReader reader; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + defaultOptions_.localFileSystem = mockfs; + MockCurveFilesystemAdaptor *cfa = new MockCurveFilesystemAdaptor(); + auto sfs = new scoped_refptr(cfa); + copysetNode.SetSnapshotFileSystem(sfs); + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*cfa, delete_file(_, _)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*cfa, rename(_, _)).Times(1).WillOnce(Return(false)); + + ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); + } + + // on_snapshot_load: Dir exist, rename success + // file exist, open failed + { + LogicPoolID logicPoolID = 1; + CopysetID copysetID = 1; + Configuration conf; + std::vector files; + files.push_back("test-1.txt"); + + CopysetNode copysetNode(logicPoolID, copysetID, conf); + FakeClosure closure; + FakeSnapshotReader reader; + std::shared_ptr mockfs = + std::make_shared(); + std::unique_ptr epochFile(new ConfEpochFile(mockfs)); + ; + defaultOptions_.localFileSystem = mockfs; + MockCurveFilesystemAdaptor *cfa = new MockCurveFilesystemAdaptor(); + auto sfs = new scoped_refptr(cfa); + copysetNode.SetSnapshotFileSystem(sfs); + copysetNode.SetLocalFileSystem(mockfs); + copysetNode.SetConfEpochFile(std::move(epochFile)); + EXPECT_CALL(*mockfs, DirExists(_)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*cfa, delete_file(_, _)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*cfa, rename(_, _)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*mockfs, FileExists(_)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); + + ASSERT_EQ(-1, copysetNode.on_snapshot_load(&reader)); + LOG(INFO) << "OK"; + } + /* on_error */ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + braft::Error error; + ASSERT_DEATH(copysetNode.on_error(error), ".*raft error.*"); + } + /* Fini, raftNode is null */ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + copysetNode.Fini(); + } + /* Fini, raftNode is not null */ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + std::vector files; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + defaultOptions_.localFileSystem = fs; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + copysetNode.Fini(); + } + /* Load/SaveConfEpoch */ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + defaultOptions_.localFileSystem = fs; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + ASSERT_EQ(0, copysetNode.SaveConfEpoch(kCurveConfEpochFilename)); + ASSERT_EQ(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); + ASSERT_EQ(0, copysetNode.GetConfEpoch()); + copysetNode.Fini(); + ::system(rmCmd.c_str()); + } + /* load: ConfEpochFile load failed*/ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + defaultOptions_.localFileSystem = fs; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + ASSERT_NE(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); + copysetNode.Fini(); + ::system(rmCmd.c_str()); + } + /* Load: logic pool id error */ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + uint64_t epoch = 12; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + auto fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); + ConfEpochFile confEpochFile(fs); + ASSERT_EQ(0, confEpochFile.Save(kCurveConfEpochFilename, + logicPoolID + 1, copysetID, epoch)); + defaultOptions_.localFileSystem = fs; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + ASSERT_NE(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); + copysetNode.Fini(); + ::system(rmCmd.c_str()); + } + /* Load: copyset id error */ + { + LogicPoolID logicPoolID = 123; + CopysetID copysetID = 1345; + uint64_t epoch = 12; + Configuration conf; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + auto fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, ""); + ConfEpochFile confEpochFile(fs); + ASSERT_EQ(0, confEpochFile.Save(kCurveConfEpochFilename, logicPoolID, + copysetID + 1, epoch)); + defaultOptions_.localFileSystem = fs; + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + ASSERT_NE(0, copysetNode.LoadConfEpoch(kCurveConfEpochFilename)); + copysetNode.Fini(); + ::system(rmCmd.c_str()); + } + } - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(3)); - EXPECT_CALL(*mockfs, Fstat(_, _)).Times(1).WillOnce(Return(-1)); + TEST_F(CopysetNodeTest, get_conf_change) + { + std::shared_ptr fs( + LocalFsFactory::CreateFs(FileSystemType::EXT4, "")); // NOLINT + std::string rmCmd("rm -f "); + rmCmd += kCurveConfEpochFilename; + + LogicPoolID logicPoolID = 1; + CopysetID copysetID = 1; + Configuration conf; + Configuration conf1; + Configuration conf2; + PeerId peer("127.0.0.1:3200:0"); + PeerId peer1("127.0.0.1:3201:0"); + PeerId emptyPeer; + conf.add_peer(peer); + conf1.add_peer(peer); + conf1.add_peer(peer1); + conf2.add_peer(peer1); + + // There are currently no configuration changes in progress + { + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + copysetNode.SetCopysetNode(mockNode); + + ConfigChangeType type; + Configuration oldConf; + Peer alterPeer; + + copysetNode.on_leader_start(8); + NodeStatus status; + status.state = braft::State::STATE_LEADER; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); + EXPECT_EQ(ConfigChangeType::NONE, type); + } + // Currently adding Peer + { + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + copysetNode.SetCopysetNode(mockNode); + + ConfigChangeType type; + Configuration oldConf; + Peer alterPeer; + + copysetNode.on_leader_start(8); + + EXPECT_CALL(*mockNode, add_peer(_, _)).Times(1); + EXPECT_CALL(*mockNode, remove_peer(_, _)) + .WillOnce(Invoke([](const PeerId &peer, braft::Closure *done) + { done->status().set_error(-1, + "another config change is ongoing"); })); + Peer addPeer; + addPeer.set_address("127.0.0.1:3202:0"); + Peer removePeer; + removePeer.set_address("127.0.0.1:3200:0"); + copysetNode.AddPeer(addPeer); + copysetNode.RemovePeer(removePeer); + + NodeStatus status; + status.state = braft::State::STATE_LEADER; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); + EXPECT_EQ(ConfigChangeType::ADD_PEER, type); + EXPECT_EQ(addPeer.address(), alterPeer.address()); + } + // Currently removing Peer + { + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + copysetNode.SetCopysetNode(mockNode); + + ConfigChangeType type; + Configuration oldConf; + Peer alterPeer; + + copysetNode.on_leader_start(8); + + EXPECT_CALL(*mockNode, remove_peer(_, _)).Times(1); + EXPECT_CALL(*mockNode, add_peer(_, _)) + .WillOnce( + Invoke([](const braft::PeerId &peer, braft::Closure *done) + { done->status().set_error( + -1, "another config change is ongoing"); })); + Peer addPeer1; + addPeer1.set_address("127.0.0.1:3202:0"); + Peer removePeer; + removePeer.set_address("127.0.0.1:3200:0"); + copysetNode.RemovePeer(removePeer); + copysetNode.AddPeer(addPeer1); + + NodeStatus status; + status.state = braft::State::STATE_LEADER; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); + EXPECT_EQ(ConfigChangeType::REMOVE_PEER, type); + EXPECT_EQ(removePeer.address(), alterPeer.address()); + } + // Currently transferring leader + { + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + copysetNode.SetCopysetNode(mockNode); + + ConfigChangeType type; + Configuration oldConf; + Peer alterPeer; + + copysetNode.on_leader_start(8); + + Peer transferee1; + transferee1.set_address("127.0.0.1:3201:0"); + Peer transferee2; + transferee2.set_address("127.0.0.1:3200:0"); + EXPECT_CALL(*mockNode, transfer_leadership_to(_)) + .WillOnce(Return(0)) + .WillOnce(Return(-1)); + EXPECT_CALL(*mockNode, leader_id()) + .WillOnce(Return(peer)) + .WillOnce(Return(peer1)) + .WillOnce(Return(peer)); + copysetNode.TransferLeader(transferee1); + copysetNode.TransferLeader(transferee2); + copysetNode.TransferLeader(transferee2); + + NodeStatus status; + status.state = braft::State::STATE_TRANSFERRING; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); + EXPECT_EQ(ConfigChangeType::TRANSFER_LEADER, type); + EXPECT_EQ(transferee1.address(), alterPeer.address()); + } + // Currently changing Peer + { + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + copysetNode.SetCopysetNode(mockNode); + + ConfigChangeType type; + Configuration oldConf; + Peer alterPeer; + + copysetNode.on_leader_start(8); + + EXPECT_CALL(*mockNode, change_peers(_, _)).Times(1); + + Peer addPeer1; + addPeer1.set_address("127.0.0.1:3201:0"); + std::vector peers; + peers.emplace_back(addPeer1); + copysetNode.ChangePeer(peers); + Peer addPeer2; + addPeer2.set_address("127.0.0.1:3202:0"); + peers.emplace_back(addPeer2); + copysetNode.ChangePeer(peers); + + NodeStatus status; + status.state = braft::State::STATE_LEADER; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); + EXPECT_EQ(ConfigChangeType::CHANGE_PEER, type); + EXPECT_EQ(addPeer1.address(), alterPeer.address()); + } + // leader term is less than 0 + { + CopysetNode copysetNode(logicPoolID, copysetID, conf); + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + ConfigChangeType type; + Configuration oldConf; + Peer alterPeer; + copysetNode.on_leader_start(-1); + EXPECT_EQ(0, copysetNode.GetConfChange(&type, &oldConf, &alterPeer)); + EXPECT_EQ(ConfigChangeType::NONE, type); + } + } - ASSERT_EQ(-1, copysetNode.GetHash(&hash)); - } + TEST_F(CopysetNodeTest, get_hash) + { + std::shared_ptr fs( + LocalFsFactory::CreateFs(FileSystemType::EXT4, "")); // NOLINT + std::string rmCmd("rm -f "); + rmCmd += kCurveConfEpochFilename; + + LogicPoolID logicPoolID = 1 + 1; + CopysetID copysetID = 1 + 1; + Configuration conf; + Configuration conf1; + PeerId peer("127.0.0.1:3200:0"); + PeerId peer1("127.0.0.1:3201:0"); + PeerId emptyPeer; + conf.add_peer(peer); + conf1.add_peer(peer); + conf1.add_peer(peer1); + + std::string hashValue = std::to_string(1355371765); + // get hash + { + std::string hash; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + + // Generate multiple files with data + ::system( + "echo \"abcddddddddd333\" >" + "copyset_node_test/8589934594/data/test-2.txt"); + ::system( + "echo \"mmmmmmmm\" >" + "copyset_node_test/8589934594/data/test-4.txt"); + ::system( + "dd if=/dev/zero of=" + "copyset_node_test/8589934594/data/test-3.txt bs=512 count=15"); // NOLINT + ::system( + "echo \"eeeeeeeeeee\" > " + "copyset_node_test/8589934594/data/test-5.txt"); + + ::system("touch copyset_node_test/8589934594/data/test-1.txt"); + ::system( + "echo \"wwwww\" > " + "copyset_node_test/8589934594/data/test-1.txt"); + + // Get hash + ASSERT_EQ(0, copysetNode.GetHash(&hash)); + ASSERT_STREQ(hashValue.c_str(), hash.c_str()); + ::system("rm -fr copyset_node_test/8589934594"); + } + + { + std::string hash; + // Using different copyset IDs to make the directory different + CopysetNode copysetNode(logicPoolID, copysetID + 1, conf); + + ASSERT_EQ(0, copysetNode.Init(defaultOptions_)); + + // Generate multiple files with data and exchange the order of generated + // files + ::system("touch copyset_node_test/8589934595/data/test-1.txt"); + ::system( + "echo \"wwwww\" > " + "copyset_node_test/8589934595/data/test-1.txt"); + + ::system( + "echo \"mmmmmmmm\" > " + "copyset_node_test/8589934595/data/test-4.txt"); + ::system( + "echo \"eeeeeeeeeee\" > " + "copyset_node_test/8589934595/data/test-5.txt"); + ::system( + "dd if=/dev/zero of=" + "copyset_node_test/8589934595/data/test-3.txt bs=512 count=15"); // NOLINT + ::system( + "echo \"abcddddddddd333\" > " + "copyset_node_test/8589934595/data/test-2.txt"); + + // Get hash + ASSERT_EQ(0, copysetNode.GetHash(&hash)); + ASSERT_STREQ(hashValue.c_str(), hash.c_str()); + ::system("rm -fr copyset_node_test/8589934595"); + } + + // List failed + { + std::string hash; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + + std::shared_ptr mockfs = + std::make_shared(); + copysetNode.SetLocalFileSystem(mockfs); + + std::vector files; + files.push_back("test-1.txt"); + + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); + + ASSERT_EQ(-1, copysetNode.GetHash(&hash)); + } + + // List success + { + std::string hash; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockfs = + std::make_shared(); + copysetNode.SetLocalFileSystem(mockfs); + + std::vector files; + + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + + ASSERT_EQ(0, copysetNode.GetHash(&hash)); + ASSERT_EQ(hash, "0"); + } + + // List success, open failed + { + std::string hash; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockfs = + std::make_shared(); + copysetNode.SetLocalFileSystem(mockfs); + + std::vector files; + files.push_back("test-1.txt"); + + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(-1)); + + ASSERT_EQ(-1, copysetNode.GetHash(&hash)); + } + + // List success, open success，fstat failed + { + std::string hash; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockfs = + std::make_shared(); + copysetNode.SetLocalFileSystem(mockfs); + + std::vector files; + files.push_back("test-1.txt"); + + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(3)); + EXPECT_CALL(*mockfs, Fstat(_, _)).Times(1).WillOnce(Return(-1)); + + ASSERT_EQ(-1, copysetNode.GetHash(&hash)); + } + + // List success, open success, fstat success, read failed + { + std::string hash; + struct stat fileInfo; + fileInfo.st_size = 1024; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockfs = + std::make_shared(); + copysetNode.SetLocalFileSystem(mockfs); + + std::vector files; + files.push_back("test-1.txt"); + + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(3)); + EXPECT_CALL(*mockfs, Fstat(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0))); + EXPECT_CALL(*mockfs, Read(_, _, _, _)).Times(1).WillOnce(Return(-1)); + + ASSERT_EQ(-1, copysetNode.GetHash(&hash)); + } + + // List success, open success, fstat success, read success + { + char *buff = new (std::nothrow) char[1024]; + ::memset(buff, 'a', 1024); + std::string hash; + struct stat fileInfo; + fileInfo.st_size = 1024; + CopysetNode copysetNode(logicPoolID, copysetID, conf); + std::shared_ptr mockfs = + std::make_shared(); + copysetNode.SetLocalFileSystem(mockfs); + + std::vector files; + files.push_back("test-1.txt"); + + EXPECT_CALL(*mockfs, List(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); + EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(3)); + EXPECT_CALL(*mockfs, Fstat(_, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0))); + EXPECT_CALL(*mockfs, Read(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<1>(*buff), Return(1024))); + + ASSERT_EQ(0, copysetNode.GetHash(&hash)); + } + } - // List success, open success, fstat success, read failed - { - std::string hash; - struct stat fileInfo; - fileInfo.st_size = 1024; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockfs = - std::make_shared(); - copysetNode.SetLocalFileSystem(mockfs); - - std::vector files; - files.push_back("test-1.txt"); - - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(3)); - EXPECT_CALL(*mockfs, Fstat(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0))); - EXPECT_CALL(*mockfs, Read(_, _, _, _)).Times(1).WillOnce(Return(-1)); - - ASSERT_EQ(-1, copysetNode.GetHash(&hash)); - } - - // List success, open success, fstat success, read success - { - char* buff = new (std::nothrow) char[1024]; - ::memset(buff, 'a', 1024); - std::string hash; - struct stat fileInfo; - fileInfo.st_size = 1024; - CopysetNode copysetNode(logicPoolID, copysetID, conf); - std::shared_ptr mockfs = - std::make_shared(); - copysetNode.SetLocalFileSystem(mockfs); - - std::vector files; - files.push_back("test-1.txt"); - - EXPECT_CALL(*mockfs, List(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(files), Return(0))); - EXPECT_CALL(*mockfs, Open(_, _)).Times(1).WillOnce(Return(3)); - EXPECT_CALL(*mockfs, Fstat(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0))); - EXPECT_CALL(*mockfs, Read(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(*buff), Return(1024))); - - ASSERT_EQ(0, copysetNode.GetHash(&hash)); - } -} - -TEST_F(CopysetNodeTest, get_leader_status) { - LogicPoolID logicPoolID = 1; - CopysetID copysetID = 1; - Configuration conf; - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - CopysetNode copysetNode(logicPoolID, copysetID, conf); - copysetNode.SetCopysetNode(mockNode); - - // The current peer is not a leader, and there is currently no leader - { - NodeStatus status; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - NodeStatus leaderStatus; - ASSERT_FALSE(copysetNode.GetLeaderStatus(&leaderStatus)); - } - - // The current peer is the leader - { - NodeStatus status; - status.leader_id.parse("127.0.0.1:3200:0"); - status.peer_id = status.leader_id; - status.committed_index = 6666; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(status)); - NodeStatus leaderStatus; - ASSERT_TRUE(copysetNode.GetLeaderStatus(&leaderStatus)); - ASSERT_EQ(status.committed_index, leaderStatus.committed_index); - } - - // There is a leader, but it is not the current peer - { - // Simulate starting chunkserver - CopysetNodeManager* copysetNodeManager = - &CopysetNodeManager::GetInstance(); - ASSERT_EQ(0, copysetNodeManager->Init(defaultOptions_)); - ASSERT_EQ(0, copysetNodeManager->Run()); - PeerId leader_peer("127.0.0.1:9044:0"); - brpc::Server server; - ASSERT_EQ(0, copysetNodeManager->AddService(&server, leader_peer.addr)); - if (server.Start(port, NULL) != 0) { - LOG(FATAL) << "Fail to start Server"; + TEST_F(CopysetNodeTest, get_leader_status) + { + LogicPoolID logicPoolID = 1; + CopysetID copysetID = 1; + Configuration conf; + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + CopysetNode copysetNode(logicPoolID, copysetID, conf); + copysetNode.SetCopysetNode(mockNode); + + // The current peer is not a leader, and there is currently no leader + { + NodeStatus status; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + NodeStatus leaderStatus; + ASSERT_FALSE(copysetNode.GetLeaderStatus(&leaderStatus)); + } + + // The current peer is the leader + { + NodeStatus status; + status.leader_id.parse("127.0.0.1:3200:0"); + status.peer_id = status.leader_id; + status.committed_index = 6666; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(status)); + NodeStatus leaderStatus; + ASSERT_TRUE(copysetNode.GetLeaderStatus(&leaderStatus)); + ASSERT_EQ(status.committed_index, leaderStatus.committed_index); + } + + // There is a leader, but it is not the current peer + { + // Simulate starting chunkserver + CopysetNodeManager *copysetNodeManager = + &CopysetNodeManager::GetInstance(); + ASSERT_EQ(0, copysetNodeManager->Init(defaultOptions_)); + ASSERT_EQ(0, copysetNodeManager->Run()); + PeerId leader_peer("127.0.0.1:9044:0"); + brpc::Server server; + ASSERT_EQ(0, copysetNodeManager->AddService(&server, leader_peer.addr)); + if (server.Start(port, NULL) != 0) + { + LOG(FATAL) << "Fail to start Server"; + } + // Construct a leader copyset + ASSERT_TRUE(copysetNodeManager->CreateCopysetNode(logicPoolID, + copysetID, conf)); + auto leaderNode = + copysetNodeManager->GetCopysetNode(logicPoolID, copysetID); + ASSERT_TRUE(nullptr != leaderNode); + // Set expected values + std::shared_ptr mockLeader = + std::make_shared(logicPoolID, copysetID); + leaderNode->SetCopysetNode(mockLeader); + NodeStatus mockLeaderStatus; + mockLeaderStatus.leader_id = leader_peer; + mockLeaderStatus.peer_id = leader_peer; + mockLeaderStatus.committed_index = 10000; + mockLeaderStatus.known_applied_index = 6789; + EXPECT_CALL(*mockLeader, get_status(_)) + .WillRepeatedly(SetArgPointee<0>(mockLeaderStatus)); + + // Test obtaining the committed index of the leader through the node of + // the follower + NodeStatus followerStatus; + followerStatus.leader_id = leader_peer; + followerStatus.peer_id.parse("127.0.0.1:3201:0"); + followerStatus.committed_index = 3456; + followerStatus.known_applied_index = 3456; + EXPECT_CALL(*mockNode, get_status(_)) + .WillOnce(SetArgPointee<0>(followerStatus)); + + NodeStatus leaderStatus; + ASSERT_TRUE(copysetNode.GetLeaderStatus(&leaderStatus)); + ASSERT_EQ(mockLeaderStatus.committed_index, + leaderStatus.committed_index); + ASSERT_EQ(mockLeaderStatus.known_applied_index, + leaderStatus.known_applied_index); + } } - // Construct a leader copyset - ASSERT_TRUE(copysetNodeManager->CreateCopysetNode(logicPoolID, - copysetID, conf)); - auto leaderNode = - copysetNodeManager->GetCopysetNode(logicPoolID, copysetID); - ASSERT_TRUE(nullptr != leaderNode); - // Set expected values - std::shared_ptr mockLeader = - std::make_shared(logicPoolID, copysetID); - leaderNode->SetCopysetNode(mockLeader); - NodeStatus mockLeaderStatus; - mockLeaderStatus.leader_id = leader_peer; - mockLeaderStatus.peer_id = leader_peer; - mockLeaderStatus.committed_index = 10000; - mockLeaderStatus.known_applied_index = 6789; - EXPECT_CALL(*mockLeader, get_status(_)) - .WillRepeatedly(SetArgPointee<0>(mockLeaderStatus)); - - // Test obtaining the committed index of the leader through the node of - // the follower - NodeStatus followerStatus; - followerStatus.leader_id = leader_peer; - followerStatus.peer_id.parse("127.0.0.1:3201:0"); - followerStatus.committed_index = 3456; - followerStatus.known_applied_index = 3456; - EXPECT_CALL(*mockNode, get_status(_)) - .WillOnce(SetArgPointee<0>(followerStatus)); - - NodeStatus leaderStatus; - ASSERT_TRUE(copysetNode.GetLeaderStatus(&leaderStatus)); - ASSERT_EQ(mockLeaderStatus.committed_index, - leaderStatus.committed_index); - ASSERT_EQ(mockLeaderStatus.known_applied_index, - leaderStatus.known_applied_index); - } -} - -TEST_F(CopysetNodeTest, is_lease_leader) { - LogicPoolID logicPoolID = 1; - CopysetID copysetID = 1; - Configuration conf; - std::shared_ptr mockNode = - std::make_shared(logicPoolID, copysetID); - CopysetNode copysetNode(logicPoolID, copysetID, conf); - copysetNode.Init(defaultOptions_); - copysetNode.SetCopysetNode(mockNode); - - EXPECT_FALSE(copysetNode.IsLeaderTerm()); - EXPECT_EQ(-1, copysetNode.LeaderTerm()); - - // not leader now - { - std::vector states = { - braft::LEASE_DISABLED, braft::LEASE_VALID, braft::LEASE_NOT_READY, - braft::LEASE_EXPIRED}; - braft::LeaderLeaseStatus status; - for (auto& state : states) { - status.state = state; - ASSERT_FALSE(copysetNode.IsLeaseLeader(status)); + + TEST_F(CopysetNodeTest, is_lease_leader) + { + LogicPoolID logicPoolID = 1; + CopysetID copysetID = 1; + Configuration conf; + std::shared_ptr mockNode = + std::make_shared(logicPoolID, copysetID); + CopysetNode copysetNode(logicPoolID, copysetID, conf); + copysetNode.Init(defaultOptions_); + copysetNode.SetCopysetNode(mockNode); + + EXPECT_FALSE(copysetNode.IsLeaderTerm()); + EXPECT_EQ(-1, copysetNode.LeaderTerm()); + + // not leader now + { + std::vector states = { + braft::LEASE_DISABLED, braft::LEASE_VALID, braft::LEASE_NOT_READY, + braft::LEASE_EXPIRED}; + braft::LeaderLeaseStatus status; + for (auto &state : states) + { + status.state = state; + ASSERT_FALSE(copysetNode.IsLeaseLeader(status)); + } + } + + // ABA problem, current node is term 8(on leader start), + // but leader lease term is 10 + { + copysetNode.on_leader_start(8); + braft::LeaderLeaseStatus status; + status.term = 10; + status.state = braft::LEASE_NOT_READY; + ASSERT_FALSE(copysetNode.IsLeaseLeader(status)); + } + + // normal condition + { + copysetNode.on_leader_start(10); + braft::LeaderLeaseStatus status; + status.term = 10; + status.state = braft::LEASE_VALID; + ASSERT_TRUE(copysetNode.IsLeaseLeader(status)); + } } - } - // ABA problem, current node is term 8(on leader start), - // but leader lease term is 10 - { - copysetNode.on_leader_start(8); - braft::LeaderLeaseStatus status; - status.term = 10; - status.state = braft::LEASE_NOT_READY; - ASSERT_FALSE(copysetNode.IsLeaseLeader(status)); - } - - // normal condition - { - copysetNode.on_leader_start(10); - braft::LeaderLeaseStatus status; - status.term = 10; - status.state = braft::LEASE_VALID; - ASSERT_TRUE(copysetNode.IsLeaseLeader(status)); - } -} - -} // namespace chunkserver -} // namespace curve + } // namespace chunkserver +} // namespace curve diff --git a/test/client/copyset_client_test.cpp b/test/client/copyset_client_test.cpp index 548db4f6d0..759cb6fc3c 100644 --- a/test/client/copyset_client_test.cpp +++ b/test/client/copyset_client_test.cpp @@ -28,8 +28,8 @@ #include #include -#include // NOLINT -#include //NOLINT +#include // NOLINT +#include //NOLINT #include "src/client/chunk_closure.h" #include "src/client/metacache.h" @@ -42,3990 +42,4030 @@ #include "test/client/mock/mock_request_context.h" #include "test/client/mock/mock_request_scheduler.h" -namespace curve { -namespace client { - -using curve::chunkserver::CHUNK_OP_STATUS; -using curve::chunkserver::ChunkRequest; - -using curve::client::MetaCache; -using curve::common::TimeUtility; -using ::testing::_; -using ::testing::AnyNumber; -using ::testing::AtLeast; -using ::testing::DoAll; -using ::testing::InSequence; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::SaveArgPointee; -using ::testing::SetArgPointee; -using ::testing::SetArgReferee; - -class CopysetClientTest : public testing::Test { - protected: - virtual void SetUp() { - listenAddr_ = "127.0.0.1:9109"; - server_ = new brpc::Server(); - } - - virtual void TearDown() { - server_->Stop(0); - server_->Join(); - delete server_; - server_ = nullptr; - } - - public: - std::string listenAddr_; - brpc::Server* server_; -}; - -/* TODO(wudemiao) current controller error cannot be returned through mock */ -int gWriteCntlFailedCode = 0; -int gReadCntlFailedCode = 0; - -static void WriteChunkFunc(::google::protobuf::RpcController* controller, - const ::curve::chunkserver::ChunkRequest* request, - ::curve::chunkserver::ChunkResponse* response, - google::protobuf::Closure* done) { - /* return response */ - brpc::ClosureGuard doneGuard(done); - if (0 != gWriteCntlFailedCode) { - if (gWriteCntlFailedCode == brpc::ERPCTIMEDOUT) { - std::this_thread::sleep_for(std::chrono::milliseconds(3500)); +namespace curve +{ + namespace client + { + + using curve::chunkserver::CHUNK_OP_STATUS; + using curve::chunkserver::ChunkRequest; + + using curve::client::MetaCache; + using curve::common::TimeUtility; + using ::testing::_; + using ::testing::AnyNumber; + using ::testing::AtLeast; + using ::testing::DoAll; + using ::testing::InSequence; + using ::testing::Invoke; + using ::testing::Return; + using ::testing::SaveArgPointee; + using ::testing::SetArgPointee; + using ::testing::SetArgReferee; + + class CopysetClientTest : public testing::Test + { + protected: + virtual void SetUp() + { + listenAddr_ = "127.0.0.1:9109"; + server_ = new brpc::Server(); + } + + virtual void TearDown() + { + server_->Stop(0); + server_->Join(); + delete server_; + server_ = nullptr; + } + + public: + std::string listenAddr_; + brpc::Server *server_; + }; + + /* TODO(wudemiao) current controller error cannot be returned through mock */ + int gWriteCntlFailedCode = 0; + int gReadCntlFailedCode = 0; + + static void WriteChunkFunc(::google::protobuf::RpcController *controller, + const ::curve::chunkserver::ChunkRequest *request, + ::curve::chunkserver::ChunkResponse *response, + google::protobuf::Closure *done) + { + /* return response */ + brpc::ClosureGuard doneGuard(done); + if (0 != gWriteCntlFailedCode) + { + if (gWriteCntlFailedCode == brpc::ERPCTIMEDOUT) + { + std::this_thread::sleep_for(std::chrono::milliseconds(3500)); + } + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(gWriteCntlFailedCode, "write controller error"); + } } - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(gWriteCntlFailedCode, "write controller error"); - } -} - -static void ReadChunkFunc(::google::protobuf::RpcController* controller, - const ::curve::chunkserver::ChunkRequest* request, - ::curve::chunkserver::ChunkResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - if (0 != gReadCntlFailedCode) { - if (gReadCntlFailedCode == brpc::ERPCTIMEDOUT) { - std::this_thread::sleep_for(std::chrono::milliseconds(4000)); + + static void ReadChunkFunc(::google::protobuf::RpcController *controller, + const ::curve::chunkserver::ChunkRequest *request, + ::curve::chunkserver::ChunkResponse *response, + google::protobuf::Closure *done) + { + brpc::ClosureGuard doneGuard(done); + if (0 != gReadCntlFailedCode) + { + if (gReadCntlFailedCode == brpc::ERPCTIMEDOUT) + { + std::this_thread::sleep_for(std::chrono::milliseconds(4000)); + } + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(gReadCntlFailedCode, "read controller error"); + } } - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(gReadCntlFailedCode, "read controller error"); - } -} - -static void ReadChunkSnapshotFunc( - ::google::protobuf::RpcController* controller, - const ::curve::chunkserver::ChunkRequest* request, // NOLINT - ::curve::chunkserver::ChunkResponse* response, // NOLINT - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - if (0 != gReadCntlFailedCode) { - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(-1, "read snapshot controller error"); - } -} - -static void DeleteChunkSnapshotFunc( - ::google::protobuf::RpcController* controller, // NOLINT - const ::curve::chunkserver::ChunkRequest* request, // NOLINT - ::curve::chunkserver::ChunkResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - if (0 != gReadCntlFailedCode) { - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(-1, "delete snapshot controller error"); - } -} - -static void CreateCloneChunkFunc( - ::google::protobuf::RpcController* controller, - const ::curve::chunkserver::ChunkRequest* request, - ::curve::chunkserver::ChunkResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - if (0 != gReadCntlFailedCode) { - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(-1, "create clone chunk controller error"); - } -} - -static void RecoverChunkFunc( - ::google::protobuf::RpcController* controller, // NOLINT - const ::curve::chunkserver::ChunkRequest* request, // NOLINT - ::curve::chunkserver::ChunkResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - if (0 != gReadCntlFailedCode) { - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(-1, "recover chunk controller error"); - } -} - -static void GetChunkInfoFunc( - ::google::protobuf::RpcController* controller, - const ::curve::chunkserver::GetChunkInfoRequest* request, // NOLINT - ::curve::chunkserver::GetChunkInfoResponse* response, // NOLINT - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - if (0 != gReadCntlFailedCode) { - brpc::Controller* cntl = dynamic_cast(controller); - cntl->SetFailed(-1, "get chunk info controller error"); - } -} - -TEST_F(CopysetClientTest, normal_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - - RequestScheduler scheduler; - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t fileId = 1; - uint64_t epoch = 1; - uint64_t sn = 1; - size_t len = 8; - char buff1[8 + 1]; - char buff2[8 + 1]; - memset(buff1, 'a', 8); - memset(buff2, 'a', 8); - buff1[8] = '\0'; - buff2[8] = '\0'; - off_t offset = 0; - - butil::IOBuf iobuf; - iobuf.append(buff1, sizeof(buff1) - 1); - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - iot.PrepareReadIOBuffers(1); - - // write success - for (int i = 0; i < 10; ++i) { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - - reqCtx->offset_ = i * 8; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = offset; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = offset; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - // read success - for (int i = 0; i < 10; ++i) { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = i * 8; - reqCtx->rawlength_ = len; - reqCtx->subIoIndex_ = 0; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = offset; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = offset; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } -} - -/** - * write error testing - */ -TEST_F(CopysetClientTest, write_error_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 1000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 5000; - ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 3500; - ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 3500000; - - RequestScheduleOption reqopt; - reqopt.ioSenderOpt = ioSenderOpt; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - - RequestScheduler scheduler; - scheduler.Init(reqopt, &mockMetaCache); - scheduler.Run(); - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t fileId = 1; - uint64_t epoch = 1; - size_t len = 8; - char buff1[8 + 1]; - char buff2[8 + 1]; - memset(buff1, 'a', 8); - memset(buff2, 'a', 8); - buff1[8] = '\0'; - buff2[8] = '\0'; - off_t offset = 0; - - butil::IOBuf iobuf; - iobuf.append(buff1, sizeof(buff1) - 1); - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - // The retry sleep time set in the configuration file is 5000, as there - // is no triggering of underlying index backoff, so there will be no - // sleep between retries - uint64_t start = TimeUtility::GetTimeofDayUs(); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gWriteCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(WriteChunkFunc)); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayUs(); - ASSERT_GT(end - start, 10000); - gWriteCntlFailedCode = 0; - } - /* controller set timeout */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - // The retry timeout set by the configuration file is 5000 because the - // chunkserver setting returns timeout Causing the triggering of an - // exponential backoff of the underlying timeout time, increasing the - // interval between each retry. Retrying three times is normal, only 3 * - // 1000 sleep is required But after increasing the index backoff, the - // timeout will increase to 1000+2000+2000=5000 Adding random factors, - // the three retry times should be greater than 7000 and less than 8000 - uint64_t start = TimeUtility::GetTimeofDayMs(); - - reqCtx->done_ = reqDone; - gWriteCntlFailedCode = brpc::ERPCTIMEDOUT; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(3)) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(WriteChunkFunc)); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(end - start, 3000); - ASSERT_LT(end - start, 6000); - std::this_thread::sleep_for(std::chrono::seconds(8)); - - gWriteCntlFailedCode = 0; - } - - /* controller set timeout */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - // The retry sleep time set in the configuration file is 5000 because - // the chunkserver setting returns timeout Causing triggering of - // low-level exponential backoff, increasing the interval between each - // retry. Retrying three times is normal, only 3 * 5000 sleep is - // required But after increasing the index retreat, the sleep interval - // will increase to 10000+20000=30000 Adding random factors, the three - // retry times should be greater than 29000 and less than 50000 - uint64_t start = TimeUtility::GetTimeofDayUs(); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, - reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayUs(); - ASSERT_GT(end - start, 28000); - ASSERT_LT(end - start, 2 * 50000); - gWriteCntlFailedCode = 0; - } - - /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* Not a leader, returning the correct leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - - ASSERT_EQ(1, fm.writeRPC.redirectQps.count.get_value()); - } - /* Not a leader, did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - // response1.set_redirect(leaderStr2); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, did not return a leader, refreshing the meta cache failed - */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - // response1.set_redirect(leaderStr2); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - FileMetric fm("test"); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - auto startTimeUs = curve::common::TimeUtility::GetTimeofDayUs(); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - auto elpased = - curve::common::TimeUtility::GetTimeofDayUs() - startTimeUs; - // chunkserverOPRetryIntervalUS = 5000 - // redirect sleep for 500us each time and retry a total of 2 times - // (chunkserverOPMaxRetry=3, returns if it is greater than or equal to, - // so only two retries were made) So the total time spent is greater - // than 1000us - ASSERT_GE(elpased, 1000); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - ASSERT_EQ(3, fm.writeRPC.redirectQps.count.get_value()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - // epoch too old - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_EPOCH_TOO_OLD); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(1) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_EPOCH_TOO_OLD, - reqDone->GetErrorCode()); - } - - scheduler.Fini(); -} - -/** - * write failed testing - */ -TEST_F(CopysetClientTest, write_failed_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 500; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 50; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 5000; - ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 1000; - ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 100000; - - RequestScheduleOption reqopt; - reqopt.ioSenderOpt = ioSenderOpt; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - - RequestScheduler scheduler; - scheduler.Init(reqopt, &mockMetaCache); - scheduler.Run(); - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t fileId = 1; - uint64_t epoch = 1; - size_t len = 8; - char buff1[8 + 1]; - char buff2[8 + 1]; - memset(buff1, 'a', 8); - memset(buff2, 'a', 8); - buff1[8] = '\0'; - buff2[8] = '\0'; - off_t offset = 0; - butil::IOBuf iobuf; - iobuf.append(buff1, sizeof(buff1) - 1); - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* controller set timeout */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - // The retry timeout set by the configuration file is 500 because the - // chunkserver setting returns timeout Causing the triggering of an - // exponential backoff of the underlying timeout time, increasing the - // interval between each retry. Retrying 50 times normally only requires - // a timeout of 49 * 500 But after increasing the index backoff, the - // timeout will increase to 49 * 1000=49000 - uint64_t start = TimeUtility::GetTimeofDayMs(); - - reqCtx->done_ = reqDone; - gWriteCntlFailedCode = brpc::ERPCTIMEDOUT; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(50)) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(50) - .WillRepeatedly(Invoke(WriteChunkFunc)); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(end - start, 25000); - ASSERT_LT(end - start, 55000); - std::this_thread::sleep_for(std::chrono::seconds(8)); - - gWriteCntlFailedCode = 0; - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - // The retry sleep time set in the configuration file is 5000us because - // the chunkserver setting returns timeout Causing triggering of - // low-level exponential backoff, increasing the interval between each - // retry. Retrying 50 times normally only requires 49 * 5000us of sleep - // But after increasing the index of retreat, the sleep interval will - // increase to 10000 + 20000 + 40000... ~= 4650000 - uint64_t start = TimeUtility::GetTimeofDayUs(); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(50) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(50) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, - reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayUs(); - ASSERT_GT(end - start, 250000); - ASSERT_LT(end - start, 4650000); - gWriteCntlFailedCode = 0; - } - scheduler.Fini(); -} - -/** - * read failed testing - */ -TEST_F(CopysetClientTest, read_failed_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 500; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 50; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 5000; - ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 1000; - ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 100000; - - RequestScheduleOption reqopt; - reqopt.ioSenderOpt = ioSenderOpt; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - - RequestScheduler scheduler; - scheduler.Init(reqopt, &mockMetaCache); - scheduler.Run(); - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t sn = 1; - size_t len = 8; - char buff1[8 + 1]; - char buff2[8 + 1]; - memset(buff1, 'a', 8); - memset(buff2, 'a', 8); - buff1[8] = '\0'; - buff2[8] = '\0'; - off_t offset = 0; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - iot.PrepareReadIOBuffers(1); - - /* controller set timeout */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - // The retry timeout set by the configuration file is 500 because the - // chunkserver setting returns timeout Causing the triggering of an - // exponential backoff of the underlying timeout time, increasing the - // interval between each retry. Retrying 50 times normally only requires - // 50 * 500 But after increasing the index retreat, the timeout will - // increase to 500+1000+2000...~=60000 - uint64_t start = TimeUtility::GetTimeofDayMs(); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = brpc::ERPCTIMEDOUT; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(50)) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(50) - .WillRepeatedly(Invoke(ReadChunkFunc)); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(end - start, 25000); - ASSERT_LT(end - start, 60000); - - std::this_thread::sleep_for(std::chrono::seconds(8)); - - gReadCntlFailedCode = 0; - } - - /* Set overload */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - // The retry sleep time set in the configuration file is 5000us because - // the chunkserver setting returns timeout Causing triggering of - // low-level exponential backoff, increasing the interval between each - // retry. Retrying 50 times is normal, only requiring 49 * 5000 sleep - // But after increasing the index of retreat, the sleep interval will - // increase to 10000 + 20000 + 40000 ... = 4650000 Adding random - // factors, the three retry times should be greater than 2900 and less - // than 5000 - uint64_t start = TimeUtility::GetTimeofDayUs(); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(50) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(50) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, - reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayUs(); - ASSERT_GT(end - start, 250000); - ASSERT_LT(end - start, 4650000); - } - scheduler.Fini(); -} - -/** - * read error testing - */ -TEST_F(CopysetClientTest, read_error_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 1000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 3500; - ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 3500000; - - RequestScheduleOption reqopt; - reqopt.ioSenderOpt = ioSenderOpt; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - - RequestScheduler scheduler; - scheduler.Init(reqopt, &mockMetaCache); - scheduler.Run(); - - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t sn = 1; - size_t len = 8; - char buff1[8 + 1]; - char buff2[8 + 1]; - memset(buff1, 'a', 8); - memset(buff2, 'a', 8); - buff1[8] = '\0'; - buff2[8] = '\0'; - off_t offset = 0; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - iot.PrepareReadIOBuffers(1); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* chunk not exist */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_CHUNK_NOTEXIST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(0, reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - // The retry sleep time set in the configuration file is 5000, as there - // is no triggering of underlying index backoff, so there will be no - // sleep between retries - uint64_t start = TimeUtility::GetTimeofDayUs(); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(3)) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(ReadChunkFunc)); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayUs(); - ASSERT_GT(end - start, 1000); - gReadCntlFailedCode = 0; - } - - /* controller set timeout */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - // The timeout configured in the settings file is 1000, but due to chunk - // server timeout, it triggers exponential backoff, increasing the - // interval between retries. In normal conditions, three retries would - // only require a sleep time of 3 * 1000. However, with the added - // exponential backoff, the timeout intervals will increase to 1000 + - // 2000 + 2000 = 5000. Considering the random factor, the total time for - // three retries should be greater than 7000 and less than 8000. - uint64_t start = TimeUtility::GetTimeofDayMs(); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = brpc::ERPCTIMEDOUT; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(3)) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(ReadChunkFunc)); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(end - start, 3000); - ASSERT_LT(end - start, 6000); - - std::this_thread::sleep_for(std::chrono::seconds(8)); - - gReadCntlFailedCode = 0; - } - - /* Set overload */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - // The retry sleep time set in the configuration file is 500, but due to - // chunkserver timeouts, it triggers exponential backoff, increasing the - // interval between retries. In normal conditions, three retries would - // only require a sleep time of 3 * 500. However, with the added - // exponential backoff, the sleep intervals will increase to 1000 + 2000 - // = 3000. Considering the random factor, the total time for three - // retries should be greater than 2900 and less than 5000. - uint64_t start = TimeUtility::GetTimeofDayUs(); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, - reqDone->GetErrorCode()); - - uint64_t end = TimeUtility::GetTimeofDayUs(); - ASSERT_GT(end - start, 2900); - ASSERT_LT(end - start, 3 * 5000); - } - - /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* Not a leader, returning the correct leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - // response1.set_redirect(leaderStr2); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - // response1.set_redirect(leaderStr2); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(-1))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->subIoIndex_ = 0; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); - copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - scheduler.Fini(); -} - -/** - * read snapshot error testing - */ -TEST_F(CopysetClientTest, read_snapshot_error_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - RequestScheduler scheduler; - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - size_t len = 8; - int sn = 1; - char buff1[8 + 1]; - char buff2[8 + 1]; - memset(buff1, 'a', 8); - memset(buff2, 'a', 8); - buff1[8] = '\0'; - buff2[8] = '\0'; - off_t offset = 0; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* chunk snapshot not exist */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_CHUNK_NOTEXIST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_CHUNK_NOTEXIST, - reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(ReadChunkSnapshotFunc)); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - gReadCntlFailedCode = 0; - } - /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* Not a leader, returning the correct leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(ReadChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(ReadChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(ReadChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(ReadChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::READ_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(ReadChunkSnapshotFunc))); - copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } -} - -/** - * delete snapshot error testing - */ -TEST_F(CopysetClientTest, delete_snapshot_error_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - RequestScheduler scheduler; - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t sn = 1; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(3) - .WillRepeatedly(Invoke(DeleteChunkSnapshotFunc)); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - gReadCntlFailedCode = 0; - } - /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* Not a leader, returning the correct leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(ReadChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(DeleteChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(-1))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(DeleteChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - ; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(2) - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(DeleteChunkSnapshotFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(-1))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(-1))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, - DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(DeleteChunkSnapshotFunc))); - copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, - reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } -} - -/** - * create clone chunk error testing - */ -TEST_F(CopysetClientTest, create_s3_clone_error_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - RequestScheduler scheduler; - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t sn = 1; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::CREATE_CLONE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->seq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(1) // NOLINT - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(3) // NOLINT - .WillRepeatedly(Invoke(CreateCloneChunkFunc)); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - gReadCntlFailedCode = 0; - } - // /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(3) // NOLINT - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* op success */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(2) // NOLINT - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(CreateCloneChunkFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(2) // NOLINT - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(CreateCloneChunkFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(3) // NOLINT - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(3) // NOLINT - .WillRepeatedly(DoAll(SetArgPointee<2>(response), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(2) // NOLINT - .WillOnce(DoAll(SetArgPointee<2>(response1), - Invoke(CreateCloneChunkFunc))) - .WillOnce(DoAll(SetArgPointee<2>(response2), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) - .Times(1) // NOLINT - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(CreateCloneChunkFunc))); - copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, - 1024, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } -} - -/** - * recover chunk error testing - */ -TEST_F(CopysetClientTest, recover_chunk_error_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - RequestScheduler scheduler; - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t sn = 1; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - gReadCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(RecoverChunkFunc)); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - gReadCntlFailedCode = 0; - } - /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* Not a leader, returning the correct leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(RecoverChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(RecoverChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - - reqCtx->done_ = reqDone; - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(RecoverChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::DELETE_SNAP; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - reqCtx->correctedSeq_ = sn; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - ChunkResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(Return(-1)) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); - copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } -} - -/** - * get chunk info error testing - */ -TEST_F(CopysetClientTest, get_chunk_info_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - RequestScheduler scheduler; - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - - ChunkServerID leaderId = 10000; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); - - /* Illegal parameter */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, - reqDone->GetErrorCode()); - } - /* controller error */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - gReadCntlFailedCode = -1; - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(3) - .WillRepeatedly(Invoke(GetChunkInfoFunc)); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_NE(0, reqDone->GetErrorCode()); - gReadCntlFailedCode = 0; - } - /* Other errors */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, - reqDone->GetErrorCode()); - } - /* Not a leader, returning the correct leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - GetChunkInfoResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - GetChunkInfoResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but did not return a leader, refreshing the meta cache - * failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - GetChunkInfoResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(Return(-1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - /* Not a leader, but returned an incorrect leader */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(3) - .WillRepeatedly(Return(0)); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader still failed */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(6) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(3) - .WillRepeatedly( - DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, - reqDone->GetErrorCode()); - } - /* copyset does not exist, updating leader succeeded */ - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); - response1.set_redirect(leaderStr); - GetChunkInfoResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response2.set_redirect(leaderStr); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - { - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::GET_CHUNK_INFO; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - GetChunkInfoResponse response; - response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(AtLeast(1)) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(-1))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(-1))) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) - .Times(1) - .WillOnce( - DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); - copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); - cond.Wait(); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } -} - -namespace { - -bool gWriteSuccessFlag = false; - -void WriteCallBack(CurveAioContext* aioctx) { - gWriteSuccessFlag = true; - delete aioctx; -} - -void PrepareOpenFile(FakeCurveFSService* service, OpenFileResponse* openresp, - FakeReturn* fakeReturn) { - openresp->set_statuscode(curve::mds::StatusCode::kOK); - auto* session = openresp->mutable_protosession(); - session->set_sessionid("xxx"); - session->set_leasetime(10000); - session->set_createtime(10000); - session->set_sessionstatus(curve::mds::SessionStatus::kSessionOK); - auto* fileinfo = openresp->mutable_fileinfo(); - fileinfo->set_id(1); - fileinfo->set_filename("filename"); - fileinfo->set_parentid(0); - fileinfo->set_length(10ULL * 1024 * 1024 * 1024); - fileinfo->set_blocksize(4096); - - *fakeReturn = FakeReturn(nullptr, static_cast(openresp)); - - service->SetOpenFile(fakeReturn); -} - -} // namespace - -TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) { - const std::string endpoint = "127.0.0.1:9102"; - - ClientConfig cc; - const std::string& configPath = "./conf/client.conf"; - cc.Init(configPath.c_str()); - FileInstance fileinstance; - UserInfo userinfo; - userinfo.owner = "userinfo"; - - std::shared_ptr mdsclient = std::make_shared(); - - // set mds addr - auto mdsopts = cc.GetFileServiceOption().metaServerOpt; - mdsopts.rpcRetryOpt.addrs.clear(); - mdsopts.rpcRetryOpt.addrs.push_back(endpoint); - - ASSERT_EQ(LIBCURVE_ERROR::OK, mdsclient->Initialize(mdsopts)); - ASSERT_TRUE(fileinstance.Initialize( - "/test", mdsclient, userinfo, OpenFlags{}, cc.GetFileServiceOption())); - - // create fake chunkserver service - FakeChunkServerService fakechunkservice; - // Set up cli service - CliServiceFake fakeCliservice; - - FakeCurveFSService curvefsService; - OpenFileResponse openresp; - FakeReturn fakeReturn; - - PrepareOpenFile(&curvefsService, &openresp, &fakeReturn); - - brpc::Server server; - ASSERT_EQ(0, server.AddService(&fakechunkservice, - brpc::SERVER_DOESNT_OWN_SERVICE)) - << "Fail to add fakechunkservice"; - ASSERT_EQ( - 0, server.AddService(&fakeCliservice, brpc::SERVER_DOESNT_OWN_SERVICE)) - << "Fail to add fakecliservice"; - ASSERT_EQ( - 0, server.AddService(&curvefsService, brpc::SERVER_DOESNT_OWN_SERVICE)) - << "Fail to add curvefsService"; - - ASSERT_EQ(0, server.Start(endpoint.c_str(), nullptr)) - << "Fail to start server at " << endpoint; - - // fill metacache - curve::client::MetaCache* mc = - fileinstance.GetIOManager4File()->GetMetaCache(); - curve::client::ChunkIDInfo_t chunkinfo(1, 2, 3); - mc->UpdateChunkInfoByIndex(0, chunkinfo); - curve::client::CopysetInfo cpinfo; - curve::client::EndPoint ep; - butil::str2endpoint("127.0.0.1", 9102, &ep); - - braft::PeerId pd(ep); - curve::client::PeerAddr addr = curve::client::PeerAddr(ep); - curve::client::CopysetPeerInfo peer(1, addr, addr); - cpinfo.csinfos_.push_back(peer); - mc->UpdateCopysetInfo(2, 3, cpinfo); - - fakeCliservice.SetPeerID(pd); - - curve::chunkserver::ChunkResponse response; - response.set_status( - curve::chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - response.set_appliedindex(0); - FakeReturn writeFakeRet(nullptr, static_cast(&response)); - fakechunkservice.SetFakeWriteReturn(&writeFakeRet); - - const int kNewFileSn = 100; - const int kOldFileSn = 30; - - ASSERT_EQ(LIBCURVE_ERROR::OK, fileinstance.Open()); - - // Set file version number - fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); - - // Send a write request and wait for seconds to check if IO returns - auto startWriteAndCheckResult = - [&fileinstance](int sec) -> bool { // NOLINT - CurveAioContext* aioctx = new CurveAioContext(); - char buffer[4096]; - - aioctx->buf = buffer; - aioctx->offset = 0; - aioctx->length = sizeof(buffer); - aioctx->op = LIBCURVE_OP::LIBCURVE_OP_WRITE; - aioctx->cb = WriteCallBack; - - // Send write request - fileinstance.AioWrite(aioctx, UserDataType::RawBuffer); - - std::this_thread::sleep_for(std::chrono::seconds(sec)); - return gWriteSuccessFlag; - }; - - // Successfully written for the first time and updated the file version - // number on the chunkserver side - ASSERT_TRUE(startWriteAndCheckResult(3)); - - // Set an old version number to write - fileinstance.GetIOManager4File()->SetLatestFileSn(kOldFileSn); - gWriteSuccessFlag = false; - - // chunkserver returns the feedback, and after obtaining the version number - // again, it is still the old version IO hang - ASSERT_FALSE(startWriteAndCheckResult(3)); - - // Update version number to normal state - fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); - std::this_thread::sleep_for(std::chrono::seconds(1)); - - // Last write request successful - ASSERT_EQ(true, gWriteSuccessFlag); - - server.Stop(0); - server.Join(); -} - -TEST_F(CopysetClientTest, retry_rpc_sleep_test) { - MockChunkServiceImpl mockChunkService; - ASSERT_EQ( - server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); - - const uint64_t sleepUsBeforeRetry = 5 * 1000 * 1000; - - IOSenderOption ioSenderOpt; - ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 1000; - ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; - ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = - sleepUsBeforeRetry; - ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 3500; - ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 3500000; - - RequestScheduleOption reqopt; - reqopt.ioSenderOpt = ioSenderOpt; - - CopysetClient copysetClient; - MockMetaCache mockMetaCache; - mockMetaCache.DelegateToFake(); - - RequestScheduler scheduler; - scheduler.Init(reqopt, &mockMetaCache); - scheduler.Run(); - copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); - - LogicPoolID logicPoolId = 1; - CopysetID copysetId = 100001; - ChunkID chunkId = 1; - uint64_t fileId = 1; - uint64_t epoch = 1; - size_t len = 8; - char buff1[8] = {0}; - butil::IOBuf iobuf; - iobuf.append(buff1, sizeof(len)); - off_t offset = 0; - - ChunkServerID leaderId = 10000; - ChunkServerID leaderId2 = 10001; - butil::EndPoint leaderAddr; - std::string leaderStr = "127.0.0.1:9109"; - butil::str2endpoint(leaderStr.c_str(), &leaderAddr); - - FileMetric fm("test"); - IOTracker iot(nullptr, nullptr, nullptr, &fm); + static void ReadChunkSnapshotFunc( + ::google::protobuf::RpcController *controller, + const ::curve::chunkserver::ChunkRequest *request, // NOLINT + ::curve::chunkserver::ChunkResponse *response, // NOLINT + google::protobuf::Closure *done) + { + brpc::ClosureGuard doneGuard(done); + if (0 != gReadCntlFailedCode) + { + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(-1, "read snapshot controller error"); + } + } - { - // In the redirect case, chunkserver returns a new leader - // Will not sleep until retry - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - // reqCtx->writeBuffer_ = buff1; - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId2), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - - auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); - - // Returns a new leader ID, so there will be no sleep before retrying - ASSERT_LE(endUs - startUs, sleepUsBeforeRetry / 10); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } + static void DeleteChunkSnapshotFunc( + ::google::protobuf::RpcController *controller, // NOLINT + const ::curve::chunkserver::ChunkRequest *request, // NOLINT + ::curve::chunkserver::ChunkResponse *response, + google::protobuf::Closure *done) + { + brpc::ClosureGuard doneGuard(done); + if (0 != gReadCntlFailedCode) + { + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(-1, "delete snapshot controller error"); + } + } - { - // In the redirect case, chunkserver returns the old leader - // Sleep before retrying - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - // reqCtx->writeBuffer_ = buff1; - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - response1.set_redirect(leaderStr); - - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) - .Times(1) - .WillOnce(Return(0)); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); - - // Return the same leader ID and sleep before retrying - ASSERT_GE(endUs - startUs, sleepUsBeforeRetry / 10); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } + static void CreateCloneChunkFunc( + ::google::protobuf::RpcController *controller, + const ::curve::chunkserver::ChunkRequest *request, + ::curve::chunkserver::ChunkResponse *response, + google::protobuf::Closure *done) + { + brpc::ClosureGuard doneGuard(done); + if (0 != gReadCntlFailedCode) + { + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(-1, "create clone chunk controller error"); + } + } - { - // In the redirect case, chunkserver did not return a leader - // Actively refresh to obtain a new leader - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - // reqCtx->writeBuffer_ = buff1; - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillOnce(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId2), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)).Times(0); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); - - // Returns a new leader id, so there will be no sleep before retrying - ASSERT_LE(endUs - startUs, sleepUsBeforeRetry / 10); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } + static void RecoverChunkFunc( + ::google::protobuf::RpcController *controller, // NOLINT + const ::curve::chunkserver::ChunkRequest *request, // NOLINT + ::curve::chunkserver::ChunkResponse *response, + google::protobuf::Closure *done) + { + brpc::ClosureGuard doneGuard(done); + if (0 != gReadCntlFailedCode) + { + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(-1, "recover chunk controller error"); + } + } - { - // In the redirect case, chunkserver did not return a leader - // Actively refresh to obtain old leader - RequestContext* reqCtx = new FakeRequestContext(); - reqCtx->optype_ = OpType::WRITE; - reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); - // reqCtx->writeBuffer_ = buff1; - reqCtx->writeData_ = iobuf; - reqCtx->offset_ = 0; - reqCtx->rawlength_ = len; - - curve::common::CountDownEvent cond(1); - RequestClosure* reqDone = new FakeRequestClosure(&cond, reqCtx); - reqDone->SetFileMetric(&fm); - reqDone->SetIOTracker(&iot); - reqCtx->done_ = reqDone; - - ChunkResponse response1; - response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); - - ChunkResponse response2; - response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); - - EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) - .Times(3) - .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), - SetArgPointee<3>(leaderAddr), Return(0))); - EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)).Times(0); - EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) - .Times(2) - .WillOnce( - DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) - .WillOnce( - DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); - auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); - copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, - offset, len, {}, reqDone); - cond.Wait(); - auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); - - ASSERT_GE(endUs - startUs, sleepUsBeforeRetry / 10); - ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, - reqDone->GetErrorCode()); - } - scheduler.Fini(); -} - -class TestRunnedRequestClosure : public RequestClosure { - public: - TestRunnedRequestClosure() : RequestClosure(nullptr) {} - - void Run() override { runned_ = true; } - - bool IsRunned() const { return runned_; } - - private: - bool runned_ = false; -}; - -// After the test session fails, the retry request will be placed back in the -// request queue -TEST(CopysetClientBasicTest, TestReScheduleWhenSessionNotValid) { - MockRequestScheduler requestScheduler; - CopysetClient copysetClient; - IOSenderOption ioSenderOption; - MetaCache metaCache; - - ASSERT_EQ(0, copysetClient.Init(&metaCache, ioSenderOption, - &requestScheduler, nullptr)); - - // Set session not valid - copysetClient.StartRecycleRetryRPC(); + static void GetChunkInfoFunc( + ::google::protobuf::RpcController *controller, + const ::curve::chunkserver::GetChunkInfoRequest *request, // NOLINT + ::curve::chunkserver::GetChunkInfoResponse *response, // NOLINT + google::protobuf::Closure *done) + { + brpc::ClosureGuard doneGuard(done); + if (0 != gReadCntlFailedCode) + { + brpc::Controller *cntl = dynamic_cast(controller); + cntl->SetFailed(-1, "get chunk info controller error"); + } + } - { - EXPECT_CALL(requestScheduler, ReSchedule(_)).Times(1); + TEST_F(CopysetClientTest, normal_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + + RequestScheduler scheduler; + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t fileId = 1; + uint64_t epoch = 1; + uint64_t sn = 1; + size_t len = 8; + char buff1[8 + 1]; + char buff2[8 + 1]; + memset(buff1, 'a', 8); + memset(buff2, 'a', 8); + buff1[8] = '\0'; + buff2[8] = '\0'; + off_t offset = 0; + + butil::IOBuf iobuf; + iobuf.append(buff1, sizeof(buff1) - 1); + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + iot.PrepareReadIOBuffers(1); + + // write success + for (int i = 0; i < 10; ++i) + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + + reqCtx->offset_ = i * 8; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = offset; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = offset; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + // read success + for (int i = 0; i < 10; ++i) + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = i * 8; + reqCtx->rawlength_ = len; + reqCtx->subIoIndex_ = 0; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = offset; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = offset; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + } - TestRunnedRequestClosure closure; - copysetClient.ReadChunk({}, 0, 0, 0, {}, &closure); - ASSERT_FALSE(closure.IsRunned()); - } + /** + * write error testing + */ + TEST_F(CopysetClientTest, write_error_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 1000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 5000; + ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 3500; + ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 3500000; + + RequestScheduleOption reqopt; + reqopt.ioSenderOpt = ioSenderOpt; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + + RequestScheduler scheduler; + scheduler.Init(reqopt, &mockMetaCache); + scheduler.Run(); + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t fileId = 1; + uint64_t epoch = 1; + size_t len = 8; + char buff1[8 + 1]; + char buff2[8 + 1]; + memset(buff1, 'a', 8); + memset(buff2, 'a', 8); + buff1[8] = '\0'; + buff2[8] = '\0'; + off_t offset = 0; + + butil::IOBuf iobuf; + iobuf.append(buff1, sizeof(buff1) - 1); + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + // The retry sleep time set in the configuration file is 5000, as there + // is no triggering of underlying index backoff, so there will be no + // sleep between retries + uint64_t start = TimeUtility::GetTimeofDayUs(); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gWriteCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(WriteChunkFunc)); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayUs(); + ASSERT_GT(end - start, 10000); + gWriteCntlFailedCode = 0; + } + /* controller set timeout */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + // The retry timeout set by the configuration file is 5000 because the + // chunkserver setting returns timeout Causing the triggering of an + // exponential backoff of the underlying timeout time, increasing the + // interval between each retry. Retrying three times is normal, only 3 * + // 1000 sleep is required But after increasing the index backoff, the + // timeout will increase to 1000+2000+2000=5000 Adding random factors, + // the three retry times should be greater than 7000 and less than 8000 + uint64_t start = TimeUtility::GetTimeofDayMs(); + + reqCtx->done_ = reqDone; + gWriteCntlFailedCode = brpc::ERPCTIMEDOUT; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(3)) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(WriteChunkFunc)); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(end - start, 3000); + ASSERT_LT(end - start, 6000); + std::this_thread::sleep_for(std::chrono::seconds(8)); + + gWriteCntlFailedCode = 0; + } + + /* controller set timeout */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + // The retry sleep time set in the configuration file is 5000 because + // the chunkserver setting returns timeout Causing triggering of + // low-level exponential backoff, increasing the interval between each + // retry. Retrying three times is normal, only 3 * 5000 sleep is + // required But after increasing the index retreat, the sleep interval + // will increase to 10000+20000=30000 Adding random factors, the three + // retry times should be greater than 29000 and less than 50000 + uint64_t start = TimeUtility::GetTimeofDayUs(); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, + reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayUs(); + ASSERT_GT(end - start, 28000); + ASSERT_LT(end - start, 2 * 50000); + gWriteCntlFailedCode = 0; + } + + /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* Not a leader, returning the correct leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + + ASSERT_EQ(1, fm.writeRPC.redirectQps.count.get_value()); + } + /* Not a leader, did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + // response1.set_redirect(leaderStr2); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, did not return a leader, refreshing the meta cache failed + */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + // response1.set_redirect(leaderStr2); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + FileMetric fm("test"); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + auto startTimeUs = curve::common::TimeUtility::GetTimeofDayUs(); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + auto elpased = + curve::common::TimeUtility::GetTimeofDayUs() - startTimeUs; + // chunkserverOPRetryIntervalUS = 5000 + // redirect sleep for 500us each time and retry a total of 2 times + // (chunkserverOPMaxRetry=3, returns if it is greater than or equal to, + // so only two retries were made) So the total time spent is greater + // than 1000us + ASSERT_GE(elpased, 1000); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + ASSERT_EQ(3, fm.writeRPC.redirectQps.count.get_value()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + // epoch too old + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_EPOCH_TOO_OLD); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(1) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_EPOCH_TOO_OLD, + reqDone->GetErrorCode()); + } + + scheduler.Fini(); + } - { - EXPECT_CALL(requestScheduler, ReSchedule(_)).Times(1); + /** + * write failed testing + */ + TEST_F(CopysetClientTest, write_failed_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 500; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 50; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 5000; + ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 1000; + ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 100000; + + RequestScheduleOption reqopt; + reqopt.ioSenderOpt = ioSenderOpt; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + + RequestScheduler scheduler; + scheduler.Init(reqopt, &mockMetaCache); + scheduler.Run(); + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t fileId = 1; + uint64_t epoch = 1; + size_t len = 8; + char buff1[8 + 1]; + char buff2[8 + 1]; + memset(buff1, 'a', 8); + memset(buff2, 'a', 8); + buff1[8] = '\0'; + buff2[8] = '\0'; + off_t offset = 0; + butil::IOBuf iobuf; + iobuf.append(buff1, sizeof(buff1) - 1); + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* controller set timeout */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + // The retry timeout set by the configuration file is 500 because the + // chunkserver setting returns timeout Causing the triggering of an + // exponential backoff of the underlying timeout time, increasing the + // interval between each retry. Retrying 50 times normally only requires + // a timeout of 49 * 500 But after increasing the index backoff, the + // timeout will increase to 49 * 1000=49000 + uint64_t start = TimeUtility::GetTimeofDayMs(); + + reqCtx->done_ = reqDone; + gWriteCntlFailedCode = brpc::ERPCTIMEDOUT; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(50)) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(50) + .WillRepeatedly(Invoke(WriteChunkFunc)); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(end - start, 25000); + ASSERT_LT(end - start, 55000); + std::this_thread::sleep_for(std::chrono::seconds(8)); + + gWriteCntlFailedCode = 0; + } + + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + // The retry sleep time set in the configuration file is 5000us because + // the chunkserver setting returns timeout Causing triggering of + // low-level exponential backoff, increasing the interval between each + // retry. Retrying 50 times normally only requires 49 * 5000us of sleep + // But after increasing the index of retreat, the sleep interval will + // increase to 10000 + 20000 + 40000... ~= 4650000 + uint64_t start = TimeUtility::GetTimeofDayUs(); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(50) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(50) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(WriteChunkFunc))); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, + reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayUs(); + ASSERT_GT(end - start, 250000); + ASSERT_LT(end - start, 4650000); + gWriteCntlFailedCode = 0; + } + scheduler.Fini(); + } + + /** + * read failed testing + */ + TEST_F(CopysetClientTest, read_failed_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 500; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 50; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 5000; + ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 1000; + ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 100000; + + RequestScheduleOption reqopt; + reqopt.ioSenderOpt = ioSenderOpt; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + + RequestScheduler scheduler; + scheduler.Init(reqopt, &mockMetaCache); + scheduler.Run(); + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t sn = 1; + size_t len = 8; + char buff1[8 + 1]; + char buff2[8 + 1]; + memset(buff1, 'a', 8); + memset(buff2, 'a', 8); + buff1[8] = '\0'; + buff2[8] = '\0'; + off_t offset = 0; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + iot.PrepareReadIOBuffers(1); + + /* controller set timeout */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + // The retry timeout set by the configuration file is 500 because the + // chunkserver setting returns timeout Causing the triggering of an + // exponential backoff of the underlying timeout time, increasing the + // interval between each retry. Retrying 50 times normally only requires + // 50 * 500 But after increasing the index retreat, the timeout will + // increase to 500+1000+2000...~=60000 + uint64_t start = TimeUtility::GetTimeofDayMs(); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = brpc::ERPCTIMEDOUT; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(50)) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(50) + .WillRepeatedly(Invoke(ReadChunkFunc)); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(end - start, 25000); + ASSERT_LT(end - start, 60000); + + std::this_thread::sleep_for(std::chrono::seconds(8)); + + gReadCntlFailedCode = 0; + } + + /* Set overload */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + // The retry sleep time set in the configuration file is 5000us because + // the chunkserver setting returns timeout Causing triggering of + // low-level exponential backoff, increasing the interval between each + // retry. Retrying 50 times is normal, only requiring 49 * 5000 sleep + // But after increasing the index of retreat, the sleep interval will + // increase to 10000 + 20000 + 40000 ... = 4650000 Adding random + // factors, the three retry times should be greater than 2900 and less + // than 5000 + uint64_t start = TimeUtility::GetTimeofDayUs(); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(50) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(50) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, + reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayUs(); + ASSERT_GT(end - start, 250000); + ASSERT_LT(end - start, 4650000); + } + scheduler.Fini(); + } + + /** + * read error testing + */ + TEST_F(CopysetClientTest, read_error_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 1000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 3500; + ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 3500000; + + RequestScheduleOption reqopt; + reqopt.ioSenderOpt = ioSenderOpt; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + + RequestScheduler scheduler; + scheduler.Init(reqopt, &mockMetaCache); + scheduler.Run(); + + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t sn = 1; + size_t len = 8; + char buff1[8 + 1]; + char buff2[8 + 1]; + memset(buff1, 'a', 8); + memset(buff2, 'a', 8); + buff1[8] = '\0'; + buff2[8] = '\0'; + off_t offset = 0; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + iot.PrepareReadIOBuffers(1); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* chunk not exist */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_CHUNK_NOTEXIST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(0, reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + // The retry sleep time set in the configuration file is 5000, as there + // is no triggering of underlying index backoff, so there will be no + // sleep between retries + uint64_t start = TimeUtility::GetTimeofDayUs(); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(3)) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(ReadChunkFunc)); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayUs(); + ASSERT_GT(end - start, 1000); + gReadCntlFailedCode = 0; + } + + /* controller set timeout */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + // The timeout configured in the settings file is 1000, but due to chunk + // server timeout, it triggers exponential backoff, increasing the + // interval between retries. In normal conditions, three retries would + // only require a sleep time of 3 * 1000. However, with the added + // exponential backoff, the timeout intervals will increase to 1000 + + // 2000 + 2000 = 5000. Considering the random factor, the total time for + // three retries should be greater than 7000 and less than 8000. + uint64_t start = TimeUtility::GetTimeofDayMs(); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = brpc::ERPCTIMEDOUT; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(3)) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(ReadChunkFunc)); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(end - start, 3000); + ASSERT_LT(end - start, 6000); + + std::this_thread::sleep_for(std::chrono::seconds(8)); + + gReadCntlFailedCode = 0; + } + + /* Set overload */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + // The retry sleep time set in the configuration file is 500, but due to + // chunkserver timeouts, it triggers exponential backoff, increasing the + // interval between retries. In normal conditions, three retries would + // only require a sleep time of 3 * 500. However, with the added + // exponential backoff, the sleep intervals will increase to 1000 + 2000 + // = 3000. Considering the random factor, the total time for three + // retries should be greater than 2900 and less than 5000. + uint64_t start = TimeUtility::GetTimeofDayUs(); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_OVERLOAD, + reqDone->GetErrorCode()); + + uint64_t end = TimeUtility::GetTimeofDayUs(); + ASSERT_GT(end - start, 2900); + ASSERT_LT(end - start, 3 * 5000); + } + + /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* Not a leader, returning the correct leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + // response1.set_redirect(leaderStr2); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + // response1.set_redirect(leaderStr2); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(-1))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->subIoIndex_ = 0; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunk(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), Invoke(ReadChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(ReadChunkFunc))); + copysetClient.ReadChunk(reqCtx->idinfo_, sn, offset, len, {}, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + scheduler.Fini(); + } + + /** + * read snapshot error testing + */ + TEST_F(CopysetClientTest, read_snapshot_error_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + RequestScheduler scheduler; + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + size_t len = 8; + int sn = 1; + char buff1[8 + 1]; + char buff2[8 + 1]; + memset(buff1, 'a', 8); + memset(buff2, 'a', 8); + buff1[8] = '\0'; + buff2[8] = '\0'; + off_t offset = 0; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* chunk snapshot not exist */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_CHUNK_NOTEXIST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_CHUNK_NOTEXIST, + reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(ReadChunkSnapshotFunc)); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + gReadCntlFailedCode = 0; + } + /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* Not a leader, returning the correct leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(ReadChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(ReadChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(ReadChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(ReadChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::READ_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, ReadChunkSnapshot(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(ReadChunkSnapshotFunc))); + copysetClient.ReadChunkSnapshot(reqCtx->idinfo_, sn, offset, len, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + } + + /** + * delete snapshot error testing + */ + TEST_F(CopysetClientTest, delete_snapshot_error_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + RequestScheduler scheduler; + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t sn = 1; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(3) + .WillRepeatedly(Invoke(DeleteChunkSnapshotFunc)); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + gReadCntlFailedCode = 0; + } + /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* Not a leader, returning the correct leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(ReadChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(DeleteChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(-1))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(DeleteChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + ; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(2) + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(DeleteChunkSnapshotFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(-1))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(-1))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, + DeleteChunkSnapshotOrCorrectSn(_, _, _, _)) // NOLINT + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(DeleteChunkSnapshotFunc))); + copysetClient.DeleteChunkSnapshotOrCorrectSn(reqCtx->idinfo_, sn, + reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + } + + /** + * create clone chunk error testing + */ + TEST_F(CopysetClientTest, create_s3_clone_error_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + RequestScheduler scheduler; + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t sn = 1; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::CREATE_CLONE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->seq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(1) // NOLINT + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(3) // NOLINT + .WillRepeatedly(Invoke(CreateCloneChunkFunc)); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + gReadCntlFailedCode = 0; + } + // /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(3) // NOLINT + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* op success */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(2) // NOLINT + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(CreateCloneChunkFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(2) // NOLINT + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(CreateCloneChunkFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(3) // NOLINT + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(3) // NOLINT + .WillRepeatedly(DoAll(SetArgPointee<2>(response), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(2) // NOLINT + .WillOnce(DoAll(SetArgPointee<2>(response1), + Invoke(CreateCloneChunkFunc))) + .WillOnce(DoAll(SetArgPointee<2>(response2), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, CreateCloneChunk(_, _, _, _)) + .Times(1) // NOLINT + .WillOnce(DoAll(SetArgPointee<2>(response), + Invoke(CreateCloneChunkFunc))); + copysetClient.CreateCloneChunk(reqCtx->idinfo_, "destination", sn, 1, + 1024, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + } + + /** + * recover chunk error testing + */ + TEST_F(CopysetClientTest, recover_chunk_error_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + RequestScheduler scheduler; + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t sn = 1; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + gReadCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(RecoverChunkFunc)); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + gReadCntlFailedCode = 0; + } + /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* Not a leader, returning the correct leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(RecoverChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(RecoverChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + + reqCtx->done_ = reqDone; + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(RecoverChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::DELETE_SNAP; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + reqCtx->correctedSeq_ = sn; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + ChunkResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(-1)) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, RecoverChunk(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(RecoverChunkFunc))); + copysetClient.RecoverChunk(reqCtx->idinfo_, 0, 4096, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + } - TestRunnedRequestClosure closure; - copysetClient.WriteChunk({}, 1, 1, 0, {}, 0, 0, {}, &closure); - ASSERT_FALSE(closure.IsRunned()); - } -} + /** + * get chunk info error testing + */ + TEST_F(CopysetClientTest, get_chunk_info_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 5000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = 500; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + RequestScheduler scheduler; + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + + ChunkServerID leaderId = 10000; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + /* Illegal parameter */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST, + reqDone->GetErrorCode()); + } + /* controller error */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + gReadCntlFailedCode = -1; + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(3) + .WillRepeatedly(Invoke(GetChunkInfoFunc)); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_NE(0, reqDone->GetErrorCode()); + gReadCntlFailedCode = 0; + } + /* Other errors */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_FAILURE_UNKNOWN, + reqDone->GetErrorCode()); + } + /* Not a leader, returning the correct leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + GetChunkInfoResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + GetChunkInfoResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but did not return a leader, refreshing the meta cache + * failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + GetChunkInfoResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(Return(-1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + /* Not a leader, but returned an incorrect leader */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader still failed */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(6) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST, + reqDone->GetErrorCode()); + } + /* copyset does not exist, updating leader succeeded */ + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_COPYSET_NOTEXIST); + response1.set_redirect(leaderStr); + GetChunkInfoResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response2.set_redirect(leaderStr); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(GetChunkInfoFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + { + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::GET_CHUNK_INFO; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + GetChunkInfoResponse response; + response.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(-1))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(-1))) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockChunkService, GetChunkInfo(_, _, _, _)) + .Times(1) + .WillOnce( + DoAll(SetArgPointee<2>(response), Invoke(GetChunkInfoFunc))); + copysetClient.GetChunkInfo(reqCtx->idinfo_, reqDone); + cond.Wait(); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + } + + namespace + { + + bool gWriteSuccessFlag = false; + + void WriteCallBack(CurveAioContext *aioctx) + { + gWriteSuccessFlag = true; + delete aioctx; + } + + void PrepareOpenFile(FakeCurveFSService *service, OpenFileResponse *openresp, + FakeReturn *fakeReturn) + { + openresp->set_statuscode(curve::mds::StatusCode::kOK); + auto *session = openresp->mutable_protosession(); + session->set_sessionid("xxx"); + session->set_leasetime(10000); + session->set_createtime(10000); + session->set_sessionstatus(curve::mds::SessionStatus::kSessionOK); + auto *fileinfo = openresp->mutable_fileinfo(); + fileinfo->set_id(1); + fileinfo->set_filename("filename"); + fileinfo->set_parentid(0); + fileinfo->set_length(10ULL * 1024 * 1024 * 1024); + fileinfo->set_blocksize(4096); + + *fakeReturn = FakeReturn(nullptr, static_cast(openresp)); + + service->SetOpenFile(fakeReturn); + } + + } // namespace + + TEST(ChunkServerBackwardTest, ChunkServerBackwardTest) + { + const std::string endpoint = "127.0.0.1:9102"; + + ClientConfig cc; + const std::string &configPath = "./conf/client.conf"; + cc.Init(configPath.c_str()); + FileInstance fileinstance; + UserInfo userinfo; + userinfo.owner = "userinfo"; + + std::shared_ptr mdsclient = std::make_shared(); + + // set mds addr + auto mdsopts = cc.GetFileServiceOption().metaServerOpt; + mdsopts.rpcRetryOpt.addrs.clear(); + mdsopts.rpcRetryOpt.addrs.push_back(endpoint); + + ASSERT_EQ(LIBCURVE_ERROR::OK, mdsclient->Initialize(mdsopts)); + ASSERT_TRUE(fileinstance.Initialize( + "/test", mdsclient, userinfo, OpenFlags{}, cc.GetFileServiceOption())); + + // create fake chunkserver service + FakeChunkServerService fakechunkservice; + // Set up cli service + CliServiceFake fakeCliservice; + + FakeCurveFSService curvefsService; + OpenFileResponse openresp; + FakeReturn fakeReturn; + + PrepareOpenFile(&curvefsService, &openresp, &fakeReturn); + + brpc::Server server; + ASSERT_EQ(0, server.AddService(&fakechunkservice, + brpc::SERVER_DOESNT_OWN_SERVICE)) + << "Fail to add fakechunkservice"; + ASSERT_EQ( + 0, server.AddService(&fakeCliservice, brpc::SERVER_DOESNT_OWN_SERVICE)) + << "Fail to add fakecliservice"; + ASSERT_EQ( + 0, server.AddService(&curvefsService, brpc::SERVER_DOESNT_OWN_SERVICE)) + << "Fail to add curvefsService"; + + ASSERT_EQ(0, server.Start(endpoint.c_str(), nullptr)) + << "Fail to start server at " << endpoint; + + // fill metacache + curve::client::MetaCache *mc = + fileinstance.GetIOManager4File()->GetMetaCache(); + curve::client::ChunkIDInfo_t chunkinfo(1, 2, 3); + mc->UpdateChunkInfoByIndex(0, chunkinfo); + curve::client::CopysetInfo cpinfo; + curve::client::EndPoint ep; + butil::str2endpoint("127.0.0.1", 9102, &ep); + + braft::PeerId pd(ep); + curve::client::PeerAddr addr = curve::client::PeerAddr(ep); + curve::client::CopysetPeerInfo peer(1, addr, addr); + cpinfo.csinfos_.push_back(peer); + mc->UpdateCopysetInfo(2, 3, cpinfo); + + fakeCliservice.SetPeerID(pd); + + curve::chunkserver::ChunkResponse response; + response.set_status( + curve::chunkserver::CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + response.set_appliedindex(0); + FakeReturn writeFakeRet(nullptr, static_cast(&response)); + fakechunkservice.SetFakeWriteReturn(&writeFakeRet); + + const int kNewFileSn = 100; + const int kOldFileSn = 30; + + ASSERT_EQ(LIBCURVE_ERROR::OK, fileinstance.Open()); + + // Set file version number + fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); + + // Send a write request and wait for seconds to check if IO returns + auto startWriteAndCheckResult = + [&fileinstance](int sec) -> bool { // NOLINT + CurveAioContext *aioctx = new CurveAioContext(); + char buffer[4096]; + + aioctx->buf = buffer; + aioctx->offset = 0; + aioctx->length = sizeof(buffer); + aioctx->op = LIBCURVE_OP::LIBCURVE_OP_WRITE; + aioctx->cb = WriteCallBack; + + // Send write request + fileinstance.AioWrite(aioctx, UserDataType::RawBuffer); + + std::this_thread::sleep_for(std::chrono::seconds(sec)); + return gWriteSuccessFlag; + }; + + // Successfully written for the first time and updated the file version + // number on the chunkserver side + ASSERT_TRUE(startWriteAndCheckResult(3)); + + // Set an old version number to write + fileinstance.GetIOManager4File()->SetLatestFileSn(kOldFileSn); + gWriteSuccessFlag = false; + + // chunkserver returns the feedback, and after obtaining the version number + // again, it is still the old version IO hang + ASSERT_FALSE(startWriteAndCheckResult(3)); + + // Update version number to normal state + fileinstance.GetIOManager4File()->SetLatestFileSn(kNewFileSn); + std::this_thread::sleep_for(std::chrono::seconds(1)); + + // Last write request successful + ASSERT_EQ(true, gWriteSuccessFlag); + + server.Stop(0); + server.Join(); + } + + TEST_F(CopysetClientTest, retry_rpc_sleep_test) + { + MockChunkServiceImpl mockChunkService; + ASSERT_EQ( + server_->AddService(&mockChunkService, brpc::SERVER_DOESNT_OWN_SERVICE), + 0); + ASSERT_EQ(server_->Start(listenAddr_.c_str(), nullptr), 0); + + const uint64_t sleepUsBeforeRetry = 5 * 1000 * 1000; + + IOSenderOption ioSenderOpt; + ioSenderOpt.failRequestOpt.chunkserverRPCTimeoutMS = 1000; + ioSenderOpt.failRequestOpt.chunkserverOPMaxRetry = 3; + ioSenderOpt.failRequestOpt.chunkserverOPRetryIntervalUS = + sleepUsBeforeRetry; + ioSenderOpt.failRequestOpt.chunkserverMaxRPCTimeoutMS = 3500; + ioSenderOpt.failRequestOpt.chunkserverMaxRetrySleepIntervalUS = 3500000; + + RequestScheduleOption reqopt; + reqopt.ioSenderOpt = ioSenderOpt; + + CopysetClient copysetClient; + MockMetaCache mockMetaCache; + mockMetaCache.DelegateToFake(); + + RequestScheduler scheduler; + scheduler.Init(reqopt, &mockMetaCache); + scheduler.Run(); + copysetClient.Init(&mockMetaCache, ioSenderOpt, &scheduler, nullptr); + + LogicPoolID logicPoolId = 1; + CopysetID copysetId = 100001; + ChunkID chunkId = 1; + uint64_t fileId = 1; + uint64_t epoch = 1; + size_t len = 8; + char buff1[8] = {0}; + butil::IOBuf iobuf; + iobuf.append(buff1, sizeof(len)); + off_t offset = 0; + + ChunkServerID leaderId = 10000; + ChunkServerID leaderId2 = 10001; + butil::EndPoint leaderAddr; + std::string leaderStr = "127.0.0.1:9109"; + butil::str2endpoint(leaderStr.c_str(), &leaderAddr); + + FileMetric fm("test"); + IOTracker iot(nullptr, nullptr, nullptr, &fm); + + { + // In the redirect case, chunkserver returns a new leader + // Will not sleep until retry + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + // reqCtx->writeBuffer_ = buff1; + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId2), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + + auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); + + // Returns a new leader ID, so there will be no sleep before retrying + ASSERT_LE(endUs - startUs, sleepUsBeforeRetry / 10); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + + { + // In the redirect case, chunkserver returns the old leader + // Sleep before retrying + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + // reqCtx->writeBuffer_ = buff1; + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + response1.set_redirect(leaderStr); + + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)) + .Times(1) + .WillOnce(Return(0)); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); + + // Return the same leader ID and sleep before retrying + ASSERT_GE(endUs - startUs, sleepUsBeforeRetry / 10); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + + { + // In the redirect case, chunkserver did not return a leader + // Actively refresh to obtain a new leader + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + // reqCtx->writeBuffer_ = buff1; + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillOnce(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId2), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)).Times(0); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); + + // Returns a new leader id, so there will be no sleep before retrying + ASSERT_LE(endUs - startUs, sleepUsBeforeRetry / 10); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + + { + // In the redirect case, chunkserver did not return a leader + // Actively refresh to obtain old leader + RequestContext *reqCtx = new FakeRequestContext(); + reqCtx->optype_ = OpType::WRITE; + reqCtx->idinfo_ = ChunkIDInfo(chunkId, logicPoolId, copysetId); + // reqCtx->writeBuffer_ = buff1; + reqCtx->writeData_ = iobuf; + reqCtx->offset_ = 0; + reqCtx->rawlength_ = len; + + curve::common::CountDownEvent cond(1); + RequestClosure *reqDone = new FakeRequestClosure(&cond, reqCtx); + reqDone->SetFileMetric(&fm); + reqDone->SetIOTracker(&iot); + reqCtx->done_ = reqDone; + + ChunkResponse response1; + response1.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_REDIRECTED); + + ChunkResponse response2; + response2.set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS); + + EXPECT_CALL(mockMetaCache, GetLeader(_, _, _, _, _, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<2>(leaderId), + SetArgPointee<3>(leaderAddr), Return(0))); + EXPECT_CALL(mockMetaCache, UpdateLeader(_, _, _)).Times(0); + EXPECT_CALL(mockChunkService, WriteChunk(_, _, _, _)) + .Times(2) + .WillOnce( + DoAll(SetArgPointee<2>(response1), Invoke(WriteChunkFunc))) + .WillOnce( + DoAll(SetArgPointee<2>(response2), Invoke(WriteChunkFunc))); + auto startUs = curve::common::TimeUtility::GetTimeofDayUs(); + copysetClient.WriteChunk(reqCtx->idinfo_, fileId, epoch, 0, iobuf, + offset, len, {}, reqDone); + cond.Wait(); + auto endUs = curve::common::TimeUtility::GetTimeofDayUs(); + + ASSERT_GE(endUs - startUs, sleepUsBeforeRetry / 10); + ASSERT_EQ(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS, + reqDone->GetErrorCode()); + } + scheduler.Fini(); + } + + class TestRunnedRequestClosure : public RequestClosure + { + public: + TestRunnedRequestClosure() : RequestClosure(nullptr) {} + + void Run() override { runned_ = true; } + + bool IsRunned() const { return runned_; } + + private: + bool runned_ = false; + }; + + // After the test session fails, the retry request will be placed back in the + // request queue + TEST(CopysetClientBasicTest, TestReScheduleWhenSessionNotValid) + { + MockRequestScheduler requestScheduler; + CopysetClient copysetClient; + IOSenderOption ioSenderOption; + MetaCache metaCache; + + ASSERT_EQ(0, copysetClient.Init(&metaCache, ioSenderOption, + &requestScheduler, nullptr)); + + // Set session not valid + copysetClient.StartRecycleRetryRPC(); + + { + EXPECT_CALL(requestScheduler, ReSchedule(_)).Times(1); + + TestRunnedRequestClosure closure; + copysetClient.ReadChunk({}, 0, 0, 0, {}, &closure); + ASSERT_FALSE(closure.IsRunned()); + } + + { + EXPECT_CALL(requestScheduler, ReSchedule(_)).Times(1); + + TestRunnedRequestClosure closure; + copysetClient.WriteChunk({}, 1, 1, 0, {}, 0, 0, {}, &closure); + ASSERT_FALSE(closure.IsRunned()); + } + } -} // namespace client -} // namespace curve + } // namespace client +} // namespace curve diff --git a/test/client/mds_failover_test.cpp b/test/client/mds_failover_test.cpp index c466457d99..df487eca62 100644 --- a/test/client/mds_failover_test.cpp +++ b/test/client/mds_failover_test.cpp @@ -25,9 +25,9 @@ #include #include -#include //NOLINT +#include //NOLINT #include -#include //NOLINT +#include //NOLINT #include #include "include/client/libcurve.h" @@ -47,239 +47,262 @@ #include "test/integration/cluster_common/cluster.h" #include "test/util/config_generator.h" -namespace curve { -namespace client { - -// Testing mds failover switching state machine -TEST(MDSChangeTest, MDSFailoverTest) { - RPCExcutorRetryPolicy rpcexcutor; - - MetaServerOption metaopt; - metaopt.rpcRetryOpt.addrs.push_back("127.0.0.1:9903"); - metaopt.rpcRetryOpt.addrs.push_back("127.0.0.1:9904"); - metaopt.rpcRetryOpt.addrs.push_back("127.0.0.1:9905"); - - metaopt.rpcRetryOpt.rpcTimeoutMs = 1000; - metaopt.rpcRetryOpt.rpcRetryIntervalUS = 10000; // 10ms - metaopt.rpcRetryOpt.maxFailedTimesBeforeChangeAddr = 2; - metaopt.rpcRetryOpt.rpcTimeoutMs = 1500; - - rpcexcutor.SetOption(metaopt.rpcRetryOpt); - - int mds0RetryTimes = 0; - int mds1RetryTimes = 0; - int mds2RetryTimes = 0; - - // Scenario 1: mds0, 1, 2, currentworkindex=0, mds0, mds1, and mds2 are all - // down, - // All RPCs sent to them are returned as EHOSTDOWN, resulting in - // upper level clients constantly switching to mds and retrying - // Continue according to 0-->1-->2 - // Every time rpc returns -EHOSTDOWN, it will directly trigger RPC - // switching. The final currentworkindex did not switch - auto task1 = [&](int mdsindex, uint64_t rpctimeoutMS, - brpc::Channel* channel, brpc::Controller* cntl) -> int { - if (mdsindex == 0) { - mds0RetryTimes++; +namespace curve +{ + namespace client + { + + // Testing mds failover switching state machine + TEST(MDSChangeTest, MDSFailoverTest) + { + RPCExcutorRetryPolicy rpcexcutor; + + MetaServerOption metaopt; + metaopt.rpcRetryOpt.addrs.push_back("127.0.0.1:9903"); + metaopt.rpcRetryOpt.addrs.push_back("127.0.0.1:9904"); + metaopt.rpcRetryOpt.addrs.push_back("127.0.0.1:9905"); + + metaopt.rpcRetryOpt.rpcTimeoutMs = 1000; + metaopt.rpcRetryOpt.rpcRetryIntervalUS = 10000; // 10ms + metaopt.rpcRetryOpt.maxFailedTimesBeforeChangeAddr = 2; + metaopt.rpcRetryOpt.rpcTimeoutMs = 1500; + + rpcexcutor.SetOption(metaopt.rpcRetryOpt); + + int mds0RetryTimes = 0; + int mds1RetryTimes = 0; + int mds2RetryTimes = 0; + + // Scenario 1: mds0, 1, 2, currentworkindex=0, mds0, mds1, and mds2 are all + // down, + // All RPCs sent to them are returned as EHOSTDOWN, resulting in + // upper level clients constantly switching to mds and retrying + // Continue according to 0-->1-->2 + // Every time rpc returns -EHOSTDOWN, it will directly trigger RPC + // switching. The final currentworkindex did not switch + auto task1 = [&](int mdsindex, uint64_t rpctimeoutMS, + brpc::Channel *channel, brpc::Controller *cntl) -> int + { + if (mdsindex == 0) + { + mds0RetryTimes++; + } + + if (mdsindex == 1) + { + mds1RetryTimes++; + } + + if (mdsindex == 2) + { + mds2RetryTimes++; + } + return -EHOSTDOWN; + }; + + uint64_t startMS = TimeUtility::GetTimeofDayMs(); + // Control surface interface call, 1000 is the total retry time of this RPC + rpcexcutor.DoRPCTask(task1, 1000); + uint64_t endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(endMS - startMS, 1000 - 1); + + // This retry is a polling retry, and the number of retries per mds should + // be close to and not exceed the total number of mds + ASSERT_LT(abs(mds0RetryTimes - mds1RetryTimes), 3); + ASSERT_LT(abs(mds2RetryTimes - mds1RetryTimes), 3); + + startMS = TimeUtility::GetTimeofDayMs(); + rpcexcutor.DoRPCTask(task1, 3000); + endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(endMS - startMS, 3000 - 1); + ASSERT_EQ(0, rpcexcutor.GetCurrentWorkIndex()); + + // Scenario 2: mds0, 1, 2, currentworkindex = 0, mds0 goes down, and it will + // be working at this time + // Mds index switches to index2, and it is expected that the client + // will directly switch to index2 after retrying with index = 0 At + // this point, mds2 directly returns OK and rpc stops trying again. + // Expected client to send a total of two RPCs, one to mds0 and the + // other to mds2, skipping the middle mds1。 + mds0RetryTimes = 0; + mds1RetryTimes = 0; + mds2RetryTimes = 0; + auto task2 = [&](int mdsindex, uint64_t rpctimeoutMS, + brpc::Channel *channel, brpc::Controller *cntl) -> int + { + if (mdsindex == 0) + { + mds0RetryTimes++; + rpcexcutor.SetCurrentWorkIndex(2); + return -ECONNRESET; + } + + if (mdsindex == 1) + { + mds1RetryTimes++; + return -ECONNRESET; + } + + if (mdsindex == 2) + { + mds2RetryTimes++; + // If OK is returned this time, then RPC should have succeeded and + // will not try again + return LIBCURVE_ERROR::OK; + } + + return 0; + }; + startMS = TimeUtility::GetTimeofDayMs(); + rpcexcutor.DoRPCTask(task2, 1000); + endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_LT(endMS - startMS, 1000); + ASSERT_EQ(2, rpcexcutor.GetCurrentWorkIndex()); + ASSERT_EQ(mds0RetryTimes, 1); + ASSERT_EQ(mds1RetryTimes, 0); + ASSERT_EQ(mds2RetryTimes, 1); + + // Scenario 3: mds0, 1, 2, currentworkindex = 1, and mds1 is down, + // At this point, it will switch to mds0 and mds2 + // After switching to 2, mds1 resumed, and then switched to mds1, and + // the rpc was successfully sent. At this point, the switching order is + // 1->2->0, 1->2->0, 1. + mds0RetryTimes = 0; + mds1RetryTimes = 0; + mds2RetryTimes = 0; + rpcexcutor.SetCurrentWorkIndex(1); + auto task3 = [&](int mdsindex, uint64_t rpctimeoutMS, + brpc::Channel *channel, brpc::Controller *cntl) -> int + { + if (mdsindex == 0) + { + mds0RetryTimes++; + return -ECONNRESET; + } + + if (mdsindex == 1) + { + mds1RetryTimes++; + // When retrying on mds1 for the third time, success is returned + // upwards and the retry is stopped + if (mds1RetryTimes == 3) + { + return LIBCURVE_ERROR::OK; + } + return -ECONNREFUSED; + } + + if (mdsindex == 2) + { + mds2RetryTimes++; + return -brpc::ELOGOFF; + } + + return 0; + }; + + startMS = TimeUtility::GetTimeofDayMs(); + rpcexcutor.DoRPCTask(task3, 1000); + endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_LT(endMS - startMS, 1000); + ASSERT_EQ(mds0RetryTimes, 2); + ASSERT_EQ(mds1RetryTimes, 3); + ASSERT_EQ(mds2RetryTimes, 2); + + ASSERT_EQ(1, rpcexcutor.GetCurrentWorkIndex()); + + // Scenario 4: mds0, 1, 2, currentWorkindex = 0, but the rpc request to mds1 + // consistently times out + // The final result returned by rpc is timeout + // For timeout mds nodes, they will continuously retry + // mds.maxFailedTimesBeforeChangeMDS and switch Current + // mds.maxFailedTimesBeforeChangeMDS=2. + // So the retry logic should be: 0->0->1->2, 0->0->1->2, 0->0->1->2, + // ... + LOG(INFO) << "case 4"; + mds0RetryTimes = 0; + mds1RetryTimes = 0; + mds2RetryTimes = 0; + rpcexcutor.SetCurrentWorkIndex(0); + auto task4 = [&](int mdsindex, uint64_t rpctimeoutMS, + brpc::Channel *channel, brpc::Controller *cntl) -> int + { + if (mdsindex == 0) + { + mds0RetryTimes++; + return mds0RetryTimes % 2 == 0 ? -brpc::ERPCTIMEDOUT : -ETIMEDOUT; + } + + if (mdsindex == 1) + { + mds1RetryTimes++; + return -ECONNREFUSED; + } + + if (mdsindex == 2) + { + mds2RetryTimes++; + return -brpc::ELOGOFF; + } + + return 0; + }; + + startMS = TimeUtility::GetTimeofDayMs(); + rpcexcutor.DoRPCTask(task4, 3000); + endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_GT(endMS - startMS, 3000 - 1); + ASSERT_EQ(0, rpcexcutor.GetCurrentWorkIndex()); + // This retry is a polling retry, and the number of retries per mds should + // be close to and not exceed the total number of mds + ASSERT_GT(mds0RetryTimes, mds1RetryTimes + mds2RetryTimes); + + // Scenario 5: mds0, 1, 2, currentWorkIndex = 0 + // But the first 10 requests from rpc all returned EHOSTDOWN + // Mds retries sleep for 10ms, so it takes a total of 100ms + rpcexcutor.SetCurrentWorkIndex(0); + int hostDownTimes = 10; + auto task5 = [&](int mdsindex, uint64_t rpctimeoutMs, + brpc::Channel *channel, brpc::Controller *cntl) + { + static int count = 0; + if (++count <= hostDownTimes) + { + return -EHOSTDOWN; + } + + return 0; + }; + startMS = TimeUtility::GetTimeofDayMs(); + rpcexcutor.DoRPCTask(task5, 10000); // Total retry time 10s + endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_GE(endMS - startMS, 100); + + // Scenario 6: mds keeps returning EHOSTDOWN during the retry process, with + // a total of 5 retries + rpcexcutor.SetCurrentWorkIndex(0); + int calledTimes = 0; + auto task6 = [&](int mdsindex, uint64_t rpctimeoutMs, + brpc::Channel *channel, brpc::Controller *cntl) + { + ++calledTimes; + return -EHOSTDOWN; + }; + + startMS = TimeUtility::GetTimeofDayMs(); + rpcexcutor.DoRPCTask(task6, 5 * 1000); // Total retry time 5s + endMS = TimeUtility::GetTimeofDayMs(); + ASSERT_GE(endMS - startMS, 5 * 1000 - 1); + + // In each hostdown situation, sleep for 10ms and the total retry time is + // 5s, so the total number of retries is less than or equal to 500 times In + // order to minimize false positives, 10 redundant attempts were added + LOG(INFO) << "called times " << calledTimes; + ASSERT_LE(calledTimes, 510); } - if (mdsindex == 1) { - mds1RetryTimes++; - } - - if (mdsindex == 2) { - mds2RetryTimes++; - } - return -EHOSTDOWN; - }; - - uint64_t startMS = TimeUtility::GetTimeofDayMs(); - // Control surface interface call, 1000 is the total retry time of this RPC - rpcexcutor.DoRPCTask(task1, 1000); - uint64_t endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(endMS - startMS, 1000 - 1); - - // This retry is a polling retry, and the number of retries per mds should - // be close to and not exceed the total number of mds - ASSERT_LT(abs(mds0RetryTimes - mds1RetryTimes), 3); - ASSERT_LT(abs(mds2RetryTimes - mds1RetryTimes), 3); - - startMS = TimeUtility::GetTimeofDayMs(); - rpcexcutor.DoRPCTask(task1, 3000); - endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(endMS - startMS, 3000 - 1); - ASSERT_EQ(0, rpcexcutor.GetCurrentWorkIndex()); - - // Scenario 2: mds0, 1, 2, currentworkindex = 0, mds0 goes down, and it will - // be working at this time - // Mds index switches to index2, and it is expected that the client - // will directly switch to index2 after retrying with index = 0 At - // this point, mds2 directly returns OK and rpc stops trying again. - // Expected client to send a total of two RPCs, one to mds0 and the - // other to mds2, skipping the middle mds1。 - mds0RetryTimes = 0; - mds1RetryTimes = 0; - mds2RetryTimes = 0; - auto task2 = [&](int mdsindex, uint64_t rpctimeoutMS, - brpc::Channel* channel, brpc::Controller* cntl) -> int { - if (mdsindex == 0) { - mds0RetryTimes++; - rpcexcutor.SetCurrentWorkIndex(2); - return -ECONNRESET; - } - - if (mdsindex == 1) { - mds1RetryTimes++; - return -ECONNRESET; - } - - if (mdsindex == 2) { - mds2RetryTimes++; - // If OK is returned this time, then RPC should have succeeded and - // will not try again - return LIBCURVE_ERROR::OK; - } - - return 0; - }; - startMS = TimeUtility::GetTimeofDayMs(); - rpcexcutor.DoRPCTask(task2, 1000); - endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_LT(endMS - startMS, 1000); - ASSERT_EQ(2, rpcexcutor.GetCurrentWorkIndex()); - ASSERT_EQ(mds0RetryTimes, 1); - ASSERT_EQ(mds1RetryTimes, 0); - ASSERT_EQ(mds2RetryTimes, 1); - - // Scenario 3: mds0, 1, 2, currentworkindex = 1, and mds1 is down, - // At this point, it will switch to mds0 and mds2 - // After switching to 2, mds1 resumed, and then switched to mds1, and - // the rpc was successfully sent. At this point, the switching order is - // 1->2->0, 1->2->0, 1. - mds0RetryTimes = 0; - mds1RetryTimes = 0; - mds2RetryTimes = 0; - rpcexcutor.SetCurrentWorkIndex(1); - auto task3 = [&](int mdsindex, uint64_t rpctimeoutMS, - brpc::Channel* channel, brpc::Controller* cntl) -> int { - if (mdsindex == 0) { - mds0RetryTimes++; - return -ECONNRESET; - } - - if (mdsindex == 1) { - mds1RetryTimes++; - // When retrying on mds1 for the third time, success is returned - // upwards and the retry is stopped - if (mds1RetryTimes == 3) { - return LIBCURVE_ERROR::OK; - } - return -ECONNREFUSED; - } - - if (mdsindex == 2) { - mds2RetryTimes++; - return -brpc::ELOGOFF; - } - - return 0; - }; - - startMS = TimeUtility::GetTimeofDayMs(); - rpcexcutor.DoRPCTask(task3, 1000); - endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_LT(endMS - startMS, 1000); - ASSERT_EQ(mds0RetryTimes, 2); - ASSERT_EQ(mds1RetryTimes, 3); - ASSERT_EQ(mds2RetryTimes, 2); - - ASSERT_EQ(1, rpcexcutor.GetCurrentWorkIndex()); - - // Scenario 4: mds0, 1, 2, currentWorkindex = 0, but the rpc request to mds1 - // consistently times out - // The final result returned by rpc is timeout - // For timeout mds nodes, they will continuously retry - // mds.maxFailedTimesBeforeChangeMDS and switch Current - // mds.maxFailedTimesBeforeChangeMDS=2. - // So the retry logic should be: 0->0->1->2, 0->0->1->2, 0->0->1->2, - // ... - LOG(INFO) << "case 4"; - mds0RetryTimes = 0; - mds1RetryTimes = 0; - mds2RetryTimes = 0; - rpcexcutor.SetCurrentWorkIndex(0); - auto task4 = [&](int mdsindex, uint64_t rpctimeoutMS, - brpc::Channel* channel, brpc::Controller* cntl) -> int { - if (mdsindex == 0) { - mds0RetryTimes++; - return mds0RetryTimes % 2 == 0 ? -brpc::ERPCTIMEDOUT : -ETIMEDOUT; - } - - if (mdsindex == 1) { - mds1RetryTimes++; - return -ECONNREFUSED; - } - - if (mdsindex == 2) { - mds2RetryTimes++; - return -brpc::ELOGOFF; - } - - return 0; - }; - - startMS = TimeUtility::GetTimeofDayMs(); - rpcexcutor.DoRPCTask(task4, 3000); - endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_GT(endMS - startMS, 3000 - 1); - ASSERT_EQ(0, rpcexcutor.GetCurrentWorkIndex()); - // This retry is a polling retry, and the number of retries per mds should - // be close to and not exceed the total number of mds - ASSERT_GT(mds0RetryTimes, mds1RetryTimes + mds2RetryTimes); - - // Scenario 5: mds0, 1, 2, currentWorkIndex = 0 - // But the first 10 requests from rpc all returned EHOSTDOWN - // Mds retries sleep for 10ms, so it takes a total of 100ms - rpcexcutor.SetCurrentWorkIndex(0); - int hostDownTimes = 10; - auto task5 = [&](int mdsindex, uint64_t rpctimeoutMs, - brpc::Channel* channel, brpc::Controller* cntl) { - static int count = 0; - if (++count <= hostDownTimes) { - return -EHOSTDOWN; - } - - return 0; - }; - startMS = TimeUtility::GetTimeofDayMs(); - rpcexcutor.DoRPCTask(task5, 10000); // Total retry time 10s - endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_GE(endMS - startMS, 100); - - // Scenario 6: mds keeps returning EHOSTDOWN during the retry process, with - // a total of 5 retries - rpcexcutor.SetCurrentWorkIndex(0); - int calledTimes = 0; - auto task6 = [&](int mdsindex, uint64_t rpctimeoutMs, - brpc::Channel* channel, brpc::Controller* cntl) { - ++calledTimes; - return -EHOSTDOWN; - }; - - startMS = TimeUtility::GetTimeofDayMs(); - rpcexcutor.DoRPCTask(task6, 5 * 1000); // Total retry time 5s - endMS = TimeUtility::GetTimeofDayMs(); - ASSERT_GE(endMS - startMS, 5 * 1000 - 1); - - // In each hostdown situation, sleep for 10ms and the total retry time is - // 5s, so the total number of retries is less than or equal to 500 times In - // order to minimize false positives, 10 redundant attempts were added - LOG(INFO) << "called times " << calledTimes; - ASSERT_LE(calledTimes, 510); -} - -} // namespace client -} // namespace curve + } // namespace client +} // namespace curve const std::vector registConfOff{ std::string("mds.listen.addr=127.0.0.1:9903,127.0.0.1:9904,127.0.0.1:9905"), @@ -311,11 +334,12 @@ const std::vector registConfON{ std::string("mds.registerToMDS=true")}; std::string mdsMetaServerAddr = - "127.0.0.1:9903,127.0.0.1:9904,127.0.0.1:9905"; // NOLINT -uint32_t segment_size = 1 * 1024 * 1024 * 1024ul; // NOLINT -uint32_t chunk_size = 4 * 1024 * 1024; // NOLINT -std::string configpath = "./test/client/configs/mds_failover.conf"; // NOLINT -int main(int argc, char** argv) { + "127.0.0.1:9903,127.0.0.1:9904,127.0.0.1:9905"; // NOLINT +uint32_t segment_size = 1 * 1024 * 1024 * 1024ul; // NOLINT +uint32_t chunk_size = 4 * 1024 * 1024; // NOLINT +std::string configpath = "./test/client/configs/mds_failover.conf"; // NOLINT +int main(int argc, char **argv) +{ ::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleMock(&argc, argv); google::ParseCommandLineFlags(&argc, &argv, false); diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp index 4bcc65f8e7..1ff6116653 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp @@ -39,52 +39,52 @@ using curve::client::FileClient; using curve::client::SourceReader; using curve::client::UserInfo_t; -const std::string kTestPrefix = "SCSTest"; // NOLINT +const std::string kTestPrefix = "SCSTest"; // NOLINT const uint64_t chunkSize = 16ULL * 1024 * 1024; const uint64_t segmentSize = 32ULL * 1024 * 1024; const uint64_t chunkGap = 1; -const char* kEtcdClientIpPort = "127.0.0.1:10001"; -const char* kEtcdPeerIpPort = "127.0.0.1:10002"; -const char* kMdsIpPort = "127.0.0.1:10003"; -const char* kChunkServerIpPort1 = "127.0.0.1:10004"; -const char* kChunkServerIpPort2 = "127.0.0.1:10005"; -const char* kChunkServerIpPort3 = "127.0.0.1:10006"; -const char* kSnapshotCloneServerIpPort = "127.0.0.1:10007"; +const char *kEtcdClientIpPort = "127.0.0.1:10001"; +const char *kEtcdPeerIpPort = "127.0.0.1:10002"; +const char *kMdsIpPort = "127.0.0.1:10003"; +const char *kChunkServerIpPort1 = "127.0.0.1:10004"; +const char *kChunkServerIpPort2 = "127.0.0.1:10005"; +const char *kChunkServerIpPort3 = "127.0.0.1:10006"; +const char *kSnapshotCloneServerIpPort = "127.0.0.1:10007"; const int kMdsDummyPort = 10008; -const char* kSnapshotCloneServerDummyServerPort = "12000"; -const char* kLeaderCampaginPrefix = "snapshotcloneserverleaderlock3"; +const char *kSnapshotCloneServerDummyServerPort = "12000"; +const char *kLeaderCampaginPrefix = "snapshotcloneserverleaderlock3"; -static const char* kDefaultPoolset = "default"; +static const char *kDefaultPoolset = "default"; -const std::string kLogPath = "./runlog/" + kTestPrefix + "Log"; // NOLINT -const std::string kMdsDbName = kTestPrefix + "DB"; // NOLINT -const std::string kMdsConfigPath = // NOLINT +const std::string kLogPath = "./runlog/" + kTestPrefix + "Log"; // NOLINT +const std::string kMdsDbName = kTestPrefix + "DB"; // NOLINT +const std::string kMdsConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_mds.conf"; -const std::string kCSConfigPath = // NOLINT +const std::string kCSConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_chunkserver.conf"; -const std::string kCsClientConfigPath = // NOLINT +const std::string kCsClientConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_cs_client.conf"; -const std::string kSnapClientConfigPath = // NOLINT +const std::string kSnapClientConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_snap_client.conf"; -const std::string kS3ConfigPath = // NOLINT +const std::string kS3ConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_s3.conf"; -const std::string kSCSConfigPath = // NOLINT +const std::string kSCSConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_scs.conf"; -const std::string kClientConfigPath = // NOLINT +const std::string kClientConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_client.conf"; @@ -130,13 +130,13 @@ const std::vector chunkserverConf1{ {"--graceful_quit_on_sigterm"}, {"-chunkServerStoreUri=local://./" + kTestPrefix + "1/"}, {"-chunkServerMetaUri=local://./" + kTestPrefix + - "1/chunkserver.dat"}, // NOLINT + "1/chunkserver.dat"}, // NOLINT {"-copySetUri=local://./" + kTestPrefix + "1/copysets"}, {"-raftSnapshotUri=curve://./" + kTestPrefix + "1/copysets"}, {"-recycleUri=local://./" + kTestPrefix + "1/recycler"}, {"-chunkFilePoolDir=./" + kTestPrefix + "1/chunkfilepool/"}, {"-chunkFilePoolMetaPath=./" + kTestPrefix + - "1/chunkfilepool.meta"}, // NOLINT + "1/chunkfilepool.meta"}, // NOLINT std::string("-conf=") + kCSConfigPath, {"-raft_sync_segments=true"}, std::string("--log_dir=") + kLogPath, @@ -150,13 +150,13 @@ const std::vector chunkserverConf2{ {"--graceful_quit_on_sigterm"}, {"-chunkServerStoreUri=local://./" + kTestPrefix + "2/"}, {"-chunkServerMetaUri=local://./" + kTestPrefix + - "2/chunkserver.dat"}, // NOLINT + "2/chunkserver.dat"}, // NOLINT {"-copySetUri=local://./" + kTestPrefix + "2/copysets"}, {"-raftSnapshotUri=curve://./" + kTestPrefix + "2/copysets"}, {"-recycleUri=local://./" + kTestPrefix + "2/recycler"}, {"-chunkFilePoolDir=./" + kTestPrefix + "2/chunkfilepool/"}, {"-chunkFilePoolMetaPath=./" + kTestPrefix + - "2/chunkfilepool.meta"}, // NOLINT + "2/chunkfilepool.meta"}, // NOLINT std::string("-conf=") + kCSConfigPath, {"-raft_sync_segments=true"}, std::string("--log_dir=") + kLogPath, @@ -170,13 +170,13 @@ const std::vector chunkserverConf3{ {"--graceful_quit_on_sigterm"}, {"-chunkServerStoreUri=local://./" + kTestPrefix + "3/"}, {"-chunkServerMetaUri=local://./" + kTestPrefix + - "3/chunkserver.dat"}, // NOLINT + "3/chunkserver.dat"}, // NOLINT {"-copySetUri=local://./" + kTestPrefix + "3/copysets"}, {"-raftSnapshotUri=curve://./" + kTestPrefix + "3/copysets"}, {"-recycleUri=local://./" + kTestPrefix + "3/recycler"}, {"-chunkFilePoolDir=./" + kTestPrefix + "3/chunkfilepool/"}, {"-chunkFilePoolMetaPath=./" + kTestPrefix + - "3/chunkfilepool.meta"}, // NOLINT + "3/chunkfilepool.meta"}, // NOLINT std::string("-conf=") + kCSConfigPath, {"-raft_sync_segments=true"}, std::string("--log_dir=") + kLogPath, @@ -217,908 +217,954 @@ const std::vector clientConfigOptions{ std::string("mds.rpcTimeoutMS=4000"), }; -const char* testFile1_ = "/ItUser1/file1"; -const char* testFile2_ = "/ItUser1/file2"; -const char* testFile3_ = "/ItUser2/file3"; -const char* testFile4_ = "/ItUser1/file3"; -const char* testFile5_ = "/ItUser1/file4"; -const char* testUser1_ = "ItUser1"; -const char* testUser2_ = "ItUser2"; - -namespace curve { -namespace snapshotcloneserver { - -class SnapshotCloneServerTest : public ::testing::Test { - public: - static void SetUpTestCase() { - std::string mkLogDirCmd = std::string("mkdir -p ") + kLogPath; - system(mkLogDirCmd.c_str()); - - cluster_ = new CurveCluster(); - ASSERT_NE(nullptr, cluster_); - - // Initialize db - system(std::string("rm -rf " + kTestPrefix + ".etcd").c_str()); - system(std::string("rm -rf " + kTestPrefix + "1").c_str()); - system(std::string("rm -rf " + kTestPrefix + "2").c_str()); - system(std::string("rm -rf " + kTestPrefix + "3").c_str()); - - // Start etcd - pid_t pid = cluster_->StartSingleEtcd( - 1, kEtcdClientIpPort, kEtcdPeerIpPort, - std::vector{"--name=" + kTestPrefix}); - LOG(INFO) << "etcd 1 started on " << kEtcdPeerIpPort - << ", pid = " << pid; - ASSERT_GT(pid, 0); - - cluster_->InitSnapshotCloneMetaStoreEtcd(kEtcdClientIpPort); - - cluster_->PrepareConfig(kMdsConfigPath, - mdsConfigOptions); - - // Start an mds - pid = cluster_->StartSingleMDS(1, kMdsIpPort, kMdsDummyPort, mdsConf1, - true); - LOG(INFO) << "mds 1 started on " << kMdsIpPort << ", pid = " << pid; - ASSERT_GT(pid, 0); - - // Creating a physical pool - ASSERT_EQ(0, cluster_->PreparePhysicalPool( - 1, - "./test/integration/snapshotcloneserver/" - "config/topo.json")); // NOLINT - - // format chunkfilepool and walfilepool - std::vector threadpool(3); - - threadpool[0] = std::thread(&CurveCluster::FormatFilePool, cluster_, - "./" + kTestPrefix + "1/chunkfilepool/", - "./" + kTestPrefix + "1/chunkfilepool.meta", - "./" + kTestPrefix + "1/chunkfilepool/", 1); - threadpool[1] = std::thread(&CurveCluster::FormatFilePool, cluster_, - "./" + kTestPrefix + "2/chunkfilepool/", - "./" + kTestPrefix + "2/chunkfilepool.meta", - "./" + kTestPrefix + "2/chunkfilepool/", 1); - threadpool[2] = std::thread(&CurveCluster::FormatFilePool, cluster_, - "./" + kTestPrefix + "3/chunkfilepool/", - "./" + kTestPrefix + "3/chunkfilepool.meta", - "./" + kTestPrefix + "3/chunkfilepool/", 1); - for (int i = 0; i < 3; i++) { - threadpool[i].join(); +const char *testFile1_ = "/ItUser1/file1"; +const char *testFile2_ = "/ItUser1/file2"; +const char *testFile3_ = "/ItUser2/file3"; +const char *testFile4_ = "/ItUser1/file3"; +const char *testFile5_ = "/ItUser1/file4"; +const char *testUser1_ = "ItUser1"; +const char *testUser2_ = "ItUser2"; + +namespace curve +{ + namespace snapshotcloneserver + { + + class SnapshotCloneServerTest : public ::testing::Test + { + public: + static void SetUpTestCase() + { + std::string mkLogDirCmd = std::string("mkdir -p ") + kLogPath; + system(mkLogDirCmd.c_str()); + + cluster_ = new CurveCluster(); + ASSERT_NE(nullptr, cluster_); + + // Initialize db + system(std::string("rm -rf " + kTestPrefix + ".etcd").c_str()); + system(std::string("rm -rf " + kTestPrefix + "1").c_str()); + system(std::string("rm -rf " + kTestPrefix + "2").c_str()); + system(std::string("rm -rf " + kTestPrefix + "3").c_str()); + + // Start etcd + pid_t pid = cluster_->StartSingleEtcd( + 1, kEtcdClientIpPort, kEtcdPeerIpPort, + std::vector{"--name=" + kTestPrefix}); + LOG(INFO) << "etcd 1 started on " << kEtcdPeerIpPort + << ", pid = " << pid; + ASSERT_GT(pid, 0); + + cluster_->InitSnapshotCloneMetaStoreEtcd(kEtcdClientIpPort); + + cluster_->PrepareConfig(kMdsConfigPath, + mdsConfigOptions); + + // Start an mds + pid = cluster_->StartSingleMDS(1, kMdsIpPort, kMdsDummyPort, mdsConf1, + true); + LOG(INFO) << "mds 1 started on " << kMdsIpPort << ", pid = " << pid; + ASSERT_GT(pid, 0); + + // Creating a physical pool + ASSERT_EQ(0, cluster_->PreparePhysicalPool( + 1, + "./test/integration/snapshotcloneserver/" + "config/topo.json")); // NOLINT + + // format chunkfilepool and walfilepool + std::vector threadpool(3); + + threadpool[0] = std::thread(&CurveCluster::FormatFilePool, cluster_, + "./" + kTestPrefix + "1/chunkfilepool/", + "./" + kTestPrefix + "1/chunkfilepool.meta", + "./" + kTestPrefix + "1/chunkfilepool/", 1); + threadpool[1] = std::thread(&CurveCluster::FormatFilePool, cluster_, + "./" + kTestPrefix + "2/chunkfilepool/", + "./" + kTestPrefix + "2/chunkfilepool.meta", + "./" + kTestPrefix + "2/chunkfilepool/", 1); + threadpool[2] = std::thread(&CurveCluster::FormatFilePool, cluster_, + "./" + kTestPrefix + "3/chunkfilepool/", + "./" + kTestPrefix + "3/chunkfilepool.meta", + "./" + kTestPrefix + "3/chunkfilepool/", 1); + for (int i = 0; i < 3; i++) + { + threadpool[i].join(); + } + + cluster_->PrepareConfig(kCsClientConfigPath, + csClientConfigOptions); + + cluster_->PrepareConfig(kS3ConfigPath, + s3ConfigOptions); + + cluster_->PrepareConfig(kCSConfigPath, + chunkserverConfigOptions); + + // Create chunkserver + pid = cluster_->StartSingleChunkServer(1, kChunkServerIpPort1, + chunkserverConf1); + LOG(INFO) << "chunkserver 1 started on " << kChunkServerIpPort1 + << ", pid = " << pid; + ASSERT_GT(pid, 0); + pid = cluster_->StartSingleChunkServer(2, kChunkServerIpPort2, + chunkserverConf2); + LOG(INFO) << "chunkserver 2 started on " << kChunkServerIpPort2 + << ", pid = " << pid; + ASSERT_GT(pid, 0); + pid = cluster_->StartSingleChunkServer(3, kChunkServerIpPort3, + chunkserverConf3); + LOG(INFO) << "chunkserver 3 started on " << kChunkServerIpPort3 + << ", pid = " << pid; + ASSERT_GT(pid, 0); + + std::this_thread::sleep_for(std::chrono::seconds(5)); + + // Create a logical pool and sleep for a period of time to let the + // underlying copyset select the primary first + ASSERT_EQ(0, cluster_->PrepareLogicalPool( + 1, + "./test/integration/snapshotcloneserver/config/" + "topo.json")); + + cluster_->PrepareConfig( + kSnapClientConfigPath, snapClientConfigOptions); + + cluster_->PrepareConfig( + kSCSConfigPath, snapshotcloneserverConfigOptions); + + pid = cluster_->StartSnapshotCloneServer(1, kSnapshotCloneServerIpPort, + snapshotcloneConf); + LOG(INFO) << "SnapshotCloneServer 1 started on " + << kSnapshotCloneServerIpPort << ", pid = " << pid; + ASSERT_GT(pid, 0); + + cluster_->PrepareConfig(kClientConfigPath, + clientConfigOptions); + + fileClient_ = new FileClient(); + fileClient_->Init(kClientConfigPath); + + UserInfo_t userinfo; + userinfo.owner = "ItUser1"; + + ASSERT_EQ(0, fileClient_->Mkdir("/ItUser1", userinfo)); + + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CreateAndWriteFile(testFile1_, testUser1_, fakeData)); + LOG(INFO) << "Write testFile1_ success."; + + ASSERT_TRUE(CreateAndWriteFile(testFile2_, testUser1_, fakeData)); + LOG(INFO) << "Write testFile2_ success."; + + UserInfo_t userinfo2; + userinfo2.owner = "ItUser2"; + ASSERT_EQ(0, fileClient_->Mkdir("/ItUser2", userinfo2)); + + ASSERT_TRUE(CreateAndWriteFile(testFile3_, testUser2_, fakeData)); + LOG(INFO) << "Write testFile3_ success."; + + ASSERT_EQ(0, fileClient_->Create(testFile4_, userinfo, + 10ULL * 1024 * 1024 * 1024)); + + ASSERT_EQ(0, fileClient_->Create(testFile5_, userinfo, + 10ULL * 1024 * 1024 * 1024)); + } + + static bool CreateAndWriteFile(const std::string &fileName, + const std::string &user, + const std::string &dataSample) + { + UserInfo_t userinfo; + userinfo.owner = user; + int ret = + fileClient_->Create(fileName, userinfo, 10ULL * 1024 * 1024 * 1024); + if (ret < 0) + { + LOG(ERROR) << "Create fail, ret = " << ret; + return false; + } + return WriteFile(fileName, user, dataSample); + } + + static bool WriteFile(const std::string &fileName, const std::string &user, + const std::string &dataSample) + { + int ret = 0; + UserInfo_t userinfo; + userinfo.owner = user; + int testfd1_ = fileClient_->Open(fileName, userinfo); + if (testfd1_ < 0) + { + LOG(ERROR) << "Open fail, ret = " << testfd1_; + return false; + } + // Write the first 4k data and two segments for each chunk + uint64_t totalChunk = 2ULL * segmentSize / chunkSize; + for (uint64_t i = 0; i < totalChunk / chunkGap; i++) + { + ret = + fileClient_->Write(testfd1_, dataSample.c_str(), + i * chunkSize * chunkGap, dataSample.size()); + if (ret < 0) + { + LOG(ERROR) << "Write Fail, ret = " << ret; + return false; + } + } + ret = fileClient_->Close(testfd1_); + if (ret < 0) + { + LOG(ERROR) << "Close fail, ret = " << ret; + return false; + } + return true; + } + + static bool CheckFileData(const std::string &fileName, + const std::string &user, + const std::string &dataSample) + { + UserInfo_t userinfo; + userinfo.owner = user; + int dstFd = fileClient_->Open(fileName, userinfo); + if (dstFd < 0) + { + LOG(ERROR) << "Open fail, ret = " << dstFd; + return false; + } + + int ret = 0; + uint64_t totalChunk = 2ULL * segmentSize / chunkSize; + for (uint64_t i = 0; i < totalChunk / chunkGap; i++) + { + char buf[4096]; + ret = fileClient_->Read(dstFd, buf, i * chunkSize * chunkGap, 4096); + if (ret < 0) + { + LOG(ERROR) << "Read fail, ret = " << ret; + return false; + } + std::string data(buf, 4096); + if (data != dataSample) + { + LOG(ERROR) << "CheckFileData not Equal, data = [" << data + << "] , expect data = [" << dataSample << "]."; + return false; + } + } + ret = fileClient_->Close(dstFd); + if (ret < 0) + { + LOG(ERROR) << "Close fail, ret = " << ret; + return false; + } + return true; + } + + static void TearDownTestCase() + { + fileClient_->UnInit(); + delete fileClient_; + fileClient_ = nullptr; + ASSERT_EQ(0, cluster_->StopCluster()); + delete cluster_; + cluster_ = nullptr; + system(std::string("rm -rf " + kTestPrefix + ".etcd").c_str()); + system(std::string("rm -rf " + kTestPrefix + "1").c_str()); + system(std::string("rm -rf " + kTestPrefix + "2").c_str()); + system(std::string("rm -rf " + kTestPrefix + "3").c_str()); + } + + void SetUp() {} + + void TearDown() {} + + void PrepareSnapshotForTestFile1(std::string *uuid1) + { + if (!hasSnapshotForTestFile1_) + { + int ret = MakeSnapshot(testUser1_, testFile1_, "snap1", uuid1); + ASSERT_EQ(0, ret); + bool success1 = + CheckSnapshotSuccess(testUser1_, testFile1_, *uuid1); + ASSERT_TRUE(success1); + hasSnapshotForTestFile1_ = true; + snapIdForTestFile1_ = *uuid1; + } + } + + void WaitDeleteSnapshotForTestFile1() + { + if (hasSnapshotForTestFile1_) + { + ASSERT_EQ(0, DeleteAndCheckSnapshotSuccess(testUser1_, testFile1_, + snapIdForTestFile1_)); + } + } + + static CurveCluster *cluster_; + static FileClient *fileClient_; + + bool hasSnapshotForTestFile1_ = false; + std::string snapIdForTestFile1_; + }; + + CurveCluster *SnapshotCloneServerTest::cluster_ = nullptr; + FileClient *SnapshotCloneServerTest::fileClient_ = nullptr; + + // Regular test cases + // Scenario 1: Adding, deleting, and searching snapshots + TEST_F(SnapshotCloneServerTest, TestSnapshotAddDeleteGet) + { + std::string uuid1; + int ret = 0; + // Step1: User testUser1_ Take a snapshot of non-existent files + // Expected 1: Return file does not exist + ret = MakeSnapshot(testUser1_, "/ItUser1/notExistFile", "snap1", &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: User testUser2_ For testFile1_ Take a snapshot + // Expected 2: Failed to return user authentication + ret = MakeSnapshot(testUser2_, testFile1_, "snap1", &uuid1); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step3: User testUser1_ For testFile1_ Take a snapshot snap1. + // Expected 3: Successful snapshot taking + ret = MakeSnapshot(testUser1_, testFile1_, "snap1", &uuid1); + ASSERT_EQ(0, ret); + + std::string fakeData(4096, 'y'); + ASSERT_TRUE(WriteFile(testFile1_, testUser1_, fakeData)); + ASSERT_TRUE(CheckFileData(testFile1_, testUser1_, fakeData)); + + // Step4: Obtain snapshot information, user=testUser1_, filename=testFile1_ + // Expected 4: Return information for snapshot snap1 + bool success1 = CheckSnapshotSuccess(testUser1_, testFile1_, uuid1); + ASSERT_TRUE(success1); + + // Step5: Obtain snapshot information, user=testUser2_, filename=testFile1_ + // Expected 5: User authentication failure returned + FileSnapshotInfo info1; + ret = GetSnapshotInfo(testUser2_, testFile1_, uuid1, &info1); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step6: Obtain snapshot information, user=testUser2_, filename=testFile2_ + // Expected 6: Return null + std::vector infoVec; + ret = ListFileSnapshotInfo(testUser2_, testFile2_, 10, 0, &infoVec); + ASSERT_EQ(0, ret); + ASSERT_EQ(0, infoVec.size()); + + // Step7: testUser2_ Delete snapshot snap1 + // Expected 7: User authentication failure returned + ret = DeleteSnapshot(testUser2_, testFile1_, uuid1); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step8: testUser1_ Delete testFile2_ Snapshot with ID snap1 for + // Expected 8: Return file name mismatch + ret = DeleteSnapshot(testUser1_, testFile2_, uuid1); + ASSERT_EQ(kErrCodeFileNameNotMatch, ret); + + // Step9: testUser1_ Delete snapshot snap1 + // Expected 9: Successful deletion returned + ret = DeleteAndCheckSnapshotSuccess(testUser1_, testFile1_, uuid1); + ASSERT_EQ(0, ret); + + // Step10: Obtain snapshot information, user=testUser1_, filename=testFile1_ + // Expected 10: Return empty + ret = ListFileSnapshotInfo(testUser1_, testFile1_, 10, 0, &infoVec); + ASSERT_EQ(0, ret); + ASSERT_EQ(0, infoVec.size()); + + // Step11: testUser1_ Delete snapshot snap1 (duplicate deletion) + // Expected 11: Successful deletion returned + ret = DeleteAndCheckSnapshotSuccess(testUser1_, testFile1_, uuid1); + ASSERT_EQ(0, ret); + + // Restore testFile1_ + std::string fakeData2(4096, 'x'); + ASSERT_TRUE(WriteFile(testFile1_, testUser1_, fakeData2)); } - cluster_->PrepareConfig(kCsClientConfigPath, - csClientConfigOptions); - - cluster_->PrepareConfig(kS3ConfigPath, - s3ConfigOptions); - - cluster_->PrepareConfig(kCSConfigPath, - chunkserverConfigOptions); - - // Create chunkserver - pid = cluster_->StartSingleChunkServer(1, kChunkServerIpPort1, - chunkserverConf1); - LOG(INFO) << "chunkserver 1 started on " << kChunkServerIpPort1 - << ", pid = " << pid; - ASSERT_GT(pid, 0); - pid = cluster_->StartSingleChunkServer(2, kChunkServerIpPort2, - chunkserverConf2); - LOG(INFO) << "chunkserver 2 started on " << kChunkServerIpPort2 - << ", pid = " << pid; - ASSERT_GT(pid, 0); - pid = cluster_->StartSingleChunkServer(3, kChunkServerIpPort3, - chunkserverConf3); - LOG(INFO) << "chunkserver 3 started on " << kChunkServerIpPort3 - << ", pid = " << pid; - ASSERT_GT(pid, 0); - - std::this_thread::sleep_for(std::chrono::seconds(5)); - - // Create a logical pool and sleep for a period of time to let the - // underlying copyset select the primary first - ASSERT_EQ(0, cluster_->PrepareLogicalPool( - 1, - "./test/integration/snapshotcloneserver/config/" - "topo.json")); - - cluster_->PrepareConfig( - kSnapClientConfigPath, snapClientConfigOptions); - - cluster_->PrepareConfig( - kSCSConfigPath, snapshotcloneserverConfigOptions); - - pid = cluster_->StartSnapshotCloneServer(1, kSnapshotCloneServerIpPort, - snapshotcloneConf); - LOG(INFO) << "SnapshotCloneServer 1 started on " - << kSnapshotCloneServerIpPort << ", pid = " << pid; - ASSERT_GT(pid, 0); - - cluster_->PrepareConfig(kClientConfigPath, - clientConfigOptions); - - fileClient_ = new FileClient(); - fileClient_->Init(kClientConfigPath); - - UserInfo_t userinfo; - userinfo.owner = "ItUser1"; - - ASSERT_EQ(0, fileClient_->Mkdir("/ItUser1", userinfo)); - - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CreateAndWriteFile(testFile1_, testUser1_, fakeData)); - LOG(INFO) << "Write testFile1_ success."; - - ASSERT_TRUE(CreateAndWriteFile(testFile2_, testUser1_, fakeData)); - LOG(INFO) << "Write testFile2_ success."; - - UserInfo_t userinfo2; - userinfo2.owner = "ItUser2"; - ASSERT_EQ(0, fileClient_->Mkdir("/ItUser2", userinfo2)); - - ASSERT_TRUE(CreateAndWriteFile(testFile3_, testUser2_, fakeData)); - LOG(INFO) << "Write testFile3_ success."; - - ASSERT_EQ(0, fileClient_->Create(testFile4_, userinfo, - 10ULL * 1024 * 1024 * 1024)); - - ASSERT_EQ(0, fileClient_->Create(testFile5_, userinfo, - 10ULL * 1024 * 1024 * 1024)); - } - - static bool CreateAndWriteFile(const std::string& fileName, - const std::string& user, - const std::string& dataSample) { - UserInfo_t userinfo; - userinfo.owner = user; - int ret = - fileClient_->Create(fileName, userinfo, 10ULL * 1024 * 1024 * 1024); - if (ret < 0) { - LOG(ERROR) << "Create fail, ret = " << ret; - return false; + // Scenario 2: Cancel Snapshot + TEST_F(SnapshotCloneServerTest, TestCancelSnapshot) + { + std::string uuid1; + int ret = MakeSnapshot(testUser1_, testFile1_, "snapToCancle", &uuid1); + ASSERT_EQ(0, ret); + + bool success1 = false; + bool isCancel = false; + for (int i = 0; i < 600; i++) + { + FileSnapshotInfo info1; + int retCode = GetSnapshotInfo(testUser1_, testFile1_, uuid1, &info1); + if (retCode == 0) + { + if (info1.GetSnapshotInfo().GetStatus() == Status::pending || + info1.GetSnapshotInfo().GetStatus() == Status::canceling) + { + if (!isCancel) + { + // Step1: User testUser1_ For testFile1_ Take a snapshot + // snap1, + // testUser2_ before the snapshot is completed_ + // Cancel testFile1_ Snap1 of snapshot + // Expected 1: Failed to cancel user authentication + int retCode = CancelSnapshot(testUser2_, testFile1_, uuid1); + ASSERT_EQ(kErrCodeInvalidUser, retCode); + + // Step2: User testUser1_ For testFile1_ Take a snapshot + // snap1, + // testUser1_ before the snapshot is completed_ + // Cancel testFile1_ A non-existent snapshot of + // Expected 2: Return kErrCodeCannotCancelFinished + retCode = + CancelSnapshot(testUser1_, testFile1_, "notExistUUId"); + ASSERT_EQ(kErrCodeCannotCancelFinished, retCode); + + // Step3: User testUser1_ For testFile1_ Take a snapshot + // snap1, + // testUser1_ before the snapshot is completed_ + // Cancel testFile2_ Snap1 of snapshot + // Expected 3: Return file name mismatch + retCode = CancelSnapshot(testUser1_, testFile2_, uuid1); + ASSERT_EQ(kErrCodeFileNameNotMatch, retCode); + + // Step4: User testUser1_ For testFile1_ Take a snapshot, + // testUser1_ before the snapshot is completed_ + // Cancel snapshot snap1 + // Expected 4: Successfully cancelled snapshot + retCode = CancelSnapshot(testUser1_, testFile1_, uuid1); + ASSERT_EQ(0, retCode); + isCancel = true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(3000)); + continue; + } + else if (info1.GetSnapshotInfo().GetStatus() == Status::done) + { + success1 = false; + break; + } + else + { + FAIL() << "Snapshot Fail On status = " + << static_cast(info1.GetSnapshotInfo().GetStatus()); + } + } + else if (retCode == -8) + { + // Step5: Obtain snapshot information, user=testUser1_, + // filename=testFile1_ Expected 5: Return empty + success1 = true; + break; + } + } + ASSERT_TRUE(success1); + + // Step6: After the snapshot is completed, testUser1_ Cancel testFile1_ + // Snap1 of snapshot Expected 6: Returning a pending snapshot that does not + // exist or has been completed + ret = CancelSnapshot(testUser1_, testFile1_, uuid1); + ASSERT_EQ(kErrCodeCannotCancelFinished, ret); } - return WriteFile(fileName, user, dataSample); - } - - static bool WriteFile(const std::string& fileName, const std::string& user, - const std::string& dataSample) { - int ret = 0; - UserInfo_t userinfo; - userinfo.owner = user; - int testfd1_ = fileClient_->Open(fileName, userinfo); - if (testfd1_ < 0) { - LOG(ERROR) << "Open fail, ret = " << testfd1_; - return false; + + // Scenario 3: Lazy snapshot clone scene + TEST_F(SnapshotCloneServerTest, TestSnapLazyClone) + { + std::string snapId; + PrepareSnapshotForTestFile1(&snapId); + + // Step1: testUser1_ A snapshot with a clone that does not exist, + // fileName=SnapLazyClone1 Expected 1: Return snapshot does not exist + std::string uuid1, uuid2, uuid3, uuid4, uuid5; + int ret; + ret = CloneOrRecover("Clone", testUser1_, "UnExistSnapId1", + "/ItUser1/SnapLazyClone1", true, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: testUser2_ Clone snapshot snap1, fileName=SnapLazyClone1 + // Expected 2: User authentication failure returned + ret = CloneOrRecover("Clone", testUser2_, snapId, "/ItUser2/SnapLazyClone1", + true, &uuid2); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step3: testUser1_ Clone snapshot snap1, fileName=SnapLazyClone1 + // Expected 3 to return successful cloning + std::string dstFile = "/ItUser1/SnapLazyClone1"; + ret = CloneOrRecover("Clone", testUser1_, snapId, dstFile, true, &uuid3); + ASSERT_EQ(0, ret); + + // Step4: testUser1_ Clone block photo snap1, fileName=SnapLazyClone1 + // (duplicate clone) Expected 4: Returns successful cloning (idempotent) + ret = CloneOrRecover("Clone", testUser1_, snapId, "/ItUser1/SnapLazyClone1", + true, &uuid4); + ASSERT_EQ(0, ret); + + // Flatten + ret = Flatten(testUser1_, uuid3); + ASSERT_EQ(0, ret); + + // Step5: testUser1_ GetCloneTask + // Expected 5: Return clone task for SnapLazyClone1 + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid3, true); + ASSERT_TRUE(success1); + + // Step6: testUser2_ GetCloneTask + // Expected 6: Return null + std::vector infoVec; + ret = ListCloneTaskInfo(testUser2_, 10, 0, &infoVec); + ASSERT_EQ(0, ret); + ASSERT_EQ(0, infoVec.size()); + + // Step7: testUser2_ CleanCloneTask UUID is the UUID of SnapLazyClone1 + // Expected 7: User authentication failure returned + ret = CleanCloneTask(testUser2_, uuid3); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step8: testUser1_ CleanCloneTask UUID is the UUID of SnapLazyClone1 + // Expected 8: Return execution successful + ret = CleanCloneTask(testUser1_, uuid3); + ASSERT_EQ(0, ret); + + // Waiting for cleaning to complete + std::this_thread::sleep_for(std::chrono::seconds(3)); + + // Step9: testUser1_ CleanCloneTask UUID is the UUID of SnapLazyClone1 + // (repeated execution) Expected 9: Return execution successful + ret = CleanCloneTask(testUser1_, uuid3); + ASSERT_EQ(0, ret); + + // Step10: testUser1_ GetCloneTask + // Expected 10: Return empty + TaskCloneInfo info; + ret = GetCloneTaskInfo(testUser1_, uuid3, &info); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Verify data correctness + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); } - // Write the first 4k data and two segments for each chunk - uint64_t totalChunk = 2ULL * segmentSize / chunkSize; - for (uint64_t i = 0; i < totalChunk / chunkGap; i++) { - ret = - fileClient_->Write(testfd1_, dataSample.c_str(), - i * chunkSize * chunkGap, dataSample.size()); - if (ret < 0) { - LOG(ERROR) << "Write Fail, ret = " << ret; - return false; - } + + // Scenario 4: Non lazy snapshot clone scenario + TEST_F(SnapshotCloneServerTest, TestSnapNotLazyClone) + { + std::string snapId; + PrepareSnapshotForTestFile1(&snapId); + + // Step1: testUser1_ A snapshot with a clone that does not exist, + // fileName=SnapNotLazyClone1 Expected 1: Return snapshot does not exist + std::string uuid1; + int ret; + ret = CloneOrRecover("Clone", testUser1_, "UnExistSnapId2", + "/ItUser1/SnapNotLazyClone1", false, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: testUser2_ Clone snapshot snap1, fileName=SnapNotLazyClone1 + // Expected 2: User authentication failure returned + ret = CloneOrRecover("Clone", testUser2_, snapId, + "/ItUser2/SnapNotLazyClone1", false, &uuid1); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step3: testUser1_ Clone snapshot snap1, fileName=SnapNotLazyClone1 + // Expected 3 to return successful cloning + std::string dstFile = "/ItUser1/SnapNotLazyClone1"; + ret = CloneOrRecover("Clone", testUser1_, snapId, dstFile, false, &uuid1); + ASSERT_EQ(0, ret); + + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, true); + ASSERT_TRUE(success1); + + // Step4: testUser1_ Clone block photo snap1, + // fileName=SnapNotLazyClone1 (duplicate clone) + // Expected 4: Returns successful cloning (idempotent) + ret = CloneOrRecover("Clone", testUser1_, snapId, + "/ItUser1/SnapNotLazyClone1", false, &uuid1); + ASSERT_EQ(0, ret); + + // Verify data correctness + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); } - ret = fileClient_->Close(testfd1_); - if (ret < 0) { - LOG(ERROR) << "Close fail, ret = " << ret; - return false; + + // Scenario 5: Lazy snapshot recovery scenario + TEST_F(SnapshotCloneServerTest, TestSnapLazyRecover) + { + std::string snapId; + PrepareSnapshotForTestFile1(&snapId); + + // Step1: testUser1_ Recover snapshot that does not exist, + // fileName=testFile1_ Expected 1: Return snapshot does not exist + std::string uuid1; + int ret; + ret = CloneOrRecover("Recover", testUser1_, "UnExistSnapId3", testFile1_, + true, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: testUser2_ Recover snapshot snap1, fileName=testFile1_ + // Expected 2: User authentication failure returned + ret = + CloneOrRecover("Recover", testUser2_, snapId, testFile1_, true, &uuid1); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step3: testUser1_ Recover snapshot snap1, fileName=testFile1_ + // Expected 3 return recovery success + ret = + CloneOrRecover("Recover", testUser1_, snapId, testFile1_, true, &uuid1); + ASSERT_EQ(0, ret); + + // Flatten + ret = Flatten(testUser1_, uuid1); + ASSERT_EQ(0, ret); + + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, false); + ASSERT_TRUE(success1); + + // Verify data correctness + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CheckFileData(testFile1_, testUser1_, fakeData)); + + // Step4: testUser1_ Recover snapshot snap1, target file is a non-existent + // file Expected 4: Return target file does not exist + ret = CloneOrRecover("Recover", testUser1_, snapId, "/ItUser1/notExistFile", + true, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); } - return true; - } - - static bool CheckFileData(const std::string& fileName, - const std::string& user, - const std::string& dataSample) { - UserInfo_t userinfo; - userinfo.owner = user; - int dstFd = fileClient_->Open(fileName, userinfo); - if (dstFd < 0) { - LOG(ERROR) << "Open fail, ret = " << dstFd; - return false; + + // Scenario 6: Non lazy snapshot recovery scenario + TEST_F(SnapshotCloneServerTest, TestSnapNotLazyRecover) + { + std::string snapId; + PrepareSnapshotForTestFile1(&snapId); + + // Step1: testUser1_ Recover snapshot that does not exist, + // fileName=testFile1_ Expected 1: Return snapshot does not exist + std::string uuid1; + int ret; + ret = CloneOrRecover("Recover", testUser1_, "UnExistSnapId4", testFile1_, + false, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: testUser2_ Recover snapshot snap1, fileName=testFile1_ + // Expected 2: User authentication failure returned + ret = CloneOrRecover("Recover", testUser2_, snapId, testFile1_, false, + &uuid1); + ASSERT_EQ(kErrCodeInvalidUser, ret); + + // Step3: testUser1_ Recover snapshot snap1, fileName=testFile1_ + // Expected 3 return recovery success + ret = CloneOrRecover("Recover", testUser1_, snapId, testFile1_, false, + &uuid1); + ASSERT_EQ(0, ret); + + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, false); + ASSERT_TRUE(success1); + + // Verify data correctness + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CheckFileData(testFile1_, testUser1_, fakeData)); + + // Step4: testUser1_ Recover snapshot snap1, target file is a non-existent + // file Expected 4: Return target file does not exist + ret = CloneOrRecover("Recover", testUser1_, snapId, "/ItUser1/notExistFile", + false, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); } - int ret = 0; - uint64_t totalChunk = 2ULL * segmentSize / chunkSize; - for (uint64_t i = 0; i < totalChunk / chunkGap; i++) { - char buf[4096]; - ret = fileClient_->Read(dstFd, buf, i * chunkSize * chunkGap, 4096); - if (ret < 0) { - LOG(ERROR) << "Read fail, ret = " << ret; - return false; - } - std::string data(buf, 4096); - if (data != dataSample) { - LOG(ERROR) << "CheckFileData not Equal, data = [" << data - << "] , expect data = [" << dataSample << "]."; - return false; - } + // Scenario 7: Lazy Mirror Clone Scene + TEST_F(SnapshotCloneServerTest, TestImageLazyClone) + { + // Step1: testUser1_ Clone does not exist in an image, + // fileName=ImageLazyClone1 Expected 1: Return file does not exist + std::string uuid1, uuid2, uuid3, uuid4; + int ret; + ret = CloneOrRecover("Clone", testUser1_, "/UnExistFile", + "/ItUser1/ImageLazyClone1", true, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: testUser1_ Clone image testFile1_, fileName=ImageLazyClone1 + // Expected 2 to return successful cloning + std::string dstFile = "/ItUser1/ImageLazyClone1"; + ret = + CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid2); + ASSERT_EQ(0, ret); + + // Step3: testUser1_ Clone image testFile1_, + // FileName=ImageLazyClone1 (duplicate clone) + // Expected 3: Returns successful cloning (idempotent) + ret = CloneOrRecover("Clone", testUser1_, testFile1_, + "/ItUser1/ImageLazyClone1", true, &uuid3); + ASSERT_EQ(0, ret); + + // Step4: Take a snapshot snap1 of the file ImageLazyClone1 that has not + // completed the lazy clone Expected 4: Abnormal file status returned + ret = MakeSnapshot(testUser1_, testFile1_, "snap1", &uuid4); + ASSERT_EQ(kErrCodeFileStatusInvalid, ret); + FileSnapshotInfo info2; + int retCode = GetSnapshotInfo(testUser1_, testFile1_, uuid4, &info2); + ASSERT_EQ(kErrCodeFileNotExist, retCode); + + ASSERT_TRUE(WaitMetaInstalledSuccess(testUser1_, uuid2, true)); + + // Verify data correctness before Flatten + std::string fakeData1(4096, 'x'); + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData1)); + + // Flatten + ret = Flatten(testUser1_, uuid2); + ASSERT_EQ(0, ret); + + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid2, true); + ASSERT_TRUE(success1); + + // Verify data correctness after Flatten + std::string fakeData2(4096, 'x'); + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData2)); } - ret = fileClient_->Close(dstFd); - if (ret < 0) { - LOG(ERROR) << "Close fail, ret = " << ret; - return false; + + // Scenario 8: Non Lazy Mirror Clone Scene + TEST_F(SnapshotCloneServerTest, TestImageNotLazyClone) + { + // Step1: testUser1_ Clone does not exist in an image, + // fileName=ImageNotLazyClone1 Expected 1: Return snapshot does not exist + std::string uuid1; + int ret; + ret = CloneOrRecover("Clone", testUser1_, "/UnExistFile", + "/ItUser1/ImageNotLazyClone1", false, &uuid1); + ASSERT_EQ(kErrCodeFileNotExist, ret); + + // Step2: testUser1_ Clone image testFile1_, fileName=ImageNotLazyClone1 + // Expected 2 to return successful cloning + std::string dstFile = "/ItUser1/ImageNotLazyClone1"; + ret = + CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, false, &uuid1); + ASSERT_EQ(0, ret); + + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, true); + ASSERT_TRUE(success1); + + // Step3: testUser1_ Clone image testFile1_, + // FileName=ImageNotLazyClone1 (duplicate clone) + // Expected 3: Returns successful cloning (idempotent) + ret = CloneOrRecover("Clone", testUser1_, testFile1_, + "/ItUser1/ImageNotLazyClone1", false, &uuid1); + ASSERT_EQ(0, ret); + + // Verify data correctness + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); } - return true; - } - - static void TearDownTestCase() { - fileClient_->UnInit(); - delete fileClient_; - fileClient_ = nullptr; - ASSERT_EQ(0, cluster_->StopCluster()); - delete cluster_; - cluster_ = nullptr; - system(std::string("rm -rf " + kTestPrefix + ".etcd").c_str()); - system(std::string("rm -rf " + kTestPrefix + "1").c_str()); - system(std::string("rm -rf " + kTestPrefix + "2").c_str()); - system(std::string("rm -rf " + kTestPrefix + "3").c_str()); - } - - void SetUp() {} - - void TearDown() {} - - void PrepareSnapshotForTestFile1(std::string* uuid1) { - if (!hasSnapshotForTestFile1_) { - int ret = MakeSnapshot(testUser1_, testFile1_, "snap1", uuid1); + + // Scenario 9: The snapshot has a failure scenario + TEST_F(SnapshotCloneServerTest, TestSnapAndCloneWhenSnapHasError) + { + std::string snapId = "errorSnapUuid"; + SnapshotInfo snapInfo(snapId, testUser1_, testFile4_, "snapxxx", 0, 0, 0, 0, + 0, 0, kDefaultPoolset, 0, Status::error); + + cluster_->metaStore_->AddSnapshot(snapInfo); + + pid_t pid = cluster_->RestartSnapshotCloneServer(1); + LOG(INFO) << "SnapshotCloneServer 1 restarted, pid = " << pid; + ASSERT_GT(pid, 0); + std::string uuid1, uuid2; + + // Step1: lazy clone snapshot snap1 + // Expected 1: Exception in returning snapshot + int ret = CloneOrRecover("Clone", testUser1_, snapId, + "/ItUser2/SnapLazyClone1", true, &uuid2); + ASSERT_EQ(kErrCodeInvalidSnapshot, ret); + + // Step2: Non lazy clone snapshot snap1 + // Expected 2: Exception in returning snapshot + ret = CloneOrRecover("Clone", testUser1_, snapId, + "/ItUser2/SnapNotLazyClone1", false, &uuid2); + ASSERT_EQ(kErrCodeInvalidSnapshot, ret); + + // Step3: lazy snap1 recover from snapshot + // Expected 3: Exception in returning snapshot + ret = + CloneOrRecover("Recover", testUser1_, snapId, testFile4_, true, &uuid2); + ASSERT_EQ(kErrCodeInvalidSnapshot, ret); + + // Step4: Snap1 recover from snapshot without lazy + // Expected 4: Exception in returning snapshot + ret = CloneOrRecover("Recover", testUser1_, snapId, testFile4_, false, + &uuid2); + ASSERT_EQ(kErrCodeInvalidSnapshot, ret); + + // Step5: User testUser1_ For testFile4_ Take a snapshot snap1 + // Expectation 5: Clean failed snapshot and take snapshot successfully + ret = MakeSnapshot(testUser1_, testFile4_, "snap1", &uuid1); ASSERT_EQ(0, ret); - bool success1 = - CheckSnapshotSuccess(testUser1_, testFile1_, *uuid1); + + // Successfully verified snapshot + bool success1 = CheckSnapshotSuccess(testUser1_, testFile4_, uuid1); ASSERT_TRUE(success1); - hasSnapshotForTestFile1_ = true; - snapIdForTestFile1_ = *uuid1; + + // Verification cleaning failed, snapshot succeeded + FileSnapshotInfo info1; + int retCode = GetSnapshotInfo(testUser1_, testFile4_, snapId, &info1); + ASSERT_EQ(kErrCodeFileNotExist, retCode); } - } - void WaitDeleteSnapshotForTestFile1() { - if (hasSnapshotForTestFile1_) { - ASSERT_EQ(0, DeleteAndCheckSnapshotSuccess(testUser1_, testFile1_, - snapIdForTestFile1_)); + //[Online issue repair] Clone failed, rollback delete clone volume, and create + // the same uuid volume again scenario + TEST_F(SnapshotCloneServerTest, TestCloneHasSameDest) + { + std::string uuid1, uuid2, uuid3, uuid4, uuid5, uuid6, uuid7; + // Step1: testUser1_ Clone image testFile1_, fileName=CloneHasSameDestUUID + // Expected 1 to return successful cloning + std::string dstFile = "/ItUser1/CloneHasSameDest"; + int ret = + CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid1); + ASSERT_EQ(0, ret); + + // Delete Clone Volume + UserInfo_t userinfo; + userinfo.owner = testUser1_; + int ret2 = fileClient_->Unlink(dstFile, userinfo, false); + ASSERT_EQ(0, ret2); + + // Step2: testUser1_ Clone image testFile1_ again, + // fileName=CloneHasSameDestUUID + // Expected 2 to return successful cloning + ret = + CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid2); + ASSERT_EQ(0, ret); + + // Verify data correctness + std::string fakeData(4096, 'x'); + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); + + // Step3: testUser1_ Clone image testFile1_, fileName=CloneHasSameDest2 + // Expected 3 to return successful cloning + dstFile = "/ItUser1/CloneHasSameDest2"; + ret = + CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid3); + ASSERT_EQ(0, ret); + + // Delete Clone Volume + UserInfo_t userinfo2; + userinfo2.owner = testUser1_; + ret2 = fileClient_->Unlink(dstFile, userinfo2, false); + ASSERT_EQ(0, ret2); + + // Step4: testUser1_ Clone the image testFile2_ again, + // fileName=CloneHasSameDest2 + // Expected 4 to return successful cloning + ret = + CloneOrRecover("Clone", testUser1_, testFile2_, dstFile, true, &uuid4); + ASSERT_EQ(0, ret); + + // Verify data correctness + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); + + // Verify different situations when cloning lazyflag again + // Step5: testUser1_ Clone image testFile1_, fileName=CloneHasSameDest3 + // Expected 5 to return successful cloning + dstFile = "/ItUser1/CloneHasSameDest3"; + ret = + CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid5); + ASSERT_EQ(0, ret); + + // Delete Clone Volume + UserInfo_t userinfo3; + userinfo2.owner = testUser1_; + ret2 = fileClient_->Unlink(dstFile, userinfo2, false); + ASSERT_EQ(0, ret2); + + // Step6: testUser1_ Non lazy clone image testFile2_ again, + // fileName=CloneHasSameDest3 + // Expected 6 to return successful cloning + ret = + CloneOrRecover("Clone", testUser1_, testFile2_, dstFile, false, &uuid6); + ASSERT_EQ(0, ret); + + bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid6, true); + ASSERT_TRUE(success1); + + // Verify data correctness + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); + + // Delete Clone Volume + UserInfo_t userinfo4; + userinfo2.owner = testUser1_; + ret2 = fileClient_->Unlink(dstFile, userinfo2, false); + ASSERT_EQ(0, ret2); + + // Step7: testUser1_ Non lazy clone image testFile2_ again, + // fileName=CloneHasSameDest3 + // Expected 7 to return successful cloning + ret = + CloneOrRecover("Clone", testUser1_, testFile2_, dstFile, true, &uuid7); + ASSERT_EQ(0, ret); + + // Verify data correctness + ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); } - } - static CurveCluster* cluster_; - static FileClient* fileClient_; + // Lazy clone volume, delete clone volume, and then delete source volume. The + // source volume can be deleted if needed + TEST_F(SnapshotCloneServerTest, TestDeleteLazyCloneDestThenDeleteSrc) + { + // Step1: testUser1_ Clone image testFile5_, Lazy clone two volumes dstFile1 + // and dstFile2 Expected 1 to return successful cloning + std::string uuid1; + std::string uuid2; + std::string dstFile1 = "/dest1"; + std::string dstFile2 = "/dest2"; + UserInfo_t userinfo; + userinfo.owner = testUser1_; + int ret = + CloneOrRecover("Clone", testUser1_, testFile5_, dstFile1, true, &uuid1); + ASSERT_EQ(0, ret); + + ret = + CloneOrRecover("Clone", testUser1_, testFile5_, dstFile2, true, &uuid2); + ASSERT_EQ(0, ret); + + // Delete source volume, deletion failed, volume occupied - bool hasSnapshotForTestFile1_ = false; - std::string snapIdForTestFile1_; -}; + ret = fileClient_->Unlink(testFile5_, userinfo, false); + ASSERT_EQ(-27, ret); -CurveCluster* SnapshotCloneServerTest::cluster_ = nullptr; -FileClient* SnapshotCloneServerTest::fileClient_ = nullptr; - -// Regular test cases -// Scenario 1: Adding, deleting, and searching snapshots -TEST_F(SnapshotCloneServerTest, TestSnapshotAddDeleteGet) { - std::string uuid1; - int ret = 0; - // Step1: User testUser1_ Take a snapshot of non-existent files - // Expected 1: Return file does not exist - ret = MakeSnapshot(testUser1_, "/ItUser1/notExistFile", "snap1", &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: User testUser2_ For testFile1_ Take a snapshot - // Expected 2: Failed to return user authentication - ret = MakeSnapshot(testUser2_, testFile1_, "snap1", &uuid1); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step3: User testUser1_ For testFile1_ Take a snapshot snap1. - // Expected 3: Successful snapshot taking - ret = MakeSnapshot(testUser1_, testFile1_, "snap1", &uuid1); - ASSERT_EQ(0, ret); - - std::string fakeData(4096, 'y'); - ASSERT_TRUE(WriteFile(testFile1_, testUser1_, fakeData)); - ASSERT_TRUE(CheckFileData(testFile1_, testUser1_, fakeData)); - - // Step4: Obtain snapshot information, user=testUser1_, filename=testFile1_ - // Expected 4: Return information for snapshot snap1 - bool success1 = CheckSnapshotSuccess(testUser1_, testFile1_, uuid1); - ASSERT_TRUE(success1); - - // Step5: Obtain snapshot information, user=testUser2_, filename=testFile1_ - // Expected 5: User authentication failure returned - FileSnapshotInfo info1; - ret = GetSnapshotInfo(testUser2_, testFile1_, uuid1, &info1); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step6: Obtain snapshot information, user=testUser2_, filename=testFile2_ - // Expected 6: Return null - std::vector infoVec; - ret = ListFileSnapshotInfo(testUser2_, testFile2_, 10, 0, &infoVec); - ASSERT_EQ(0, ret); - ASSERT_EQ(0, infoVec.size()); - - // Step7: testUser2_ Delete snapshot snap1 - // Expected 7: User authentication failure returned - ret = DeleteSnapshot(testUser2_, testFile1_, uuid1); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step8: testUser1_ Delete testFile2_ Snapshot with ID snap1 for - // Expected 8: Return file name mismatch - ret = DeleteSnapshot(testUser1_, testFile2_, uuid1); - ASSERT_EQ(kErrCodeFileNameNotMatch, ret); - - // Step9: testUser1_ Delete snapshot snap1 - // Expected 9: Successful deletion returned - ret = DeleteAndCheckSnapshotSuccess(testUser1_, testFile1_, uuid1); - ASSERT_EQ(0, ret); - - // Step10: Obtain snapshot information, user=testUser1_, filename=testFile1_ - // Expected 10: Return empty - ret = ListFileSnapshotInfo(testUser1_, testFile1_, 10, 0, &infoVec); - ASSERT_EQ(0, ret); - ASSERT_EQ(0, infoVec.size()); - - // Step11: testUser1_ Delete snapshot snap1 (duplicate deletion) - // Expected 11: Successful deletion returned - ret = DeleteAndCheckSnapshotSuccess(testUser1_, testFile1_, uuid1); - ASSERT_EQ(0, ret); - - // Restore testFile1_ - std::string fakeData2(4096, 'x'); - ASSERT_TRUE(WriteFile(testFile1_, testUser1_, fakeData2)); -} - -// Scenario 2: Cancel Snapshot -TEST_F(SnapshotCloneServerTest, TestCancelSnapshot) { - std::string uuid1; - int ret = MakeSnapshot(testUser1_, testFile1_, "snapToCancle", &uuid1); - ASSERT_EQ(0, ret); - - bool success1 = false; - bool isCancel = false; - for (int i = 0; i < 600; i++) { - FileSnapshotInfo info1; - int retCode = GetSnapshotInfo(testUser1_, testFile1_, uuid1, &info1); - if (retCode == 0) { - if (info1.GetSnapshotInfo().GetStatus() == Status::pending || - info1.GetSnapshotInfo().GetStatus() == Status::canceling) { - if (!isCancel) { - // Step1: User testUser1_ For testFile1_ Take a snapshot - // snap1, - // testUser2_ before the snapshot is completed_ - // Cancel testFile1_ Snap1 of snapshot - // Expected 1: Failed to cancel user authentication - int retCode = CancelSnapshot(testUser2_, testFile1_, uuid1); - ASSERT_EQ(kErrCodeInvalidUser, retCode); - - // Step2: User testUser1_ For testFile1_ Take a snapshot - // snap1, - // testUser1_ before the snapshot is completed_ - // Cancel testFile1_ A non-existent snapshot of - // Expected 2: Return kErrCodeCannotCancelFinished - retCode = - CancelSnapshot(testUser1_, testFile1_, "notExistUUId"); - ASSERT_EQ(kErrCodeCannotCancelFinished, retCode); - - // Step3: User testUser1_ For testFile1_ Take a snapshot - // snap1, - // testUser1_ before the snapshot is completed_ - // Cancel testFile2_ Snap1 of snapshot - // Expected 3: Return file name mismatch - retCode = CancelSnapshot(testUser1_, testFile2_, uuid1); - ASSERT_EQ(kErrCodeFileNameNotMatch, retCode); - - // Step4: User testUser1_ For testFile1_ Take a snapshot, - // testUser1_ before the snapshot is completed_ - // Cancel snapshot snap1 - // Expected 4: Successfully cancelled snapshot - retCode = CancelSnapshot(testUser1_, testFile1_, uuid1); - ASSERT_EQ(0, retCode); - isCancel = true; + // Step2: Successfully delete the destination volume dstFile1, delete the + // source volume again Expected 2 deletion failed, volume occupied + ret = fileClient_->Unlink(dstFile1, userinfo, false); + ASSERT_EQ(0, ret); + + ret = fileClient_->Unlink(testFile5_, userinfo, false); + ASSERT_EQ(-27, ret); + + // Step3: Successfully delete the destination volume dstFile2, delete the + // source volume again Expected 3 deletion successful + ret = fileClient_->Unlink(dstFile2, userinfo, false); + ASSERT_EQ(0, ret); + + ret = fileClient_->Unlink(testFile5_, userinfo, false); + ASSERT_EQ(0, ret); + + // Step4: Wait for a period of time to see if the garbage record can be + // deleted in the background + bool noRecord = false; + for (int i = 0; i < 100; i++) + { + TaskCloneInfo info; + int ret1 = GetCloneTaskInfo(testUser1_, uuid1, &info); + int ret2 = GetCloneTaskInfo(testUser1_, uuid2, &info); + if (ret1 == kErrCodeFileNotExist && ret2 == kErrCodeFileNotExist) + { + noRecord = true; + break; } + std::this_thread::sleep_for(std::chrono::milliseconds(3000)); - continue; - } else if (info1.GetSnapshotInfo().GetStatus() == Status::done) { - success1 = false; - break; - } else { - FAIL() << "Snapshot Fail On status = " - << static_cast(info1.GetSnapshotInfo().GetStatus()); } - } else if (retCode == -8) { - // Step5: Obtain snapshot information, user=testUser1_, - // filename=testFile1_ Expected 5: Return empty - success1 = true; - break; - } - } - ASSERT_TRUE(success1); - - // Step6: After the snapshot is completed, testUser1_ Cancel testFile1_ - // Snap1 of snapshot Expected 6: Returning a pending snapshot that does not - // exist or has been completed - ret = CancelSnapshot(testUser1_, testFile1_, uuid1); - ASSERT_EQ(kErrCodeCannotCancelFinished, ret); -} - -// Scenario 3: Lazy snapshot clone scene -TEST_F(SnapshotCloneServerTest, TestSnapLazyClone) { - std::string snapId; - PrepareSnapshotForTestFile1(&snapId); - - // Step1: testUser1_ A snapshot with a clone that does not exist, - // fileName=SnapLazyClone1 Expected 1: Return snapshot does not exist - std::string uuid1, uuid2, uuid3, uuid4, uuid5; - int ret; - ret = CloneOrRecover("Clone", testUser1_, "UnExistSnapId1", - "/ItUser1/SnapLazyClone1", true, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: testUser2_ Clone snapshot snap1, fileName=SnapLazyClone1 - // Expected 2: User authentication failure returned - ret = CloneOrRecover("Clone", testUser2_, snapId, "/ItUser2/SnapLazyClone1", - true, &uuid2); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step3: testUser1_ Clone snapshot snap1, fileName=SnapLazyClone1 - // Expected 3 to return successful cloning - std::string dstFile = "/ItUser1/SnapLazyClone1"; - ret = CloneOrRecover("Clone", testUser1_, snapId, dstFile, true, &uuid3); - ASSERT_EQ(0, ret); - - // Step4: testUser1_ Clone block photo snap1, fileName=SnapLazyClone1 - // (duplicate clone) Expected 4: Returns successful cloning (idempotent) - ret = CloneOrRecover("Clone", testUser1_, snapId, "/ItUser1/SnapLazyClone1", - true, &uuid4); - ASSERT_EQ(0, ret); - - // Flatten - ret = Flatten(testUser1_, uuid3); - ASSERT_EQ(0, ret); - - // Step5: testUser1_ GetCloneTask - // Expected 5: Return clone task for SnapLazyClone1 - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid3, true); - ASSERT_TRUE(success1); - - // Step6: testUser2_ GetCloneTask - // Expected 6: Return null - std::vector infoVec; - ret = ListCloneTaskInfo(testUser2_, 10, 0, &infoVec); - ASSERT_EQ(0, ret); - ASSERT_EQ(0, infoVec.size()); - - // Step7: testUser2_ CleanCloneTask UUID is the UUID of SnapLazyClone1 - // Expected 7: User authentication failure returned - ret = CleanCloneTask(testUser2_, uuid3); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step8: testUser1_ CleanCloneTask UUID is the UUID of SnapLazyClone1 - // Expected 8: Return execution successful - ret = CleanCloneTask(testUser1_, uuid3); - ASSERT_EQ(0, ret); - - // Waiting for cleaning to complete - std::this_thread::sleep_for(std::chrono::seconds(3)); - - // Step9: testUser1_ CleanCloneTask UUID is the UUID of SnapLazyClone1 - // (repeated execution) Expected 9: Return execution successful - ret = CleanCloneTask(testUser1_, uuid3); - ASSERT_EQ(0, ret); - - // Step10: testUser1_ GetCloneTask - // Expected 10: Return empty - TaskCloneInfo info; - ret = GetCloneTaskInfo(testUser1_, uuid3, &info); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Verify data correctness - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); -} - -// Scenario 4: Non lazy snapshot clone scenario -TEST_F(SnapshotCloneServerTest, TestSnapNotLazyClone) { - std::string snapId; - PrepareSnapshotForTestFile1(&snapId); - - // Step1: testUser1_ A snapshot with a clone that does not exist, - // fileName=SnapNotLazyClone1 Expected 1: Return snapshot does not exist - std::string uuid1; - int ret; - ret = CloneOrRecover("Clone", testUser1_, "UnExistSnapId2", - "/ItUser1/SnapNotLazyClone1", false, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: testUser2_ Clone snapshot snap1, fileName=SnapNotLazyClone1 - // Expected 2: User authentication failure returned - ret = CloneOrRecover("Clone", testUser2_, snapId, - "/ItUser2/SnapNotLazyClone1", false, &uuid1); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step3: testUser1_ Clone snapshot snap1, fileName=SnapNotLazyClone1 - // Expected 3 to return successful cloning - std::string dstFile = "/ItUser1/SnapNotLazyClone1"; - ret = CloneOrRecover("Clone", testUser1_, snapId, dstFile, false, &uuid1); - ASSERT_EQ(0, ret); - - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, true); - ASSERT_TRUE(success1); - - // Step4: testUser1_ Clone block photo snap1, - // fileName=SnapNotLazyClone1 (duplicate clone) - // Expected 4: Returns successful cloning (idempotent) - ret = CloneOrRecover("Clone", testUser1_, snapId, - "/ItUser1/SnapNotLazyClone1", false, &uuid1); - ASSERT_EQ(0, ret); - - // Verify data correctness - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); -} - -// Scenario 5: Lazy snapshot recovery scenario -TEST_F(SnapshotCloneServerTest, TestSnapLazyRecover) { - std::string snapId; - PrepareSnapshotForTestFile1(&snapId); - - // Step1: testUser1_ Recover snapshot that does not exist, - // fileName=testFile1_ Expected 1: Return snapshot does not exist - std::string uuid1; - int ret; - ret = CloneOrRecover("Recover", testUser1_, "UnExistSnapId3", testFile1_, - true, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: testUser2_ Recover snapshot snap1, fileName=testFile1_ - // Expected 2: User authentication failure returned - ret = - CloneOrRecover("Recover", testUser2_, snapId, testFile1_, true, &uuid1); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step3: testUser1_ Recover snapshot snap1, fileName=testFile1_ - // Expected 3 return recovery success - ret = - CloneOrRecover("Recover", testUser1_, snapId, testFile1_, true, &uuid1); - ASSERT_EQ(0, ret); - - // Flatten - ret = Flatten(testUser1_, uuid1); - ASSERT_EQ(0, ret); - - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, false); - ASSERT_TRUE(success1); - - // Verify data correctness - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CheckFileData(testFile1_, testUser1_, fakeData)); - - // Step4: testUser1_ Recover snapshot snap1, target file is a non-existent - // file Expected 4: Return target file does not exist - ret = CloneOrRecover("Recover", testUser1_, snapId, "/ItUser1/notExistFile", - true, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); -} - -// Scenario 6: Non lazy snapshot recovery scenario -TEST_F(SnapshotCloneServerTest, TestSnapNotLazyRecover) { - std::string snapId; - PrepareSnapshotForTestFile1(&snapId); - - // Step1: testUser1_ Recover snapshot that does not exist, - // fileName=testFile1_ Expected 1: Return snapshot does not exist - std::string uuid1; - int ret; - ret = CloneOrRecover("Recover", testUser1_, "UnExistSnapId4", testFile1_, - false, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: testUser2_ Recover snapshot snap1, fileName=testFile1_ - // Expected 2: User authentication failure returned - ret = CloneOrRecover("Recover", testUser2_, snapId, testFile1_, false, - &uuid1); - ASSERT_EQ(kErrCodeInvalidUser, ret); - - // Step3: testUser1_ Recover snapshot snap1, fileName=testFile1_ - // Expected 3 return recovery success - ret = CloneOrRecover("Recover", testUser1_, snapId, testFile1_, false, - &uuid1); - ASSERT_EQ(0, ret); - - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, false); - ASSERT_TRUE(success1); - - // Verify data correctness - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CheckFileData(testFile1_, testUser1_, fakeData)); - - // Step4: testUser1_ Recover snapshot snap1, target file is a non-existent - // file Expected 4: Return target file does not exist - ret = CloneOrRecover("Recover", testUser1_, snapId, "/ItUser1/notExistFile", - false, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); -} - -// Scenario 7: Lazy Mirror Clone Scene -TEST_F(SnapshotCloneServerTest, TestImageLazyClone) { - // Step1: testUser1_ Clone does not exist in an image, - // fileName=ImageLazyClone1 Expected 1: Return file does not exist - std::string uuid1, uuid2, uuid3, uuid4; - int ret; - ret = CloneOrRecover("Clone", testUser1_, "/UnExistFile", - "/ItUser1/ImageLazyClone1", true, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: testUser1_ Clone image testFile1_, fileName=ImageLazyClone1 - // Expected 2 to return successful cloning - std::string dstFile = "/ItUser1/ImageLazyClone1"; - ret = - CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid2); - ASSERT_EQ(0, ret); - - // Step3: testUser1_ Clone image testFile1_, - // FileName=ImageLazyClone1 (duplicate clone) - // Expected 3: Returns successful cloning (idempotent) - ret = CloneOrRecover("Clone", testUser1_, testFile1_, - "/ItUser1/ImageLazyClone1", true, &uuid3); - ASSERT_EQ(0, ret); - - // Step4: Take a snapshot snap1 of the file ImageLazyClone1 that has not - // completed the lazy clone Expected 4: Abnormal file status returned - ret = MakeSnapshot(testUser1_, testFile1_, "snap1", &uuid4); - ASSERT_EQ(kErrCodeFileStatusInvalid, ret); - FileSnapshotInfo info2; - int retCode = GetSnapshotInfo(testUser1_, testFile1_, uuid4, &info2); - ASSERT_EQ(kErrCodeFileNotExist, retCode); - - ASSERT_TRUE(WaitMetaInstalledSuccess(testUser1_, uuid2, true)); - - // Verify data correctness before Flatten - std::string fakeData1(4096, 'x'); - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData1)); - - // Flatten - ret = Flatten(testUser1_, uuid2); - ASSERT_EQ(0, ret); - - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid2, true); - ASSERT_TRUE(success1); - - // Verify data correctness after Flatten - std::string fakeData2(4096, 'x'); - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData2)); -} - -// Scenario 8: Non Lazy Mirror Clone Scene -TEST_F(SnapshotCloneServerTest, TestImageNotLazyClone) { - // Step1: testUser1_ Clone does not exist in an image, - // fileName=ImageNotLazyClone1 Expected 1: Return snapshot does not exist - std::string uuid1; - int ret; - ret = CloneOrRecover("Clone", testUser1_, "/UnExistFile", - "/ItUser1/ImageNotLazyClone1", false, &uuid1); - ASSERT_EQ(kErrCodeFileNotExist, ret); - - // Step2: testUser1_ Clone image testFile1_, fileName=ImageNotLazyClone1 - // Expected 2 to return successful cloning - std::string dstFile = "/ItUser1/ImageNotLazyClone1"; - ret = - CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, false, &uuid1); - ASSERT_EQ(0, ret); - - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid1, true); - ASSERT_TRUE(success1); - - // Step3: testUser1_ Clone image testFile1_, - // FileName=ImageNotLazyClone1 (duplicate clone) - // Expected 3: Returns successful cloning (idempotent) - ret = CloneOrRecover("Clone", testUser1_, testFile1_, - "/ItUser1/ImageNotLazyClone1", false, &uuid1); - ASSERT_EQ(0, ret); - - // Verify data correctness - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); -} - -// Scenario 9: The snapshot has a failure scenario -TEST_F(SnapshotCloneServerTest, TestSnapAndCloneWhenSnapHasError) { - std::string snapId = "errorSnapUuid"; - SnapshotInfo snapInfo(snapId, testUser1_, testFile4_, "snapxxx", 0, 0, 0, 0, - 0, 0, kDefaultPoolset, 0, Status::error); - - cluster_->metaStore_->AddSnapshot(snapInfo); - - pid_t pid = cluster_->RestartSnapshotCloneServer(1); - LOG(INFO) << "SnapshotCloneServer 1 restarted, pid = " << pid; - ASSERT_GT(pid, 0); - std::string uuid1, uuid2; - - // Step1: lazy clone snapshot snap1 - // Expected 1: Exception in returning snapshot - int ret = CloneOrRecover("Clone", testUser1_, snapId, - "/ItUser2/SnapLazyClone1", true, &uuid2); - ASSERT_EQ(kErrCodeInvalidSnapshot, ret); - - // Step2: Non lazy clone snapshot snap1 - // Expected 2: Exception in returning snapshot - ret = CloneOrRecover("Clone", testUser1_, snapId, - "/ItUser2/SnapNotLazyClone1", false, &uuid2); - ASSERT_EQ(kErrCodeInvalidSnapshot, ret); - - // Step3: lazy snap1 recover from snapshot - // Expected 3: Exception in returning snapshot - ret = - CloneOrRecover("Recover", testUser1_, snapId, testFile4_, true, &uuid2); - ASSERT_EQ(kErrCodeInvalidSnapshot, ret); - - // Step4: Snap1 recover from snapshot without lazy - // Expected 4: Exception in returning snapshot - ret = CloneOrRecover("Recover", testUser1_, snapId, testFile4_, false, - &uuid2); - ASSERT_EQ(kErrCodeInvalidSnapshot, ret); - - // Step5: User testUser1_ For testFile4_ Take a snapshot snap1 - // Expectation 5: Clean failed snapshot and take snapshot successfully - ret = MakeSnapshot(testUser1_, testFile4_, "snap1", &uuid1); - ASSERT_EQ(0, ret); - - // Successfully verified snapshot - bool success1 = CheckSnapshotSuccess(testUser1_, testFile4_, uuid1); - ASSERT_TRUE(success1); - - // Verification cleaning failed, snapshot succeeded - FileSnapshotInfo info1; - int retCode = GetSnapshotInfo(testUser1_, testFile4_, snapId, &info1); - ASSERT_EQ(kErrCodeFileNotExist, retCode); -} - -//[Online issue repair] Clone failed, rollback delete clone volume, and create -//the same uuid volume again scenario -TEST_F(SnapshotCloneServerTest, TestCloneHasSameDest) { - std::string uuid1, uuid2, uuid3, uuid4, uuid5, uuid6, uuid7; - // Step1: testUser1_ Clone image testFile1_, fileName=CloneHasSameDestUUID - // Expected 1 to return successful cloning - std::string dstFile = "/ItUser1/CloneHasSameDest"; - int ret = - CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid1); - ASSERT_EQ(0, ret); - - // Delete Clone Volume - UserInfo_t userinfo; - userinfo.owner = testUser1_; - int ret2 = fileClient_->Unlink(dstFile, userinfo, false); - ASSERT_EQ(0, ret2); - - // Step2: testUser1_ Clone image testFile1_ again, - // fileName=CloneHasSameDestUUID - // Expected 2 to return successful cloning - ret = - CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid2); - ASSERT_EQ(0, ret); - - // Verify data correctness - std::string fakeData(4096, 'x'); - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); - - // Step3: testUser1_ Clone image testFile1_, fileName=CloneHasSameDest2 - // Expected 3 to return successful cloning - dstFile = "/ItUser1/CloneHasSameDest2"; - ret = - CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid3); - ASSERT_EQ(0, ret); - - // Delete Clone Volume - UserInfo_t userinfo2; - userinfo2.owner = testUser1_; - ret2 = fileClient_->Unlink(dstFile, userinfo2, false); - ASSERT_EQ(0, ret2); - - // Step4: testUser1_ Clone the image testFile2_ again, - // fileName=CloneHasSameDest2 - // Expected 4 to return successful cloning - ret = - CloneOrRecover("Clone", testUser1_, testFile2_, dstFile, true, &uuid4); - ASSERT_EQ(0, ret); - - // Verify data correctness - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); - - // Verify different situations when cloning lazyflag again - // Step5: testUser1_ Clone image testFile1_, fileName=CloneHasSameDest3 - // Expected 5 to return successful cloning - dstFile = "/ItUser1/CloneHasSameDest3"; - ret = - CloneOrRecover("Clone", testUser1_, testFile1_, dstFile, true, &uuid5); - ASSERT_EQ(0, ret); - - // Delete Clone Volume - UserInfo_t userinfo3; - userinfo2.owner = testUser1_; - ret2 = fileClient_->Unlink(dstFile, userinfo2, false); - ASSERT_EQ(0, ret2); - - // Step6: testUser1_ Non lazy clone image testFile2_ again, - // fileName=CloneHasSameDest3 - // Expected 6 to return successful cloning - ret = - CloneOrRecover("Clone", testUser1_, testFile2_, dstFile, false, &uuid6); - ASSERT_EQ(0, ret); - - bool success1 = CheckCloneOrRecoverSuccess(testUser1_, uuid6, true); - ASSERT_TRUE(success1); - - // Verify data correctness - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); - - // Delete Clone Volume - UserInfo_t userinfo4; - userinfo2.owner = testUser1_; - ret2 = fileClient_->Unlink(dstFile, userinfo2, false); - ASSERT_EQ(0, ret2); - - // Step7: testUser1_ Non lazy clone image testFile2_ again, - // fileName=CloneHasSameDest3 - // Expected 7 to return successful cloning - ret = - CloneOrRecover("Clone", testUser1_, testFile2_, dstFile, true, &uuid7); - ASSERT_EQ(0, ret); - - // Verify data correctness - ASSERT_TRUE(CheckFileData(dstFile, testUser1_, fakeData)); -} - -// Lazy clone volume, delete clone volume, and then delete source volume. The -// source volume can be deleted if needed -TEST_F(SnapshotCloneServerTest, TestDeleteLazyCloneDestThenDeleteSrc) { - // Step1: testUser1_ Clone image testFile5_, Lazy clone two volumes dstFile1 - // and dstFile2 Expected 1 to return successful cloning - std::string uuid1; - std::string uuid2; - std::string dstFile1 = "/dest1"; - std::string dstFile2 = "/dest2"; - UserInfo_t userinfo; - userinfo.owner = testUser1_; - int ret = - CloneOrRecover("Clone", testUser1_, testFile5_, dstFile1, true, &uuid1); - ASSERT_EQ(0, ret); - - ret = - CloneOrRecover("Clone", testUser1_, testFile5_, dstFile2, true, &uuid2); - ASSERT_EQ(0, ret); - - // Delete source volume, deletion failed, volume occupied - - ret = fileClient_->Unlink(testFile5_, userinfo, false); - ASSERT_EQ(-27, ret); - - // Step2: Successfully delete the destination volume dstFile1, delete the - // source volume again Expected 2 deletion failed, volume occupied - ret = fileClient_->Unlink(dstFile1, userinfo, false); - ASSERT_EQ(0, ret); - - ret = fileClient_->Unlink(testFile5_, userinfo, false); - ASSERT_EQ(-27, ret); - - // Step3: Successfully delete the destination volume dstFile2, delete the - // source volume again Expected 3 deletion successful - ret = fileClient_->Unlink(dstFile2, userinfo, false); - ASSERT_EQ(0, ret); - - ret = fileClient_->Unlink(testFile5_, userinfo, false); - ASSERT_EQ(0, ret); - - // Step4: Wait for a period of time to see if the garbage record can be - // deleted in the background - bool noRecord = false; - for (int i = 0; i < 100; i++) { - TaskCloneInfo info; - int ret1 = GetCloneTaskInfo(testUser1_, uuid1, &info); - int ret2 = GetCloneTaskInfo(testUser1_, uuid2, &info); - if (ret1 == kErrCodeFileNotExist && ret2 == kErrCodeFileNotExist) { - noRecord = true; - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(3000)); - } - - ASSERT_TRUE(noRecord); -} -} // namespace snapshotcloneserver -} // namespace curve + ASSERT_TRUE(noRecord); + } + } // namespace snapshotcloneserver +} // namespace curve diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp index 94d648ab86..9dd30a65b9 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_test.cpp @@ -25,8 +25,8 @@ #include #include -#include // NOLINT -#include // NOLINT +#include // NOLINT +#include // NOLINT #include "src/client/libcurve_file.h" #include "src/snapshotcloneserver/snapshotclone_server.h" @@ -34,35 +34,35 @@ #include "test/integration/snapshotcloneserver/test_snapshotcloneserver_helpler.h" #include "test/util/config_generator.h" -const std::string kTestPrefix = "MainSCSTest"; // NOLINT +const std::string kTestPrefix = "MainSCSTest"; // NOLINT // Some constant definitions -const char* cloneTempDir_ = "/clone"; -const char* mdsRootUser_ = "root"; -const char* mdsRootPassword_ = "root_password"; +const char *cloneTempDir_ = "/clone"; +const char *mdsRootUser_ = "root"; +const char *mdsRootPassword_ = "root_password"; const uint64_t segmentSize = 32ULL * 1024 * 1024; -const char* kEtcdClientIpPort = "127.0.0.1:10041"; -const char* kEtcdPeerIpPort = "127.0.0.1:10042"; -const char* kMdsIpPort = "127.0.0.1:10043"; -const char* kSnapshotCloneServerIpPort = "127.0.0.1:10047"; +const char *kEtcdClientIpPort = "127.0.0.1:10041"; +const char *kEtcdPeerIpPort = "127.0.0.1:10042"; +const char *kMdsIpPort = "127.0.0.1:10043"; +const char *kSnapshotCloneServerIpPort = "127.0.0.1:10047"; const int kMdsDummyPort = 10048; -const char* kSnapshotCloneServerDummyServerPort = "12004"; -const char* kLeaderCampaginPrefix = "snapshotcloneserverleaderlock4"; +const char *kSnapshotCloneServerDummyServerPort = "12004"; +const char *kLeaderCampaginPrefix = "snapshotcloneserverleaderlock4"; -const std::string kLogPath = "./runlog/" + kTestPrefix + "Log"; // NOLINT -const std::string kMdsDbName = kTestPrefix + "DB"; // NOLINT -const std::string kEtcdName = kTestPrefix; // NOLINT -const std::string kMdsConfigPath = // NOLINT +const std::string kLogPath = "./runlog/" + kTestPrefix + "Log"; // NOLINT +const std::string kMdsDbName = kTestPrefix + "DB"; // NOLINT +const std::string kEtcdName = kTestPrefix; // NOLINT +const std::string kMdsConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_mds.conf"; -const std::string kSnapClientConfigPath = // NOLINT +const std::string kSnapClientConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_snap_client.conf"; -const std::string kS3ConfigPath = // NOLINT +const std::string kS3ConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_s3.conf"; -const std::string kSCSConfigPath = // NOLINT +const std::string kSCSConfigPath = // NOLINT "./test/integration/snapshotcloneserver/config/" + kTestPrefix + "_scs.conf"; @@ -122,104 +122,110 @@ const std::vector snapshotcloneConf{ {"--stderrthreshold=3"}, }; -namespace curve { -namespace snapshotcloneserver { - -class SnapshotCloneServerMainTest : public ::testing::Test { - public: - void SetUp() { - std::string mkLogDirCmd = std::string("mkdir -p ") + kLogPath; - system(mkLogDirCmd.c_str()); - system("mkdir -p /data/log/curve ./fakes3"); - - cluster_ = new CurveCluster(); - ASSERT_NE(nullptr, cluster_); - - // Initialize db - std::string rmcmd = "rm -rf " + std::string(kEtcdName) + ".etcd"; - system(rmcmd.c_str()); - - // Start etcd - pid_t pid = cluster_->StartSingleEtcd( - 1, kEtcdClientIpPort, kEtcdPeerIpPort, - std::vector{"--name=" + std::string(kEtcdName)}); - LOG(INFO) << "etcd 1 started on " << kEtcdClientIpPort - << "::" << kEtcdPeerIpPort << ", pid = " << pid; - ASSERT_GT(pid, 0); - - cluster_->PrepareConfig(kMdsConfigPath, - mdsConfigOptions); - - // Start an mds - pid = cluster_->StartSingleMDS(1, kMdsIpPort, kMdsDummyPort, mdsConf1, - true); - LOG(INFO) << "mds 1 started on " << kMdsIpPort << ", pid = " << pid; - ASSERT_GT(pid, 0); - - cluster_->PrepareConfig(kS3ConfigPath, - s3ConfigOptions); - - cluster_->PrepareConfig( - kSnapClientConfigPath, snapClientConfigOptions); - - cluster_->PrepareConfig( - kSCSConfigPath, snapshotcloneserverConfigOptions); - } - - void TearDown() { - ASSERT_EQ(0, cluster_->StopCluster()); - delete cluster_; - cluster_ = nullptr; - - std::string rmcmd = "rm -rf " + std::string(kEtcdName) + ".etcd"; - system(rmcmd.c_str()); - } - - public: - CurveCluster* cluster_; -}; - -TEST_F(SnapshotCloneServerMainTest, testmain) { - std::shared_ptr conf = std::make_shared(); - conf->SetConfigPath(kSCSConfigPath); - - ASSERT_TRUE(conf->LoadConfig()); - LOG(INFO) << kSCSConfigPath; - conf->PrintConfig(); - - SnapShotCloneServer* snapshotCloneServer = new SnapShotCloneServer(conf); - - snapshotCloneServer->InitAllSnapshotCloneOptions(); - - snapshotCloneServer->StartDummy(); - - snapshotCloneServer->StartCompaginLeader(); - - ASSERT_TRUE(snapshotCloneServer->Init()); - - ASSERT_TRUE(snapshotCloneServer->Start()); - - std::this_thread::sleep_for(std::chrono::seconds(2)); - - // Test and verify if the status is active - // "curl "127.0.0.1:port/vars/snapshotcloneserver_status""; - std::string cmd = - "curl \"127.0.0.1:" + std::string(kSnapshotCloneServerDummyServerPort) + - "/vars/" + std::string(statusMetricName) + "\""; - // snapshotcloneserver_status : "active\r\n" - std::string expectResult = std::string(statusMetricName) + " : \"" + - std::string(ACTIVE) + "\"\r\n"; - - FILE* fp = popen(cmd.c_str(), "r"); - ASSERT_TRUE(fp != nullptr); - char buf[1024]; - fread(buf, sizeof(char), sizeof(buf), fp); - pclose(fp); - std::string result(buf); - ASSERT_EQ(result, expectResult); - - snapshotCloneServer->Stop(); - LOG(INFO) << "snapshotCloneServer Stopped"; -} -} // namespace snapshotcloneserver -} // namespace curve +namespace curve +{ + namespace snapshotcloneserver + { + + class SnapshotCloneServerMainTest : public ::testing::Test + { + public: + void SetUp() + { + std::string mkLogDirCmd = std::string("mkdir -p ") + kLogPath; + system(mkLogDirCmd.c_str()); + system("mkdir -p /data/log/curve ./fakes3"); + + cluster_ = new CurveCluster(); + ASSERT_NE(nullptr, cluster_); + + // Initialize db + std::string rmcmd = "rm -rf " + std::string(kEtcdName) + ".etcd"; + system(rmcmd.c_str()); + + // Start etcd + pid_t pid = cluster_->StartSingleEtcd( + 1, kEtcdClientIpPort, kEtcdPeerIpPort, + std::vector{"--name=" + std::string(kEtcdName)}); + LOG(INFO) << "etcd 1 started on " << kEtcdClientIpPort + << "::" << kEtcdPeerIpPort << ", pid = " << pid; + ASSERT_GT(pid, 0); + + cluster_->PrepareConfig(kMdsConfigPath, + mdsConfigOptions); + + // Start an mds + pid = cluster_->StartSingleMDS(1, kMdsIpPort, kMdsDummyPort, mdsConf1, + true); + LOG(INFO) << "mds 1 started on " << kMdsIpPort << ", pid = " << pid; + ASSERT_GT(pid, 0); + + cluster_->PrepareConfig(kS3ConfigPath, + s3ConfigOptions); + + cluster_->PrepareConfig( + kSnapClientConfigPath, snapClientConfigOptions); + + cluster_->PrepareConfig( + kSCSConfigPath, snapshotcloneserverConfigOptions); + } + + void TearDown() + { + ASSERT_EQ(0, cluster_->StopCluster()); + delete cluster_; + cluster_ = nullptr; + + std::string rmcmd = "rm -rf " + std::string(kEtcdName) + ".etcd"; + system(rmcmd.c_str()); + } + + public: + CurveCluster *cluster_; + }; + + TEST_F(SnapshotCloneServerMainTest, testmain) + { + std::shared_ptr conf = std::make_shared(); + conf->SetConfigPath(kSCSConfigPath); + + ASSERT_TRUE(conf->LoadConfig()); + LOG(INFO) << kSCSConfigPath; + conf->PrintConfig(); + + SnapShotCloneServer *snapshotCloneServer = new SnapShotCloneServer(conf); + + snapshotCloneServer->InitAllSnapshotCloneOptions(); + + snapshotCloneServer->StartDummy(); + + snapshotCloneServer->StartCompaginLeader(); + + ASSERT_TRUE(snapshotCloneServer->Init()); + + ASSERT_TRUE(snapshotCloneServer->Start()); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + // Test and verify if the status is active + // "curl "127.0.0.1:port/vars/snapshotcloneserver_status""; + std::string cmd = + "curl \"127.0.0.1:" + std::string(kSnapshotCloneServerDummyServerPort) + + "/vars/" + std::string(statusMetricName) + "\""; + // snapshotcloneserver_status : "active\r\n" + std::string expectResult = std::string(statusMetricName) + " : \"" + + std::string(ACTIVE) + "\"\r\n"; + + FILE *fp = popen(cmd.c_str(), "r"); + ASSERT_TRUE(fp != nullptr); + char buf[1024]; + fread(buf, sizeof(char), sizeof(buf), fp); + pclose(fp); + std::string result(buf); + ASSERT_EQ(result, expectResult); + + snapshotCloneServer->Stop(); + LOG(INFO) << "snapshotCloneServer Stopped"; + } + } // namespace snapshotcloneserver +} // namespace curve diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp index 2a388c8944..d7a49f2c8b 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_helper_test.cpp @@ -41,152 +41,158 @@ using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTINFOKEYEND; using ::curve::common::SEGMENTINFOKEYPREFIX; -namespace curve { -namespace mds { -TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) { - auto mockEtcdClient = std::make_shared(); - +namespace curve +{ + namespace mds { - // 1. list failed - EXPECT_CALL(*mockEtcdClient, - List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, - Matcher*>(_))) - .WillOnce(Return(EtcdErrCode::EtcdCanceled)); - std::map out; - ASSERT_EQ(-1, AllocStatisticHelper::GetExistSegmentAllocValues( - &out, mockEtcdClient)); - } + TEST(TestAllocStatisticHelper, test_GetExistSegmentAllocValues) + { + auto mockEtcdClient = std::make_shared(); - { - // 2. list successful, parsing failed - std::vector values{"hello"}; - EXPECT_CALL(*mockEtcdClient, - List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, - Matcher*>(_))) - .WillOnce( - DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); - std::map out; - ASSERT_EQ(0, AllocStatisticHelper::GetExistSegmentAllocValues( - &out, mockEtcdClient)); - } - { - // 3. Successfully obtained the existing segment alloc value - std::vector values{ - NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; - EXPECT_CALL(*mockEtcdClient, - List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, - Matcher*>(_))) - .WillOnce( - DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); - std::map out; - ASSERT_EQ(0, AllocStatisticHelper::GetExistSegmentAllocValues( - &out, mockEtcdClient)); - ASSERT_EQ(1, out.size()); - ASSERT_EQ(1024, out[1]); - } -} + { + // 1. list failed + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher *>(_))) + .WillOnce(Return(EtcdErrCode::EtcdCanceled)); + std::map out; + ASSERT_EQ(-1, AllocStatisticHelper::GetExistSegmentAllocValues( + &out, mockEtcdClient)); + } -TEST(TestAllocStatisticHelper, test_CalculateSegmentAlloc) { - auto mockEtcdClient = std::make_shared(); - { - // 1. CalculateSegmentAlloc ok - LOG(INFO) << "start test1......"; - EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( - SEGMENTINFOKEYPREFIX, - SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) - .WillOnce(Return(EtcdErrCode::EtcdUnknown)); - std::map out; - ASSERT_EQ(-1, AllocStatisticHelper::CalculateSegmentAlloc( - 2, mockEtcdClient, &out)); - } - { - // 2. ListWithLimitAndRevision succeeded, but parsing failed - LOG(INFO) << "start test2......"; - std::vector values{"hello"}; - std::string lastKey = "021"; - EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( - SEGMENTINFOKEYPREFIX, - SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) - .WillOnce( - DoAll(SetArgPointee<4>(values), Return(EtcdErrCode::EtcdOK))); - std::map out; - ASSERT_EQ(-1, AllocStatisticHelper::CalculateSegmentAlloc( - 2, mockEtcdClient, &out)); - } - { - // 3. ListWithLimitAndRevision successful, parsing successful, - // bundle=1000, number obtained is 1 - LOG(INFO) << "start test3......"; - PageFileSegment segment; - segment.set_segmentsize(1 << 30); - segment.set_logicalpoolid(1); - segment.set_chunksize(16 * 1024 * 1024); - segment.set_startoffset(0); - std::string encodeSegment; - ASSERT_TRUE( - NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - std::vector values{encodeSegment}; - std::string lastKey = - NameSpaceStorageCodec::EncodeSegmentStoreKey(1, 0); - EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( - SEGMENTINFOKEYPREFIX, - SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) - .WillOnce(DoAll(SetArgPointee<4>(values), SetArgPointee<5>(lastKey), - Return(EtcdErrCode::EtcdOK))); - std::map out; - ASSERT_EQ(0, AllocStatisticHelper::CalculateSegmentAlloc( - 2, mockEtcdClient, &out)); - ASSERT_EQ(1, out.size()); - ASSERT_EQ(1 << 30, out[1]); - } - { - // 4. ListWithLimitAndRevision successful, parsing successful - // bundle=1000, get a number of 1001 - LOG(INFO) << "start test4......"; - PageFileSegment segment; - segment.set_segmentsize(1 << 30); - segment.set_logicalpoolid(1); - segment.set_chunksize(16 * 1024 * 1024); - segment.set_startoffset(0); - std::string encodeSegment; - std::vector values; - ASSERT_TRUE( - NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - for (int i = 1; i <= 500; i++) { - values.emplace_back(encodeSegment); + { + // 2. list successful, parsing failed + std::vector values{"hello"}; + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher *>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); + std::map out; + ASSERT_EQ(0, AllocStatisticHelper::GetExistSegmentAllocValues( + &out, mockEtcdClient)); + } + { + // 3. Successfully obtained the existing segment alloc value + std::vector values{ + NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; + EXPECT_CALL(*mockEtcdClient, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher *>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); + std::map out; + ASSERT_EQ(0, AllocStatisticHelper::GetExistSegmentAllocValues( + &out, mockEtcdClient)); + ASSERT_EQ(1, out.size()); + ASSERT_EQ(1024, out[1]); + } } - segment.set_logicalpoolid(2); - ASSERT_TRUE( - NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - for (int i = 501; i <= 1000; i++) { - values.emplace_back(encodeSegment); - } - std::string lastKey1 = - NameSpaceStorageCodec::EncodeSegmentStoreKey(1, 500); - std::string lastKey2 = - NameSpaceStorageCodec::EncodeSegmentStoreKey(501, 1000); - EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( - SEGMENTINFOKEYPREFIX, - SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) - .WillOnce(DoAll(SetArgPointee<4>(values), - SetArgPointee<5>(lastKey1), - Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient, - ListWithLimitAndRevision(lastKey1, SEGMENTINFOKEYEND, - GETBUNDLE, 2, _, _)) - .WillOnce(DoAll(SetArgPointee<4>(std::vector{ - encodeSegment, encodeSegment}), - SetArgPointee<5>(lastKey2), - Return(EtcdErrCode::EtcdOK))); + TEST(TestAllocStatisticHelper, test_CalculateSegmentAlloc) + { + auto mockEtcdClient = std::make_shared(); + { + // 1. CalculateSegmentAlloc ok + LOG(INFO) << "start test1......"; + EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( + SEGMENTINFOKEYPREFIX, + SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) + .WillOnce(Return(EtcdErrCode::EtcdUnknown)); + std::map out; + ASSERT_EQ(-1, AllocStatisticHelper::CalculateSegmentAlloc( + 2, mockEtcdClient, &out)); + } + { + // 2. ListWithLimitAndRevision succeeded, but parsing failed + LOG(INFO) << "start test2......"; + std::vector values{"hello"}; + std::string lastKey = "021"; + EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( + SEGMENTINFOKEYPREFIX, + SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) + .WillOnce( + DoAll(SetArgPointee<4>(values), Return(EtcdErrCode::EtcdOK))); + std::map out; + ASSERT_EQ(-1, AllocStatisticHelper::CalculateSegmentAlloc( + 2, mockEtcdClient, &out)); + } + { + // 3. ListWithLimitAndRevision successful, parsing successful, + // bundle=1000, number obtained is 1 + LOG(INFO) << "start test3......"; + PageFileSegment segment; + segment.set_segmentsize(1 << 30); + segment.set_logicalpoolid(1); + segment.set_chunksize(16 * 1024 * 1024); + segment.set_startoffset(0); + std::string encodeSegment; + ASSERT_TRUE( + NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); + std::vector values{encodeSegment}; + std::string lastKey = + NameSpaceStorageCodec::EncodeSegmentStoreKey(1, 0); + EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( + SEGMENTINFOKEYPREFIX, + SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(values), SetArgPointee<5>(lastKey), + Return(EtcdErrCode::EtcdOK))); + std::map out; + ASSERT_EQ(0, AllocStatisticHelper::CalculateSegmentAlloc( + 2, mockEtcdClient, &out)); + ASSERT_EQ(1, out.size()); + ASSERT_EQ(1 << 30, out[1]); + } + { + // 4. ListWithLimitAndRevision successful, parsing successful + // bundle=1000, get a number of 1001 + LOG(INFO) << "start test4......"; + PageFileSegment segment; + segment.set_segmentsize(1 << 30); + segment.set_logicalpoolid(1); + segment.set_chunksize(16 * 1024 * 1024); + segment.set_startoffset(0); + std::string encodeSegment; + std::vector values; + ASSERT_TRUE( + NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); + for (int i = 1; i <= 500; i++) + { + values.emplace_back(encodeSegment); + } - std::map out; - ASSERT_EQ(0, AllocStatisticHelper::CalculateSegmentAlloc( - 2, mockEtcdClient, &out)); - ASSERT_EQ(2, out.size()); - ASSERT_EQ(500L * (1 << 30), out[1]); - ASSERT_EQ(501L * (1 << 30), out[2]); - } -} -} // namespace mds -} // namespace curve + segment.set_logicalpoolid(2); + ASSERT_TRUE( + NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); + for (int i = 501; i <= 1000; i++) + { + values.emplace_back(encodeSegment); + } + std::string lastKey1 = + NameSpaceStorageCodec::EncodeSegmentStoreKey(1, 500); + std::string lastKey2 = + NameSpaceStorageCodec::EncodeSegmentStoreKey(501, 1000); + EXPECT_CALL(*mockEtcdClient, ListWithLimitAndRevision( + SEGMENTINFOKEYPREFIX, + SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(values), + SetArgPointee<5>(lastKey1), + Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*mockEtcdClient, + ListWithLimitAndRevision(lastKey1, SEGMENTINFOKEYEND, + GETBUNDLE, 2, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(std::vector{ + encodeSegment, encodeSegment}), + SetArgPointee<5>(lastKey2), + Return(EtcdErrCode::EtcdOK))); + + std::map out; + ASSERT_EQ(0, AllocStatisticHelper::CalculateSegmentAlloc( + 2, mockEtcdClient, &out)); + ASSERT_EQ(2, out.size()); + ASSERT_EQ(500L * (1 << 30), out[1]); + ASSERT_EQ(501L * (1 << 30), out[2]); + } + } + } // namespace mds +} // namespace curve diff --git a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp index f250e7e401..3a4b579852 100644 --- a/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp +++ b/test/mds/nameserver2/allocstatistic/alloc_statistic_test.cpp @@ -28,205 +28,212 @@ #include "src/common/namespace_define.h" using ::testing::_; -using ::testing::Return; -using ::testing::SetArgPointee; using ::testing::DoAll; using ::testing::Matcher; +using ::testing::Return; +using ::testing::SetArgPointee; -using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTALLOCSIZEKEY; +using ::curve::common::SEGMENTALLOCSIZEKEYEND; using ::curve::common::SEGMENTINFOKEYEND; using ::curve::common::SEGMENTINFOKEYPREFIX; -namespace curve { -namespace mds { - -class AllocStatisticTest : public ::testing::Test { - protected: - void SetUp() override { - periodicPersistInterMs_ = 2; - retryInterMs_ = 2; - mockEtcdClient_ = std::make_shared(); - allocStatistic_ = std::make_shared( - periodicPersistInterMs_, retryInterMs_, mockEtcdClient_); - } - - protected: - int64_t periodicPersistInterMs_; - int64_t retryInterMs_; - std::shared_ptr allocStatistic_; - std::shared_ptr mockEtcdClient_; -}; - -TEST_F(AllocStatisticTest, test_Init) { - { - // 1. Failed to obtain the current revision from ETCD - LOG(INFO) << "test1......"; - EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). - WillOnce(Return(EtcdErrCode::EtcdCanceled)); - ASSERT_EQ(-1, allocStatistic_->Init()); - } +namespace curve +{ + namespace mds { - // 2. Failed to obtain the alloc size corresponding to the existing logicalPool - LOG(INFO) << "test2......"; - EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). - WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, - List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, - Matcher*>(_))) - .WillOnce(Return(EtcdErrCode::EtcdCanceled)); - ASSERT_EQ(-1, allocStatistic_->Init()); - int64_t alloc; - ASSERT_FALSE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); - } - { - // 3. init successful - LOG(INFO) << "test3......"; - std::vector values{ - NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; - EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)). - WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, - List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, - Matcher*>(_))) - .WillOnce( - DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); - ASSERT_EQ(0, allocStatistic_->Init()); - int64_t alloc; - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); - ASSERT_EQ(1024, alloc); - } -} - -TEST_F(AllocStatisticTest, test_PeriodicPersist_CalculateSegmentAlloc) { - // Initialize allocStatistics - // Old value: logicalPooId(1):1024 - std::vector values{ - NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; - EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) - .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, - List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, - Matcher*>(_))) - .WillOnce(DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); - ASSERT_EQ(0, allocStatistic_->Init()); - - PageFileSegment segment; - segment.set_segmentsize(1 << 30); - segment.set_logicalpoolid(1); - segment.set_chunksize(16*1024*1024); - segment.set_startoffset(0); - std::string encodeSegment; - values.clear(); - ASSERT_TRUE( - NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - for (int i = 1; i <= 500; i++) { - values.emplace_back(encodeSegment); - } - - // 1 Only old values can be obtained before regular persistent threads and statistical threads are started - int64_t alloc; - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); - ASSERT_EQ(1024, alloc); - ASSERT_FALSE(allocStatistic_->GetAllocByLogicalPool(2, &alloc)); - - // 2 Update the value of segment - allocStatistic_->DeAllocSpace(1, 64, 1); - allocStatistic_->AllocSpace(1, 32, 1); - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); - ASSERT_EQ(1024 - 32, alloc); - - // Set the value of segment in the ETCD of the mock - // logicalPoolId(1):500 * (1<<30) - // logicalPoolId(2):501 * (1<<30) - segment.set_logicalpoolid(2); - ASSERT_TRUE( - NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); - for (int i = 501; i <= 1000; i++) { - values.emplace_back(encodeSegment); - } - std::string lastKey1 = - NameSpaceStorageCodec::EncodeSegmentStoreKey(1, 500); - std::string lastKey2 = - NameSpaceStorageCodec::EncodeSegmentStoreKey(501, 1000); - EXPECT_CALL(*mockEtcdClient_, ListWithLimitAndRevision( - SEGMENTINFOKEYPREFIX, SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) - .Times(2) - .WillOnce(Return(EtcdErrCode::EtcdCanceled)) - .WillOnce(DoAll(SetArgPointee<4>(values), - SetArgPointee<5>(lastKey1), - Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, ListWithLimitAndRevision( - lastKey1, SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) - .WillOnce(DoAll(SetArgPointee<4>( - std::vector{encodeSegment, encodeSegment}), - SetArgPointee<5>(lastKey2), - Return(EtcdErrCode::EtcdOK))); - EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) - .Times(2) - .WillOnce(Return(EtcdErrCode::EtcdCanceled)) - .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); - - // Set the Put result of the mock - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(1), - NameSpaceStorageCodec::EncodeSegmentAllocValue( - 1, 1024 - 32 + (1L << 30)))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(2), - NameSpaceStorageCodec::EncodeSegmentAllocValue(2, 1L << 30))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(1), - NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 501L *(1 << 30)))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(2), - NameSpaceStorageCodec::EncodeSegmentAllocValue(2, 502L *(1 << 30)))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(1), - NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 500L *(1 << 30)))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(2), - NameSpaceStorageCodec::EncodeSegmentAllocValue(2, 501L *(1 << 30)))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - EXPECT_CALL(*mockEtcdClient_, Put( - NameSpaceStorageCodec::EncodeSegmentAllocKey(3), - NameSpaceStorageCodec::EncodeSegmentAllocValue(3, 1L << 30))) - .WillOnce(Return(EtcdErrCode::EtcdOK)); - - // 2 Start regular persistence and statistics threads - for (int i = 1; i <= 2; i++) { - allocStatistic_->AllocSpace(i, 1L << 30, i + 3); - } - allocStatistic_->Run(); - std::this_thread::sleep_for(std::chrono::seconds(6)); - - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); - ASSERT_EQ(501L *(1 << 30), alloc); - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(2, &alloc)); - ASSERT_EQ(502L *(1 << 30), alloc); - std::this_thread::sleep_for(std::chrono::milliseconds(30)); - - // Update through alloc again - for (int i = 1; i <= 2; i++) { - allocStatistic_->DeAllocSpace(i, 1L << 30, i + 4); - } - allocStatistic_->AllocSpace(3, 1L << 30, 10); - - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); - ASSERT_EQ(500L *(1 << 30), alloc); - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(2, &alloc)); - ASSERT_EQ(501L *(1 << 30), alloc); - ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(3, &alloc)); - ASSERT_EQ(1L << 30, alloc); - std::this_thread::sleep_for(std::chrono::milliseconds(30)); - - allocStatistic_->Stop(); -} - -} // namespace mds -} // namespace curve + + class AllocStatisticTest : public ::testing::Test + { + protected: + void SetUp() override + { + periodicPersistInterMs_ = 2; + retryInterMs_ = 2; + mockEtcdClient_ = std::make_shared(); + allocStatistic_ = std::make_shared( + periodicPersistInterMs_, retryInterMs_, mockEtcdClient_); + } + + protected: + int64_t periodicPersistInterMs_; + int64_t retryInterMs_; + std::shared_ptr allocStatistic_; + std::shared_ptr mockEtcdClient_; + }; + + TEST_F(AllocStatisticTest, test_Init) + { + { + // 1. Failed to obtain the current revision from ETCD + LOG(INFO) << "test1......"; + EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)).WillOnce(Return(EtcdErrCode::EtcdCanceled)); + ASSERT_EQ(-1, allocStatistic_->Init()); + } + { + // 2. Failed to obtain the alloc size corresponding to the existing logicalPool + LOG(INFO) << "test2......"; + EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)).WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher *>(_))) + .WillOnce(Return(EtcdErrCode::EtcdCanceled)); + ASSERT_EQ(-1, allocStatistic_->Init()); + int64_t alloc; + ASSERT_FALSE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); + } + { + // 3. init successful + LOG(INFO) << "test3......"; + std::vector values{ + NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; + EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)).WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher *>(_))) + .WillOnce( + DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); + ASSERT_EQ(0, allocStatistic_->Init()); + int64_t alloc; + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); + ASSERT_EQ(1024, alloc); + } + } + + TEST_F(AllocStatisticTest, test_PeriodicPersist_CalculateSegmentAlloc) + { + // Initialize allocStatistics + // Old value: logicalPooId(1):1024 + std::vector values{ + NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 1024)}; + EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) + .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*mockEtcdClient_, + List(SEGMENTALLOCSIZEKEY, SEGMENTALLOCSIZEKEYEND, + Matcher *>(_))) + .WillOnce(DoAll(SetArgPointee<2>(values), Return(EtcdErrCode::EtcdOK))); + ASSERT_EQ(0, allocStatistic_->Init()); + + PageFileSegment segment; + segment.set_segmentsize(1 << 30); + segment.set_logicalpoolid(1); + segment.set_chunksize(16 * 1024 * 1024); + segment.set_startoffset(0); + std::string encodeSegment; + values.clear(); + ASSERT_TRUE( + NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); + for (int i = 1; i <= 500; i++) + { + values.emplace_back(encodeSegment); + } + + // 1 Only old values can be obtained before regular persistent threads and statistical threads are started + int64_t alloc; + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); + ASSERT_EQ(1024, alloc); + ASSERT_FALSE(allocStatistic_->GetAllocByLogicalPool(2, &alloc)); + + // 2 Update the value of segment + allocStatistic_->DeAllocSpace(1, 64, 1); + allocStatistic_->AllocSpace(1, 32, 1); + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); + ASSERT_EQ(1024 - 32, alloc); + + // Set the value of segment in the ETCD of the mock + // logicalPoolId(1):500 * (1<<30) + // logicalPoolId(2):501 * (1<<30) + segment.set_logicalpoolid(2); + ASSERT_TRUE( + NameSpaceStorageCodec::EncodeSegment(segment, &encodeSegment)); + for (int i = 501; i <= 1000; i++) + { + values.emplace_back(encodeSegment); + } + std::string lastKey1 = + NameSpaceStorageCodec::EncodeSegmentStoreKey(1, 500); + std::string lastKey2 = + NameSpaceStorageCodec::EncodeSegmentStoreKey(501, 1000); + EXPECT_CALL(*mockEtcdClient_, ListWithLimitAndRevision( + SEGMENTINFOKEYPREFIX, SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) + .Times(2) + .WillOnce(Return(EtcdErrCode::EtcdCanceled)) + .WillOnce(DoAll(SetArgPointee<4>(values), + SetArgPointee<5>(lastKey1), + Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*mockEtcdClient_, ListWithLimitAndRevision( + lastKey1, SEGMENTINFOKEYEND, GETBUNDLE, 2, _, _)) + .WillOnce(DoAll(SetArgPointee<4>( + std::vector{encodeSegment, encodeSegment}), + SetArgPointee<5>(lastKey2), + Return(EtcdErrCode::EtcdOK))); + EXPECT_CALL(*mockEtcdClient_, GetCurrentRevision(_)) + .Times(2) + .WillOnce(Return(EtcdErrCode::EtcdCanceled)) + .WillOnce(DoAll(SetArgPointee<0>(2), Return(EtcdErrCode::EtcdOK))); + + // Set the Put result of the mock + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(1), + NameSpaceStorageCodec::EncodeSegmentAllocValue( + 1, 1024 - 32 + (1L << 30)))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(2), + NameSpaceStorageCodec::EncodeSegmentAllocValue(2, 1L << 30))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(1), + NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 501L * (1 << 30)))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(2), + NameSpaceStorageCodec::EncodeSegmentAllocValue(2, 502L * (1 << 30)))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(1), + NameSpaceStorageCodec::EncodeSegmentAllocValue(1, 500L * (1 << 30)))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(2), + NameSpaceStorageCodec::EncodeSegmentAllocValue(2, 501L * (1 << 30)))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + EXPECT_CALL(*mockEtcdClient_, Put( + NameSpaceStorageCodec::EncodeSegmentAllocKey(3), + NameSpaceStorageCodec::EncodeSegmentAllocValue(3, 1L << 30))) + .WillOnce(Return(EtcdErrCode::EtcdOK)); + + // 2 Start regular persistence and statistics threads + for (int i = 1; i <= 2; i++) + { + allocStatistic_->AllocSpace(i, 1L << 30, i + 3); + } + allocStatistic_->Run(); + std::this_thread::sleep_for(std::chrono::seconds(6)); + + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); + ASSERT_EQ(501L * (1 << 30), alloc); + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(2, &alloc)); + ASSERT_EQ(502L * (1 << 30), alloc); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + + // Update through alloc again + for (int i = 1; i <= 2; i++) + { + allocStatistic_->DeAllocSpace(i, 1L << 30, i + 4); + } + allocStatistic_->AllocSpace(3, 1L << 30, 10); + + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(1, &alloc)); + ASSERT_EQ(500L * (1 << 30), alloc); + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(2, &alloc)); + ASSERT_EQ(501L * (1 << 30), alloc); + ASSERT_TRUE(allocStatistic_->GetAllocByLogicalPool(3, &alloc)); + ASSERT_EQ(1L << 30, alloc); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + + allocStatistic_->Stop(); + } + + } // namespace mds +} // namespace curve