Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feat]tools-v2: improve the snapshot tool #2793

Merged
merged 1 commit into from
Nov 1, 2023
Merged

[feat]tools-v2: improve the snapshot tool #2793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions tools-v2/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ A tool for CurveFS & CurveBs.
- [update scan-state](#update-scan-state)
- [update copyset availflag](#update-copyset-availflag)
- [update leader-schedule](#update-leader-schedule)
- [update volume flatten](#update-volume-flatten)
- [create](#create-1)
- [create file](#create-file)
- [create dir](#create-dir)
Expand Down Expand Up @@ -1620,11 +1621,11 @@ curve bs delete volume clone
Output:

```bash
+------+--------------------------------------+--------------------------------------+-------+---------+
| USER | SRC | TASK ID | FILE | RESULT |
+------+--------------------------------------+--------------------------------------+-------+---------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | d26e27a8-fcbd-4f7a-adf8-53795217cbb0 | /root | success |
+------+--------------------------------------+--------------------------------------+-------+---------+
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| USER | SRC | TASK ID | FILE | RESULT | REASON |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | d26e27a8-fcbd-4f7a-adf8-53795217cbb0 | /root | success | |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
```

###### delete volume recover
Expand All @@ -1640,11 +1641,11 @@ curve bs delete volume recover
Output:

```bash
+------+--------------------------------------+--------------------------------------+-------+---------+
| USER | SRC | TASK ID | FILE | RESULT |
+------+--------------------------------------+--------------------------------------+-------+---------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | 9dfa8699-a275-4891-8ec2-e447a0ccc77c | /root | success |
+------+--------------------------------------+--------------------------------------+-------+---------+
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| USER | SRC | TASK ID | FILE | RESULT | REASON |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | 9dfa8699-a275-4891-8ec2-e447a0ccc77c | /root | success | |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
```

#### update
Expand Down
3 changes: 3 additions & 0 deletions tools-v2/internal/error/error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,9 @@ var (
ErrBsGetChunkHash = func() *CmdError {
return NewInternalCmdError(77, "get chunk hash fail, err: %s")
}
ErrBsListSnaspshot = func(requestId, code, message string) *CmdError {
return NewInternalCmdError(78, fmt.Sprintf("list snapshot fail, requestId: %s, code: %s, message: %s", requestId, code, message))
}

// http error
ErrHttpUnreadableResult = func() *CmdError {
Expand Down
37 changes: 1 addition & 36 deletions tools-v2/internal/utils/snapshot.go → tools-v2/internal/utils/snapshot/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,7 @@
* Author: baytan0720
*/

package cobrautil

import (
"fmt"
"net/url"
"strings"
)
package snapshot

const (
Version = "0.0.6"
Expand Down Expand Up @@ -61,32 +55,3 @@ const (
Limit = "100"
Offset = "0"
)

func NewSnapshotQuerySubUri(params map[string]any) string {
values := url.Values{}

values.Add(QueryVersion, Version)
for key, value := range params {
if value != "" {
values.Add(key, fmt.Sprintf("%s", value))
}
}

return "/SnapshotCloneService?" + values.Encode()
}

func NewSubUri(params map[string]any) string {
values := strings.Builder{}
for key, value := range params {
if value != "" {
values.WriteString(key)
values.WriteString("=")
values.WriteString(value.(string))
values.WriteString("&")
}
}
str := values.String()
encodedParams := str[:len(str)-1]
subUri := fmt.Sprintf("/SnapshotCloneService?%s", encodedParams)
return subUri
}
86 changes: 86 additions & 0 deletions tools-v2/internal/utils/snapshot/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2023 NetEase Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* Project: CurveCli
* Created Date: 2023-09-16
* Author: baytan0720
*/

package snapshot

import (
"fmt"
"strings"
)

func NewQuerySubUri(params map[string]any) string {
values := strings.Builder{}
for key, value := range params {
if value != "" && value != nil {
values.WriteString(key)
values.WriteString("=")
values.WriteString(fmt.Sprintf("%v", value))
values.WriteString("&")
}
}

return "/SnapshotCloneService?" + strings.TrimRight(values.String(), "&")
}

type Response struct {
Code string `json:"Code"`
Message string `json:"Message"`
RequestId string `json:"RequestId"`
}

type SnapshotInfo struct {
File string `json:"File"`
FileLength int `json:"FileLength"`
Name string `json:"Name"`
Progress float64 `json:"Progress"`
SeqNum int `json:"SeqNum"`
Status int `json:"status"`
Time int64 `json:"Time"`
UUID string `json:"UUID"`
User string `json:"User"`
}

type SnapshotInfos []SnapshotInfo

var SnapshotStatus = []string{"done", "in-progress", "deleting", "errorDeleting", "canceling", "error"}

type TaskInfo struct {
File string `json:"File"`
FileType int `json:"FileType"`
IsLazy bool `json:"IsLazy"`
NextStep int `json:"NextStep"`
Progress int `json:"Progress"`
Src string `json:"Src"`
TaskStatus int `json:"TaskStatus"`
TaskType int `json:"TaskType"`
Time int `json:"Time"`
UUID string `json:"UUID"`
User string `json:"User"`
}

type TaskInfos []TaskInfo

var TaskStatus = []string{"Done", "Cloning", "Recovering", "Cleaning", "ErrorCleaning", "Error", "Retrying", "MetaInstalled"}
var TaskType = []string{"clone", "recover"}
var FileType = []string{"file", "snapshot"}
var CloneStep = []string{"CreateCloneFile", "CreateCloneMeta", "CreateCloneChunk", "CompleteCloneMeta", "RecoverChunk", "ChangeOwner", "RenameCloneFile", "CompleteCloneFile", "End"}
var IsLazy = []string{"false", "true"}
85 changes: 49 additions & 36 deletions tools-v2/pkg/cli/command/curvebs/delete/volume/clone/clone.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (

cmderror "github.com/opencurve/curve/tools-v2/internal/error"
cobrautil "github.com/opencurve/curve/tools-v2/internal/utils"
snapshotutil "github.com/opencurve/curve/tools-v2/internal/utils/snapshot"
basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command"
"github.com/opencurve/curve/tools-v2/pkg/config"
"github.com/opencurve/curve/tools-v2/pkg/output"
Expand Down Expand Up @@ -72,80 +73,92 @@ func (rCmd *CloneCmd) Init(cmd *cobra.Command, args []string) error {
if rCmd.failed {
rCmd.status = "5"
}
rCmd.SetHeader([]string{cobrautil.ROW_USER, cobrautil.ROW_SRC, cobrautil.ROW_TASK_ID, cobrautil.ROW_FILE, cobrautil.ROW_RESULT})
rCmd.SetHeader([]string{cobrautil.ROW_USER, cobrautil.ROW_SRC, cobrautil.ROW_TASK_ID, cobrautil.ROW_FILE, cobrautil.ROW_RESULT, cobrautil.ROW_REASON})
return nil
}

func (rCmd *CloneCmd) RunCommand(cmd *cobra.Command, args []string) error {
params := map[string]any{
cobrautil.QueryAction: cobrautil.ActionGetCloneTaskList,
cobrautil.QueryType: cobrautil.TypeCloneTask,
cobrautil.QueryUser: rCmd.user,
cobrautil.QueryUUID: rCmd.taskID,
cobrautil.QuerySource: rCmd.src,
cobrautil.QueryDestination: rCmd.dest,
cobrautil.QueryStatus: rCmd.status,
cobrautil.QueryLimit: 100,
cobrautil.QueryOffset: 0,
snapshotutil.QueryAction: snapshotutil.ActionGetCloneTaskList,
snapshotutil.QueryType: snapshotutil.TypeCloneTask,
snapshotutil.QueryUser: rCmd.user,
snapshotutil.QueryUUID: rCmd.taskID,
snapshotutil.QuerySource: rCmd.src,
snapshotutil.QueryDestination: rCmd.dest,
snapshotutil.QueryStatus: rCmd.status,
snapshotutil.QueryLimit: 100,
snapshotutil.QueryOffset: 0,
}
records := make([]map[string]string, 0)
records := make(snapshotutil.TaskInfos, 0)
for {
subUri := cobrautil.NewSnapshotQuerySubUri(params)
subUri := snapshotutil.NewQuerySubUri(params)
metric := basecmd.NewMetric(rCmd.snapshotAddrs, subUri, rCmd.timeout)
result, err := basecmd.QueryMetric(metric)
if err.TypeCode() != cmderror.CODE_SUCCESS {
return err.ToError()
}

var resp struct {
Code string `json:"Code"`
TaskInfos []map[string]string `json:"TaskInfos"`
TotalCount int `json:"TotalCount"`
var payload struct {
snapshotutil.Response
TaskInfos snapshotutil.TaskInfos `json:"TaskInfos"`
TotalCount int `json:"TotalCount"`
}
if err := json.Unmarshal([]byte(result), &resp); err != nil {
if err := json.Unmarshal([]byte(result), &payload); err != nil {
return err
}
if resp.Code != cobrautil.ResultSuccess {
return fmt.Errorf("get clone list fail, error code: %s", resp.Code)
if payload.Code != snapshotutil.ResultSuccess {
return fmt.Errorf("get clone list fail, requestId: %s, code: %s, message: %s", payload.RequestId, payload.Code, payload.Message)
}
if len(resp.TaskInfos) == 0 {
if len(payload.TaskInfos) == 0 {
break
} else {
records = append(records, resp.TaskInfos...)
params[cobrautil.QueryOffset] = params[cobrautil.QueryOffset].(int) + params[cobrautil.QueryLimit].(int)
records = append(records, payload.TaskInfos...)
params[snapshotutil.QueryOffset] = params[snapshotutil.QueryOffset].(int) + params[snapshotutil.QueryLimit].(int)
}
}

wg := sync.WaitGroup{}
for _, item := range records {
wg.Add(1)
go func(item map[string]string) {
go func(clone snapshotutil.TaskInfo) {
defer wg.Done()
result := cobrautil.ROW_VALUE_SUCCESS
reason := ""
params := map[string]any{
cobrautil.QueryAction: cobrautil.ActionCleanCloneTask,
cobrautil.QueryUser: item["User"],
cobrautil.QueryUUID: item["UUID"],
snapshotutil.QueryAction: snapshotutil.ActionCleanCloneTask,
snapshotutil.QueryUser: clone.User,
snapshotutil.QueryUUID: clone.UUID,
}
subUri := cobrautil.NewSnapshotQuerySubUri(params)
subUri := snapshotutil.NewQuerySubUri(params)
metric := basecmd.NewMetric(rCmd.snapshotAddrs, subUri, rCmd.timeout)
result, err := basecmd.QueryMetric(metric)
retStr, err := basecmd.QueryMetric(metric)
if err.TypeCode() != cmderror.CODE_SUCCESS {
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_FAILED
result = cobrautil.ROW_VALUE_FAILED
reason = err.ToError().Error()
} else {
payload := map[string]any{}
if err := json.Unmarshal([]byte(result), &payload); err != nil {
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_FAILED
payload := snapshotutil.Response{}
if err := json.Unmarshal([]byte(retStr), &payload); err != nil {
result = cobrautil.ROW_VALUE_FAILED
reason = err.Error()
} else {
if payload[cobrautil.ResultCode] != cobrautil.ResultSuccess {
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_FAILED
if payload.Code != snapshotutil.ResultSuccess {
result = cobrautil.ROW_VALUE_FAILED
reason = payload.Message
}
}
}
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_SUCCESS
rCmd.TableNew.Append([]string{item["User"], item["Src"], item["UUID"], item["File"], item["Result"]})
rCmd.TableNew.Append(cobrautil.Map2List(map[string]string{
cobrautil.ROW_USER: clone.User,
cobrautil.ROW_SRC: clone.Src,
cobrautil.ROW_TASK_ID: clone.UUID,
cobrautil.ROW_FILE: clone.File,
cobrautil.ROW_RESULT: result,
cobrautil.ROW_REASON: reason,
}, rCmd.Header))
}(item)
}
wg.Wait()

rCmd.Result = records
rCmd.Error = cmderror.Success()
return nil
Expand Down
Loading