Skip to content

Commit

Permalink
[feat]tools-v2: improve the snapshot tool
Browse files Browse the repository at this point in the history 
Signed-off-by: baytan0720 <[email protected]>
  • Loading branch information
@baytan0720
baytan0720 committed Oct 31, 2023
1 parent e1aa430 commit 51b1936
Show file tree
Hide file tree
Showing 8 changed files with 229 additions and 161 deletions.
21 changes: 11 additions & 10 deletions tools-v2/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ A tool for CurveFS & CurveBs.
- [update scan-state](#update-scan-state)
- [update copyset availflag](#update-copyset-availflag)
- [update leader-schedule](#update-leader-schedule)
- [update volume flatten](#update-volume-flatten)
- [create](#create-1)
- [create file](#create-file)
- [create dir](#create-dir)
Expand Down Expand Up @@ -1620,11 +1621,11 @@ curve bs delete volume clone
Output:

```bash
+------+--------------------------------------+--------------------------------------+-------+---------+
| USER | SRC | TASK ID | FILE | RESULT |
+------+--------------------------------------+--------------------------------------+-------+---------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | d26e27a8-fcbd-4f7a-adf8-53795217cbb0 | /root | success |
+------+--------------------------------------+--------------------------------------+-------+---------+
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| USER | SRC | TASK ID | FILE | RESULT | REASON |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | d26e27a8-fcbd-4f7a-adf8-53795217cbb0 | /root | success | |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
```

###### delete volume recover
Expand All @@ -1640,11 +1641,11 @@ curve bs delete volume recover
Output:

```bash
+------+--------------------------------------+--------------------------------------+-------+---------+
| USER | SRC | TASK ID | FILE | RESULT |
+------+--------------------------------------+--------------------------------------+-------+---------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | 9dfa8699-a275-4891-8ec2-e447a0ccc77c | /root | success |
+------+--------------------------------------+--------------------------------------+-------+---------+
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| USER | SRC | TASK ID | FILE | RESULT | REASON |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
| root | a19b5e5e-b306-488f-8e6d-d87282c869cb | 9dfa8699-a275-4891-8ec2-e447a0ccc77c | /root | success | |
+------+--------------------------------------+--------------------------------------+-------+---------+--------+
```

#### update
Expand Down
3 changes: 3 additions & 0 deletions tools-v2/internal/error/error.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,9 @@ var (
ErrBsGetChunkHash = func() *CmdError {
return NewInternalCmdError(77, "get chunk hash fail, err: %s")
}
ErrBsListSnaspshot = func(requestId, code, message string) *CmdError {
return NewInternalCmdError(78, fmt.Sprintf("list snapshot fail, requestId: %s, code: %s, message: %s", requestId, code, message))
}

// http error
ErrHttpUnreadableResult = func() *CmdError {
Expand Down
37 changes: 1 addition & 36 deletions tools-v2/internal/utils/snapshot.go → tools-v2/internal/utils/snapshot/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,7 @@
* Author: baytan0720
*/

package cobrautil

import (
"fmt"
"net/url"
"strings"
)
package snapshot

const (
Version = "0.0.6"
Expand Down Expand Up @@ -61,32 +55,3 @@ const (
Limit = "100"
Offset = "0"
)

func NewSnapshotQuerySubUri(params map[string]any) string {
values := url.Values{}

values.Add(QueryVersion, Version)
for key, value := range params {
if value != "" {
values.Add(key, fmt.Sprintf("%s", value))
}
}

return "/SnapshotCloneService?" + values.Encode()
}

func NewSubUri(params map[string]any) string {
values := strings.Builder{}
for key, value := range params {
if value != "" {
values.WriteString(key)
values.WriteString("=")
values.WriteString(value.(string))
values.WriteString("&")
}
}
str := values.String()
encodedParams := str[:len(str)-1]
subUri := fmt.Sprintf("/SnapshotCloneService?%s", encodedParams)
return subUri
}
85 changes: 85 additions & 0 deletions tools-v2/internal/utils/snapshot/snapshot.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2023 NetEase Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* Project: CurveCli
* Created Date: 2023-09-16
* Author: baytan0720
*/

package snapshot

import (
"fmt"
"net/url"
)

func NewQuerySubUri(params map[string]any) string {
values := url.Values{}

values.Add(QueryVersion, Version)
for key, value := range params {
if value != nil && value != "" {
values.Add(key, fmt.Sprintf("%v", value))
}
}

return "/SnapshotCloneService?" + values.Encode()
}

type Response struct {
Code string `json:"Code"`
Message string `json:"Message"`
RequestId string `json:"RequestId"`
}

type SnapshotInfo struct {
File string `json:"File"`
FileLength int `json:"FileLength"`
Name string `json:"Name"`
Progress float64 `json:"Progress"`
SeqNum int `json:"SeqNum"`
Status int `json:"status"`
Time int64 `json:"Time"`
UUID string `json:"UUID"`
User string `json:"User"`
}

type SnapshotInfos []SnapshotInfo

var SnapshotStatus = []string{"done", "in-progress", "deleting", "errorDeleting", "canceling", "error"}

type TaskInfo struct {
File string `json:"File"`
FileType int `json:"FileType"`
IsLazy bool `json:"IsLazy"`
NextStep int `json:"NextStep"`
Progress int `json:"Progress"`
Src string `json:"Src"`
TaskStatus int `json:"TaskStatus"`
TaskType int `json:"TaskType"`
Time int `json:"Time"`
UUID string `json:"UUID"`
User string `json:"User"`
}

type TaskInfos []TaskInfo

var TaskStatus = []string{"Done", "Cloning", "Recovering", "Cleaning", "ErrorCleaning", "Error", "Retrying", "MetaInstalled"}
var TaskType = []string{"clone", "recover"}
var FileType = []string{"file", "snapshot"}
var CloneStep = []string{"CreateCloneFile", "CreateCloneMeta", "CreateCloneChunk", "CompleteCloneMeta", "RecoverChunk", "ChangeOwner", "RenameCloneFile", "CompleteCloneFile", "End"}
var IsLazy = []string{"false", "true"}
85 changes: 49 additions & 36 deletions tools-v2/pkg/cli/command/curvebs/delete/volume/clone/clone.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (

cmderror "github.com/opencurve/curve/tools-v2/internal/error"
cobrautil "github.com/opencurve/curve/tools-v2/internal/utils"
snapshotutil "github.com/opencurve/curve/tools-v2/internal/utils/snapshot"
basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command"
"github.com/opencurve/curve/tools-v2/pkg/config"
"github.com/opencurve/curve/tools-v2/pkg/output"
Expand Down Expand Up @@ -72,80 +73,92 @@ func (rCmd *CloneCmd) Init(cmd *cobra.Command, args []string) error {
if rCmd.failed {
rCmd.status = "5"
}
rCmd.SetHeader([]string{cobrautil.ROW_USER, cobrautil.ROW_SRC, cobrautil.ROW_TASK_ID, cobrautil.ROW_FILE, cobrautil.ROW_RESULT})
rCmd.SetHeader([]string{cobrautil.ROW_USER, cobrautil.ROW_SRC, cobrautil.ROW_TASK_ID, cobrautil.ROW_FILE, cobrautil.ROW_RESULT, cobrautil.ROW_REASON})
return nil
}

func (rCmd *CloneCmd) RunCommand(cmd *cobra.Command, args []string) error {
params := map[string]any{
cobrautil.QueryAction: cobrautil.ActionGetCloneTaskList,
cobrautil.QueryType: cobrautil.TypeCloneTask,
cobrautil.QueryUser: rCmd.user,
cobrautil.QueryUUID: rCmd.taskID,
cobrautil.QuerySource: rCmd.src,
cobrautil.QueryDestination: rCmd.dest,
cobrautil.QueryStatus: rCmd.status,
cobrautil.QueryLimit: 100,
cobrautil.QueryOffset: 0,
snapshotutil.QueryAction: snapshotutil.ActionGetCloneTaskList,
snapshotutil.QueryType: snapshotutil.TypeCloneTask,
snapshotutil.QueryUser: rCmd.user,
snapshotutil.QueryUUID: rCmd.taskID,
snapshotutil.QuerySource: rCmd.src,
snapshotutil.QueryDestination: rCmd.dest,
snapshotutil.QueryStatus: rCmd.status,
snapshotutil.QueryLimit: 100,
snapshotutil.QueryOffset: 0,
}
records := make([]map[string]string, 0)
records := make(snapshotutil.TaskInfos, 0)
for {
subUri := cobrautil.NewSnapshotQuerySubUri(params)
subUri := snapshotutil.NewQuerySubUri(params)
metric := basecmd.NewMetric(rCmd.snapshotAddrs, subUri, rCmd.timeout)
result, err := basecmd.QueryMetric(metric)
if err.TypeCode() != cmderror.CODE_SUCCESS {
return err.ToError()
}

var resp struct {
Code string `json:"Code"`
TaskInfos []map[string]string `json:"TaskInfos"`
TotalCount int `json:"TotalCount"`
var payload struct {
snapshotutil.Response
TaskInfos snapshotutil.TaskInfos `json:"TaskInfos"`
TotalCount int `json:"TotalCount"`
}
if err := json.Unmarshal([]byte(result), &resp); err != nil {
if err := json.Unmarshal([]byte(result), &payload); err != nil {
return err
}
if resp.Code != cobrautil.ResultSuccess {
return fmt.Errorf("get clone list fail, error code: %s", resp.Code)
if payload.Code != snapshotutil.ResultSuccess {
return fmt.Errorf("get clone list fail, requestId: %s, code: %s, message: %s", payload.RequestId, payload.Code, payload.Message)
}
if len(resp.TaskInfos) == 0 {
if len(payload.TaskInfos) == 0 {
break
} else {
records = append(records, resp.TaskInfos...)
params[cobrautil.QueryOffset] = params[cobrautil.QueryOffset].(int) + params[cobrautil.QueryLimit].(int)
records = append(records, payload.TaskInfos...)
params[snapshotutil.QueryOffset] = params[snapshotutil.QueryOffset].(int) + params[snapshotutil.QueryLimit].(int)
}
}

wg := sync.WaitGroup{}
for _, item := range records {
wg.Add(1)
go func(item map[string]string) {
go func(clone snapshotutil.TaskInfo) {
defer wg.Done()
result := cobrautil.ROW_VALUE_SUCCESS
reason := ""
params := map[string]any{
cobrautil.QueryAction: cobrautil.ActionCleanCloneTask,
cobrautil.QueryUser: item["User"],
cobrautil.QueryUUID: item["UUID"],
snapshotutil.QueryAction: snapshotutil.ActionCleanCloneTask,
snapshotutil.QueryUser: clone.User,
snapshotutil.QueryUUID: clone.UUID,
}
subUri := cobrautil.NewSnapshotQuerySubUri(params)
subUri := snapshotutil.NewQuerySubUri(params)
metric := basecmd.NewMetric(rCmd.snapshotAddrs, subUri, rCmd.timeout)
result, err := basecmd.QueryMetric(metric)
retStr, err := basecmd.QueryMetric(metric)
if err.TypeCode() != cmderror.CODE_SUCCESS {
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_FAILED
result = cobrautil.ROW_VALUE_FAILED
reason = err.ToError().Error()
} else {
payload := map[string]any{}
if err := json.Unmarshal([]byte(result), &payload); err != nil {
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_FAILED
payload := snapshotutil.Response{}
if err := json.Unmarshal([]byte(retStr), &payload); err != nil {
result = cobrautil.ROW_VALUE_FAILED
reason = err.Error()
} else {
if payload[cobrautil.ResultCode] != cobrautil.ResultSuccess {
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_FAILED
if payload.Code != snapshotutil.ResultSuccess {
result = cobrautil.ROW_VALUE_FAILED
reason = payload.Message
}
}
}
item[cobrautil.ROW_RESULT] = cobrautil.ROW_VALUE_SUCCESS
rCmd.TableNew.Append([]string{item["User"], item["Src"], item["UUID"], item["File"], item["Result"]})
rCmd.TableNew.Append(cobrautil.Map2List(map[string]string{
cobrautil.ROW_USER: clone.User,
cobrautil.ROW_SRC: clone.Src,
cobrautil.ROW_TASK_ID: clone.UUID,
cobrautil.ROW_FILE: clone.File,
cobrautil.ROW_RESULT: result,
cobrautil.ROW_REASON: reason,
}, rCmd.Header))
}(item)
}
wg.Wait()

rCmd.Result = records
rCmd.Error = cmderror.Success()
return nil
Expand Down
Loading

0 comments on commit 51b1936

Please sign in to comment.