[feat] curvebs: The formatting and deployment phases support separate partitioning of WAL and data

Signed-off-by: zyb521 <[email protected]>

Author: zyb521

internal/configure/format.go

@@ -32,6 +32,8 @@ import (
 
 const (
 	DEFAULT_CONTAINER_IMAGE = "opencurvedocker/curvebs:v1.2"
+	FORMAT_TYPE_DATA        = "data"
+	FORMAT_TYPE_WAL         = "wal"
 )
 
 /*
@@ -40,14 +42,18 @@ const (
  *   - machine2
  *   - machine3
  * disk:
- *   - /dev/sda:/data/chunkserver0:10  # device:mount_path:format_percent
- *   - /dev/sdb:/data/chunkserver1:10
- *   - /dev/sdc:/data/chunkserver2:10
+ *   - data:/dev/sda:/data/chunkserver0:10  # fortmat_type:device:mount_path:format_percent
+ *   - data:/dev/sdb:/data/chunkserver1:10
+ *   - data:/dev/sdc:/data/chunkserver2:10
+ *   - wal:/dev/nvme0n1p1:/data/wal/chunkserver0:10
+ *   - wal:/dev/nvme0n1p2:/data/wal/chunkserver1:10
+ *   - wal:/dev/nvme0n1p3:/data/wal/chunkserver2:10
  */
 type (
 	FormatConfig struct {
 		ContainerIamge     string
 		Host               string
+		Type               string
 		Device             string
 		MountPoint         string
 		FormtPercent       int
@@ -64,12 +70,15 @@ type (
 
 func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
 	items := strings.Split(disk, ":")
-	if len(items) != 3 {
+	if len(items) != 4 {
 		return nil, errno.ERR_INVALID_DISK_FORMAT.S(disk)
 	}
 
-	device, mountPoint, percent := items[0], items[1], items[2]
-	if !strings.HasPrefix(device, "/") {
+	formatType, device, mountPoint, percent := items[0], items[1], items[2], items[3]
+	if formatType != FORMAT_TYPE_DATA && formatType != FORMAT_TYPE_WAL {
+		return nil, errno.ERR_INVALID_FORMAT_TYPE.
+			F("formatType: %s", formatType)
+	} else if !strings.HasPrefix(device, "/") {
 		return nil, errno.ERR_INVALID_DEVICE.
 			F("device: %s", device)
 	} else if !strings.HasPrefix(mountPoint, "/") {
@@ -89,6 +98,7 @@ func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
 	return &FormatConfig{
 		ContainerIamge: containerImage,
 		Host:           host,
+		Type:           formatType,
 		Device:         device,
 		MountPoint:     mountPoint,
 		FormtPercent:   formatPercent,
@@ -136,6 +146,7 @@ func ParseFormat(filename string) ([]*FormatConfig, error) {
 
 func (fc *FormatConfig) GetContainerImage() string { return fc.ContainerIamge }
 func (fc *FormatConfig) GetHost() string           { return fc.Host }
+func (fc *FormatConfig) GetFormatType() string     { return fc.Type }
 func (fc *FormatConfig) GetDevice() string         { return fc.Device }
 func (fc *FormatConfig) GetMountPoint() string     { return fc.MountPoint }
 func (fc *FormatConfig) GetFormatPercent() int     { return fc.FormtPercent }

internal/configure/topology/dc_get.go

@@ -42,6 +42,7 @@ const (
 	LAYOUT_SERVICE_CONF_DIR                 = "/conf"
 	LAYOUT_SERVICE_LOG_DIR                  = "/logs"
 	LAYOUT_SERVICE_DATA_DIR                 = "/data"
+	LAYOUT_SERVICE_WAL_DIR                  = "/wal"
 	LAYOUT_TOOLS_DIR                        = "/tools"
 	LAYOUT_TOOLS_V2_DIR                     = "/tools-v2"
 	LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR       = "chunkfilepool"
@@ -131,6 +132,7 @@ func (dc *DeployConfig) GetContainerImage() string {
 }
 func (dc *DeployConfig) GetLogDir() string           { return dc.getString(CONFIG_LOG_DIR) }
 func (dc *DeployConfig) GetDataDir() string          { return dc.getString(CONFIG_DATA_DIR) }
+func (dc *DeployConfig) GetWalDir() string           { return dc.getString(CONFIG_WAL_DIR) }
 func (dc *DeployConfig) GetCoreDir() string          { return dc.getString(CONFIG_CORE_DIR) }
 func (dc *DeployConfig) GetListenIp() string         { return dc.getString(CONFIG_LISTEN_IP) }
 func (dc *DeployConfig) GetListenPort() int          { return dc.getInt(CONFIG_LISTEN_PORT) }
@@ -181,7 +183,8 @@ func (dc *DeployConfig) GetListenExternalPort() int {
  *  │   ├── conf
  *  │   ├── data
  *  │   ├── log
- *  │   └── sbin
+ *  │   ├── sbin
+ *  │   └── wal
  *  ├── snapshotclone
  *  │   ├── conf
  *  │   ├── data
@@ -212,6 +215,7 @@ type (
 		ServiceConfDir     string // /curvebs/mds/conf
 		ServiceLogDir      string // /curvebs/mds/logs
 		ServiceDataDir     string // /curvebs/mds/data
+		ServiceWalDir      string // /curvebs/chunkserver/wal
 		ServiceConfPath    string // /curvebs/mds/conf/mds.conf
 		ServiceConfSrcPath string // /curvebs/conf/mds.conf
 		ServiceConfFiles   []ConfFile
@@ -236,6 +240,9 @@ type (
 		ChunkfilePoolRootDir  string // /curvebs/chunkserver/data
 		ChunkfilePoolDir      string // /curvebs/chunkserver/data/chunkfilepool
 		ChunkfilePoolMetaPath string // /curvebs/chunkserver/data/chunkfilepool.meta
+		WalfilePoolRootDir    string // /curvebs/chunkserver/wal
+		WalfilePoolDir        string // /curvebs/chunkserver/wal/walfilepool
+		WalfilePoolMetaPath   string // /curvebs/chunkserver/wal/walfilepool.meta
 
 		// core
 		CoreSystemDir string
@@ -292,6 +299,7 @@ func (dc *DeployConfig) GetProjectLayout() Layout {
 		ServiceConfDir:     serviceRootDir + LAYOUT_SERVICE_CONF_DIR,
 		ServiceLogDir:      serviceRootDir + LAYOUT_SERVICE_LOG_DIR,
 		ServiceDataDir:     serviceRootDir + LAYOUT_SERVICE_DATA_DIR,
+		ServiceWalDir:      serviceRootDir + LAYOUT_SERVICE_WAL_DIR,
 		ServiceConfPath:    fmt.Sprintf("%s/%s.conf", serviceConfDir, role),
 		ServiceConfSrcPath: fmt.Sprintf("%s/%s.conf", confSrcDir, role),
 		ServiceConfFiles:   serviceConfFiles,

internal/configure/topology/dc_item.go

@@ -124,6 +124,13 @@ var (
 		nil,
 	)
 
+	CONFIG_WAL_DIR = itemset.insert(
+		"wal_dir",
+		REQUIRE_STRING,
+		true,
+		nil,
+	)
+
 	CONFIG_CORE_DIR = itemset.insert(
 		"core_dir",
 		REQUIRE_STRING,

internal/errno/errno.go

@@ -376,6 +376,7 @@ var (
 	ERR_MOUNT_POINT_REQUIRE_ABSOLUTE_PATH        = EC(341002, "mount point must be an absolute path")
 	ERR_FORMAT_PERCENT_REQUIRES_INTERGET         = EC(341003, "format percentage requires an integer")
 	ERR_FORMAT_PERCENT_MUST_BE_BETWEEN_1_AND_100 = EC(341004, "format percentage must be between 1 and 100")
+	ERR_INVALID_FORMAT_TYPE                      = EC(341005, "format type must be (data) or (wal)")
 
 	// 350: configure (client.yaml: parse failed)
 	ERR_PARSE_CLIENT_CONFIGURE_FAILED  = EC(350000, "parse client configure failed")

internal/task/task/bs/format.go

@@ -231,22 +231,33 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
 	}
 
 	// new task
+	formatTpye := fc.GetFormatType()
 	device := fc.GetDevice()
 	mountPoint := fc.GetMountPoint()
 	usagePercent := fc.GetFormatPercent()
-	subname := fmt.Sprintf("host=%s device=%s mountPoint=%s usage=%d%%",
-		fc.GetHost(), device, mountPoint, usagePercent)
-	t := task.NewTask("Start Format Chunkfile Pool", subname, hc.GetSSHConfig())
+	subname := fmt.Sprintf("host=%s formatTpye=%s device=%s mountPoint=%s usage=%d%%",
+		fc.GetHost(), formatTpye, device, mountPoint, usagePercent)
+	t := task.NewTask("Start Format filePool", subname, hc.GetSSHConfig())
 
 	// add step to task
 	var oldContainerId, containerId, oldUUID string
+	var filePoolRootDir, filePoolDir, filePoolMetaPath string
 	containerName := device2ContainerName(device)
 	layout := topology.GetCurveBSProjectLayout()
-	chunkfilePoolRootDir := layout.ChunkfilePoolRootDir
+
+	if formatTpye == configure.FORMAT_TYPE_WAL {
+		filePoolRootDir = layout.WalfilePoolRootDir
+		filePoolDir = layout.WalfilePoolDir
+		filePoolMetaPath = layout.WalfilePoolMetaPath
+	} else {
+		filePoolRootDir = layout.ChunkfilePoolRootDir
+		filePoolDir = layout.ChunkfilePoolDir
+		filePoolMetaPath = layout.ChunkfilePoolMetaPath
+	}
 	formatScript := scripts.SCRIPT_FORMAT
 	formatScriptPath := fmt.Sprintf("%s/format.sh", layout.ToolsBinDir)
 	formatCommand := fmt.Sprintf("%s %s %d %d %s %s", formatScriptPath, layout.FormatBinaryPath,
-		usagePercent, DEFAULT_CHUNKFILE_SIZE, layout.ChunkfilePoolDir, layout.ChunkfilePoolMetaPath)
+		usagePercent, DEFAULT_CHUNKFILE_SIZE, filePoolDir, filePoolMetaPath)
 
 	// 1: skip if formating container exist
 	t.AddStep(&step.ListContainers{
@@ -320,7 +331,7 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
 		Entrypoint:  "/bin/bash",
 		Name:        containerName,
 		Remove:      true,
-		Volumes:     []step.Volume{{HostPath: mountPoint, ContainerPath: chunkfilePoolRootDir}},
+		Volumes:     []step.Volume{{HostPath: mountPoint, ContainerPath: filePoolRootDir}},
 		Out:         &containerId,
 		ExecOptions: curveadm.ExecOptions(),
 	})

internal/task/task/common/create_container.go

@@ -111,6 +111,7 @@ func getArguments(dc *topology.DeployConfig) string {
 	// only chunkserver need so many arguments, but who cares
 	layout := dc.GetProjectLayout()
 	dataDir := layout.ServiceDataDir
+	walDir := layout.ServiceWalDir
 	chunkserverArguments := map[string]interface{}{
 		// chunkserver
 		"conf":                  layout.ServiceConfPath,
@@ -120,11 +121,11 @@ func getArguments(dc *topology.DeployConfig) string {
 		"chunkServerPort":       dc.GetListenPort(),
 		"chunkFilePoolDir":      dataDir,
 		"chunkFilePoolMetaPath": fmt.Sprintf("%s/chunkfilepool.meta", dataDir),
-		"walFilePoolDir":        dataDir,
-		"walFilePoolMetaPath":   fmt.Sprintf("%s/walfilepool.meta", dataDir),
+		"walFilePoolDir":        walDir,
+		"walFilePoolMetaPath":   fmt.Sprintf("%s/walfilepool.meta", walDir),
 		"copySetUri":            fmt.Sprintf("local://%s/copysets", dataDir),
 		"recycleUri":            fmt.Sprintf("local://%s/recycler", dataDir),
-		"raftLogUri":            fmt.Sprintf("curve://%s/copysets", dataDir),
+		"raftLogUri":            fmt.Sprintf("curve://%s/copysets", walDir),
 		"raftSnapshotUri":       fmt.Sprintf("curve://%s/copysets", dataDir),
 		"chunkServerStoreUri":   fmt.Sprintf("local://%s", dataDir),
 		"chunkServerMetaUri":    fmt.Sprintf("local://%s/chunkserver.dat", dataDir),
@@ -163,6 +164,7 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step.
 	layout := dc.GetProjectLayout()
 	logDir := dc.GetLogDir()
 	dataDir := dc.GetDataDir()
+	walDir := dc.GetWalDir()
 	coreDir := dc.GetCoreDir()
 
 	if len(logDir) > 0 {
@@ -180,6 +182,13 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step.
 		})
 	}
 
+	if len(walDir) > 0 {
+		volumes = append(volumes, step.Volume{
+			HostPath:      walDir,
+			ContainerPath: layout.ServiceWalDir,
+		})
+	}
+
 	if len(coreDir) > 0 {
 		volumes = append(volumes, step.Volume{
 			HostPath:      coreDir,