diff --git a/glustercli/cmd/volume-create.go b/glustercli/cmd/volume-create.go index aa7f5ed53..830bc2058 100644 --- a/glustercli/cmd/volume-create.go +++ b/glustercli/cmd/volume-create.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/api" log "github.com/sirupsen/logrus" @@ -315,7 +316,7 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { fmt.Println("Thin arbiter can only be enabled for replica count 2") return } - if err := addThinArbiter(&req, cmd.Flag("thin-arbiter").Value.String()); err != nil { + if err := volume.AddThinArbiter(&req, cmd.Flag("thin-arbiter").Value.String()); err != nil { fmt.Println(err) return } @@ -331,20 +332,3 @@ func volumeCreateCmdRun(cmd *cobra.Command, args []string) { fmt.Printf("%s Volume created successfully\n", vol.Name) fmt.Println("Volume ID: ", vol.ID) } - -func addThinArbiter(req *api.VolCreateReq, thinArbiter string) error { - - s := strings.Split(thinArbiter, ":") - if len(s) != 2 && len(s) != 3 { - return fmt.Errorf("thin arbiter brick must be of the form : or ::") - } - - // TODO: If required, handle this in a generic way, just like other - // volume set options that we're going to allow to be set during - // volume create. - req.Options = map[string]string{ - "replicate.thin-arbiter": thinArbiter, - } - req.AllowAdvanced = true - return nil -} diff --git a/glusterd2/volume/filters.go b/glusterd2/volume/filters.go index 1d6883ef7..8cbb4b20a 100644 --- a/glusterd2/volume/filters.go +++ b/glusterd2/volume/filters.go @@ -34,6 +34,30 @@ func ApplyCustomFilters(volumes []*Volinfo, filters ...Filter) []*Volinfo { return volumes } +// FilterThinArbiterVolumes filters out volumes that are thin arbiter volumes +func FilterThinArbiterVolumes(volumes []*Volinfo) []*Volinfo { + var volInfos []*Volinfo + for _, volume := range volumes { + _, exists := volume.Options["cluster/replicate.thin-arbiter"] + if exists { + volInfos = append(volInfos, volume) + } + } + return volInfos +} + +//FilterShardVolumes filters out volumes that are shard enabled +func FilterShardVolumes(volumes []*Volinfo) []*Volinfo { + var volInfos []*Volinfo + for _, volume := range volumes { + _, exists := volume.Options["features/shard"] + if exists { + volInfos = append(volInfos, volume) + } + } + return volInfos +} + // FilterBlockHostedVolumes filters out volume which are suitable for hosting block volume func FilterBlockHostedVolumes(volumes []*Volinfo) []*Volinfo { var volInfos []*Volinfo diff --git a/glusterd2/volume/volume-utils.go b/glusterd2/volume/volume-utils.go index 16bc2b84f..e594fbac5 100644 --- a/glusterd2/volume/volume-utils.go +++ b/glusterd2/volume/volume-utils.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "regexp" + "strconv" "strings" "syscall" @@ -399,3 +400,28 @@ func CleanBricksLoop(volinfo *Volinfo) error { } return nil } + +//AddThinArbiter adds thin arbiter option to the volume create request +func AddThinArbiter(req *api.VolCreateReq, thinArbiter string) error { + + s := strings.Split(thinArbiter, ":") + if len(s) != 2 && len(s) != 3 { + return errors.New("thin arbiter brick must be of the form : or ::") + } + + // TODO: If required, handle this in a generic way, just like other + // volume set options that we're going to allow to be set during + // volume create. + req.Options["replicate.thin-arbiter"] = thinArbiter + req.AllowAdvanced = true + return nil +} + +//AddShard adds shard options to the volume create request +func AddShard(req *api.VolCreateReq, shardSize uint64) { + + req.Options["features/shard"] = "on" + req.Options["features/shard.shard-block-size"] = strconv.FormatUint(shardSize, 10) + req.AllowAdvanced = true + return +} diff --git a/plugins/blockvolume/api/types.go b/plugins/blockvolume/api/types.go index 907b58a37..6dc71c9f9 100644 --- a/plugins/blockvolume/api/types.go +++ b/plugins/blockvolume/api/types.go @@ -11,11 +11,25 @@ type BlockVolumeInfo struct { HaCount int `json:"hacount,omitempty"` } +// HostVolumeInfo represents block volume info +type HostVolumeInfo struct { + // HostVolReplicaCnt represents the replica count of the block hosting volume + HostVolReplicaCnt int `json:"hostvolreplicacnt,omitempty"` + // HostVolThinArbPath represents the thin arbiter path + HostVolThinArbPath string `json:"hostvolthinarbpath,omitempty"` + // HostVolShardSize represents the shard size of the block hosting volume + HostVolShardSize uint64 `json:"hostvolshardsize,omitempty"` + // Size represents Block Hosting Volume size in bytes + HostVolSize uint64 `json:"hostvolsize,omitempty"` +} + // BlockVolumeCreateRequest represents req Body for Block vol create req type BlockVolumeCreateRequest struct { *BlockVolumeInfo - Clusters []string `json:"clusters,omitempty"` - Auth bool `json:"auth,omitempty"` + HostVolumeInfo + BlockType string `json:"blocktype,omitempty"` + Clusters []string `json:"clusters,omitempty"` + Auth bool `json:"auth,omitempty"` } // BlockVolumeCreateResp represents resp body for a Block Vol Create req diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go index c7d4a232d..5fb3d26c0 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -6,7 +6,7 @@ import ( "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/size" "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" - "github.com/gluster/glusterd2/plugins/blockvolume/utils" + "github.com/gluster/glusterd2/plugins/blockvolume/hostvol" "time" "github.com/gluster/gluster-block-restapi/client" @@ -79,7 +79,7 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hostVolume st } resizeFunc := func(blockHostingAvailableSize, blockSize uint64) uint64 { return blockHostingAvailableSize - blockSize } - if err = utils.ResizeBlockHostingVolume(hostVolume, size, resizeFunc); err != nil { + if err = hostvol.ResizeBlockHostingVolume(hostVolume, size, resizeFunc); err != nil { logger.WithError(err).Error("failed in updating hostvolume _block-hosting-available-size metadata") } @@ -133,7 +133,7 @@ func (g *GlusterBlock) DeleteBlockVolume(name string, options ...blockprovider.B resizeFunc := func(blockHostingAvailableSize, blockSize uint64) uint64 { return blockHostingAvailableSize + blockSize } - if err = utils.ResizeBlockHostingVolume(hostVol, blockInfo.Size, resizeFunc); err != nil { + if err = hostvol.ResizeBlockHostingVolume(hostVol, blockInfo.Size, resizeFunc); err != nil { log.WithFields(log.Fields{ "error": err, "size": blockInfo.Size, diff --git a/plugins/blockvolume/blockprovider/gluster-virtblock/gluster_virtblock.go b/plugins/blockvolume/blockprovider/gluster-virtblock/gluster_virtblock.go index b3330d108..f7032ac6e 100644 --- a/plugins/blockvolume/blockprovider/gluster-virtblock/gluster_virtblock.go +++ b/plugins/blockvolume/blockprovider/gluster-virtblock/gluster_virtblock.go @@ -12,7 +12,7 @@ import ( "github.com/gluster/glusterd2/pkg/errors" "github.com/gluster/glusterd2/pkg/utils" "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" - blkUtils "github.com/gluster/glusterd2/plugins/blockvolume/utils" + "github.com/gluster/glusterd2/plugins/blockvolume/hostvol" log "github.com/sirupsen/logrus" config "github.com/spf13/viper" @@ -95,18 +95,21 @@ func (g *GlusterVirtBlk) CreateBlockVolume(name string, size uint64, hostVolume blockFileName := hostDir + "/" + name err = utils.ExecuteCommandRun("truncate", fmt.Sprintf("-s %d", size), blockFileName) //nolint: gosec if err != nil { - logger.WithError(err).Errorf("failed to truncate block file" + blockFileName) + logger.WithError(err).Errorf("failed to truncate block file %s", blockFileName) return nil, err } - err = utils.ExecuteCommandRun("mkfs.xfs", "-f", blockFileName) //nolint: gosec - if err != nil { - logger.WithError(err).Errorf("failed to format block file " + blockFileName) - return nil, err + if blockVolOpts.BlockType != "raw" { + fsType := blockVolOpts.BlockType + err = utils.ExecuteCommandRun(fmt.Sprintf("mkfs.%s", fsType), "-f", blockFileName) //nolint: gosec + if err != nil { + logger.WithError(err).Errorf("failed to format block file %s with filesystem %s", blockFileName, fsType) + return nil, err + } } resizeFunc := func(blockHostingAvailableSize, blockSize uint64) uint64 { return blockHostingAvailableSize - blockSize } - if err = blkUtils.ResizeBlockHostingVolume(hostVolume, size, resizeFunc); err != nil { + if err = hostvol.ResizeBlockHostingVolume(hostVolume, size, resizeFunc); err != nil { logger.WithError(err).Error("failed in updating hostvolume _block-hosting-available-size metadata") return nil, err } @@ -119,7 +122,7 @@ func (g *GlusterVirtBlk) CreateBlockVolume(name string, size uint64, hostVolume volInfo, err := volume.GetVolume(hostVolume) if err != nil { - logger.WithError(err).Errorf("failed to get host volume info " + hostVolume) + logger.WithError(err).Errorf("failed to get host volume info %s", hostVolume) return nil, err } key := volume.BlockPrefix + name @@ -181,14 +184,14 @@ func (g *GlusterVirtBlk) DeleteBlockVolume(name string, options ...blockprovider hostDir, err := mountHost(g, blkVol.HostVolume()) if err != nil { - log.WithError(err).Errorf("error mounting block hosting volume :" + blkVol.HostVolume()) + log.WithError(err).Errorf("error mounting block hosting volume :%s", blkVol.HostVolume()) return err } blockFileName := hostDir + "/" + name err = os.Remove(blockFileName) if err != nil { - log.WithError(err).Errorf("error removing block :" + blockFileName) + log.WithError(err).Errorf("error removing block :%s", blockFileName) return err } @@ -199,7 +202,7 @@ func (g *GlusterVirtBlk) DeleteBlockVolume(name string, options ...blockprovider } resizeFunc := func(blockHostingAvailableSize, blockSize uint64) uint64 { return blockHostingAvailableSize + blockSize } - if err = blkUtils.ResizeBlockHostingVolume(blkVol.HostVolume(), blkVol.Size(), resizeFunc); err != nil { + if err = hostvol.ResizeBlockHostingVolume(blkVol.HostVolume(), blkVol.Size(), resizeFunc); err != nil { log.WithFields(log.Fields{ "error": err, "size": blkVol.Size(), diff --git a/plugins/blockvolume/blockprovider/options.go b/plugins/blockvolume/blockprovider/options.go index 7f33fff1b..ccc4082ee 100644 --- a/plugins/blockvolume/blockprovider/options.go +++ b/plugins/blockvolume/blockprovider/options.go @@ -13,6 +13,7 @@ type BlockVolumeOptions struct { ForceDelete bool UnlinkStorage bool Hosts []string + BlockType string } // ApplyOpts applies configured optional parameters on BlockVolumeOptions @@ -69,3 +70,14 @@ func WithHosts(hosts []string) BlockVolOption { options.Hosts = hosts } } + +// WithBlockType configures the block type +func WithBlockType(blockType string) BlockVolOption { + return func(options *BlockVolumeOptions) { + if blockType == "" { + options.BlockType = "xfs" + } else { + options.BlockType = blockType + } + } +} diff --git a/plugins/blockvolume/handlers.go b/plugins/blockvolume/handlers.go index aec8266ec..714dd1d8e 100644 --- a/plugins/blockvolume/handlers.go +++ b/plugins/blockvolume/handlers.go @@ -27,6 +27,7 @@ func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { opts = append(opts, blockprovider.WithHaCount(req.HaCount), blockprovider.WithHosts(req.Clusters), + blockprovider.WithBlockType(req.BlockType), ) if req.Auth { @@ -39,7 +40,7 @@ func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { return } - hostVolInfo, err := b.hostVolManager.GetOrCreateHostingVolume(req.HostingVolume, req.Size) + hostVolInfo, err := b.hostVolManager.GetOrCreateHostingVolume(req.HostingVolume, req.Size, &req.HostVolumeInfo) if err != nil { utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) return diff --git a/plugins/blockvolume/hostvol_manager.go b/plugins/blockvolume/hostvol/hostvol_manager.go similarity index 77% rename from plugins/blockvolume/hostvol_manager.go rename to plugins/blockvolume/hostvol/hostvol_manager.go index ab1de8ce7..46b6a2e16 100644 --- a/plugins/blockvolume/hostvol_manager.go +++ b/plugins/blockvolume/hostvol/hostvol_manager.go @@ -1,4 +1,4 @@ -package blockvolume +package hostvol import ( "context" @@ -11,7 +11,7 @@ import ( "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/glusterd2/volume" - "github.com/gluster/glusterd2/plugins/blockvolume/utils" + "github.com/gluster/glusterd2/plugins/blockvolume/api" log "github.com/sirupsen/logrus" ) @@ -23,17 +23,17 @@ const ( // HostingVolumeManager provides methods for host volume management type HostingVolumeManager interface { GetHostingVolumesInUse() []*volume.Volinfo - GetOrCreateHostingVolume(name string, minSizeLimit uint64) (*volume.Volinfo, error) + GetOrCreateHostingVolume(name string, minSizeLimit uint64, hostVolumeInfo *api.HostVolumeInfo) (*volume.Volinfo, error) } -// glusterVolManager is a concrete implementation of HostingVolumeManager -type glusterVolManager struct { +// GlusterVolManager is a concrete implementation of HostingVolumeManager +type GlusterVolManager struct { hostVolOpts *HostingVolumeOptions } -// newGlusterVolManager returns a glusterVolManager instance -func newGlusterVolManager() *glusterVolManager { - g := &glusterVolManager{ +// NewGlusterVolManager returns a glusterVolManager instance +func NewGlusterVolManager() *GlusterVolManager { + g := &GlusterVolManager{ hostVolOpts: newHostingVolumeOptions(), } @@ -41,7 +41,7 @@ func newGlusterVolManager() *glusterVolManager { } // GetHostingVolumesInUse lists all volumes which used in hosting block-vols -func (g *glusterVolManager) GetHostingVolumesInUse() []*volume.Volinfo { +func (g *GlusterVolManager) GetHostingVolumesInUse() []*volume.Volinfo { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -55,7 +55,7 @@ func (g *glusterVolManager) GetHostingVolumesInUse() []*volume.Volinfo { // GetOrCreateHostingVolume will returns volume details for a given volume name and having a minimum size of `minSizeLimit`. // If volume name is not provided then it will create a gluster volume with default size for hosting gluster block. -func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit uint64) (*volume.Volinfo, error) { +func (g *GlusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit uint64, hostVolumeInfo *api.HostVolumeInfo) (*volume.Volinfo, error) { var ( volInfo *volume.Volinfo clusterLocks = transaction.Locks{} @@ -67,9 +67,14 @@ func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit u defer clusterLocks.UnLock(context.Background()) g.hostVolOpts.SetFromClusterOptions() - volCreateReq := g.hostVolOpts.PrepareVolumeCreateReq() + g.hostVolOpts.SetFromReq(hostVolumeInfo) + volCreateReq, err := g.hostVolOpts.PrepareVolumeCreateReq() + if err != nil { + log.WithError(err).Error("failed to create block volume create request") + return nil, err + } - // ERROR if If HostingVolume is not specified and auto-create-block-hosting-volumes is false + // ERROR if HostingVolume is not specified and auto-create-block-hosting-volumes is false if name == "" && !g.hostVolOpts.AutoCreate { err := errors.New("host volume is not provided and auto creation is not enabled") log.WithError(err).Error("failed in creating block volume") @@ -91,7 +96,7 @@ func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit u // If HostingVolume is not specified. List all available volumes and see if any volume is // available with Metadata:block-hosting=yes if name == "" { - vInfo, err := utils.GetExistingBlockHostingVolume(minSizeLimit) + vInfo, err := GetExistingBlockHostingVolume(minSizeLimit, g.hostVolOpts) if err != nil { log.WithError(err).Debug("no block hosting volumes present") } @@ -102,7 +107,7 @@ func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit u // volumes(Metadata:block-hosting-available-size is less than request size), then try to create a new // block hosting Volume with generated name with default size and volume type configured. if name == "" && volInfo == nil { - vInfo, err := utils.CreateAndStartHostingVolume(volCreateReq) + vInfo, err := CreateAndStartHostingVolume(volCreateReq) if err != nil { log.WithError(err).Error("error in auto creation of block hosting volume") return nil, err diff --git a/plugins/blockvolume/hostvol_opts.go b/plugins/blockvolume/hostvol/hostvol_opts.go similarity index 58% rename from plugins/blockvolume/hostvol_opts.go rename to plugins/blockvolume/hostvol/hostvol_opts.go index 7b125827f..a51e703eb 100644 --- a/plugins/blockvolume/hostvol_opts.go +++ b/plugins/blockvolume/hostvol/hostvol_opts.go @@ -1,13 +1,17 @@ -package blockvolume +package hostvol import ( + "errors" "strconv" "github.com/gluster/glusterd2/glusterd2/options" + "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/api" "github.com/gluster/glusterd2/pkg/size" + blkapi "github.com/gluster/glusterd2/plugins/blockvolume/api" "github.com/pborman/uuid" + log "github.com/sirupsen/logrus" ) const ( @@ -23,6 +27,8 @@ type HostingVolumeOptions struct { Type string ReplicaCount int AutoCreate bool + ThinArbPath string + ShardSize uint64 } func newHostingVolumeOptions() *HostingVolumeOptions { @@ -35,7 +41,7 @@ func newHostingVolumeOptions() *HostingVolumeOptions { } // PrepareVolumeCreateReq will create a request body to be use for creating a gluster volume -func (h *HostingVolumeOptions) PrepareVolumeCreateReq() *api.VolCreateReq { +func (h *HostingVolumeOptions) PrepareVolumeCreateReq() (*api.VolCreateReq, error) { name := "block_hosting_volume_" + uuid.NewRandom().String() req := &api.VolCreateReq{ @@ -44,9 +50,44 @@ func (h *HostingVolumeOptions) PrepareVolumeCreateReq() *api.VolCreateReq { Size: uint64(h.Size), ReplicaCount: h.ReplicaCount, SubvolType: h.Type, + Force: true, + VolOptionReq: api.VolOptionReq{ + Options: map[string]string{}, + }, } - return req + if h.ThinArbPath != "" { + if h.ReplicaCount != 2 { + err := errors.New("thin arbiter can only be enabled for replica count 2") + log.WithError(err).Error("failed to prepare host vol create request") + return nil, err + } + if err := volume.AddThinArbiter(req, h.ThinArbPath); err != nil { + log.WithError(err).Error("failed to add thin arbiter options to host volume") + return nil, err + } + } + if h.ShardSize != 0 { + volume.AddShard(req, h.ShardSize) + } + + return req, nil +} + +// SetFromReq will configure HostingVolumeOptions from the values sent in the block create request +func (h *HostingVolumeOptions) SetFromReq(hvi *blkapi.HostVolumeInfo) { + if hvi.HostVolReplicaCnt != 0 { + h.ReplicaCount = hvi.HostVolReplicaCnt + } + if hvi.HostVolThinArbPath != "" { + h.ThinArbPath = hvi.HostVolThinArbPath + } + if hvi.HostVolShardSize != 0 { + h.ShardSize = hvi.HostVolShardSize + } + if hvi.HostVolSize != 0 { + h.Size = size.Size(hvi.HostVolSize) + } } // SetFromClusterOptions will configure HostingVolumeOptions using cluster options @@ -76,4 +117,6 @@ func (h *HostingVolumeOptions) SetFromClusterOptions() { h.AutoCreate = val } } + h.ThinArbPath = "" + h.ShardSize = 0 } diff --git a/plugins/blockvolume/utils/volume.go b/plugins/blockvolume/hostvol/utils.go similarity index 92% rename from plugins/blockvolume/utils/volume.go rename to plugins/blockvolume/hostvol/utils.go index 38d690cc8..86ad7fdc9 100644 --- a/plugins/blockvolume/utils/volume.go +++ b/plugins/blockvolume/hostvol/utils.go @@ -1,4 +1,4 @@ -package utils +package hostvol import ( "context" @@ -15,7 +15,6 @@ import ( "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/api" "github.com/gluster/glusterd2/pkg/size" - log "github.com/sirupsen/logrus" ) @@ -40,7 +39,7 @@ func BlockSizeFilter(size uint64) volume.Filter { } // GetExistingBlockHostingVolume returns a existing volume which is suitable for hosting a gluster-block -func GetExistingBlockHostingVolume(size uint64) (*volume.Volinfo, error) { +func GetExistingBlockHostingVolume(size uint64, h *HostingVolumeOptions) (*volume.Volinfo, error) { var ( filters = []volume.Filter{volume.FilterBlockHostedVolumes, BlockSizeFilter(size)} ctx, cancel = context.WithTimeout(context.Background(), time.Second*5) @@ -48,6 +47,13 @@ func GetExistingBlockHostingVolume(size uint64) (*volume.Volinfo, error) { defer cancel() + if h.ShardSize != 0 { + filters = append(filters, volume.FilterShardVolumes) + } + if h.ThinArbPath != "" { + filters = append(filters, volume.FilterThinArbiterVolumes) + } + volumes, err := volume.GetVolumes(ctx) if err != nil || len(volumes) == 0 { return nil, fmt.Errorf("%v/no volumes found", err) @@ -124,6 +130,8 @@ func ResizeBlockHostingVolume(volName string, blockSize interface{}, resizeFunc return err } + // TODO: If there are no blocks in the block hosting volume, delete the bhv? + log.WithFields(log.Fields{ "blockHostingAvailableSize": size.Size(availableSizeInBytes), "blockSize": blkSize, diff --git a/plugins/blockvolume/routes.go b/plugins/blockvolume/routes.go index 1eafdb8ff..273428d58 100644 --- a/plugins/blockvolume/routes.go +++ b/plugins/blockvolume/routes.go @@ -7,11 +7,12 @@ import ( "github.com/gluster/glusterd2/glusterd2/servers/rest/route" "github.com/gluster/glusterd2/pkg/utils" "github.com/gluster/glusterd2/plugins/blockvolume/api" + "github.com/gluster/glusterd2/plugins/blockvolume/hostvol" ) // BlockVolume represents BlockVolume plugin type BlockVolume struct { - hostVolManager HostingVolumeManager + hostVolManager hostvol.HostingVolumeManager initOnce sync.Once } @@ -68,6 +69,6 @@ func (*BlockVolume) RegisterStepFuncs() { // calling it multiple times will do nothing func (b *BlockVolume) Init() { b.initOnce.Do(func() { - b.hostVolManager = newGlusterVolManager() + b.hostVolManager = hostvol.NewGlusterVolManager() }) }