[Feat] ✨ Add BallastStone Mechanism to Avoid Instance Action Execute Error mayooot#13

Author: Harry-zklcdc

internal/services/replicaset.go

@@ -29,6 +29,8 @@ import (
 	"github.com/mayooot/gpu-docker-api/utils"
 )
 
+const ballastStone = "var/backups/ballaststone"
+
 var lxcfsBind = []string{
 	"/var/lib/lxcfs/proc/cpuinfo:/proc/cpuinfo:rw",
 	"/var/lib/lxcfs/proc/diskstats:/proc/diskstats:rw",
@@ -236,10 +238,10 @@ func (rs *ReplicaSetService) ExecuteContainer(name string, exec *models.Containe
 	}
 
 	hijackedResp, err := docker.Cli.ContainerExecAttach(ctx, execCreate.ID, container.ExecAttachOptions{})
-	defer hijackedResp.Close()
 	if err != nil {
 		return resp, errors.Wrapf(err, "docker.ContainerExecAttach failed, name: %s, spec: %+v", name, exec)
 	}
+	defer hijackedResp.Close()
 
 	var buf bytes.Buffer
 	_, _ = stdcopy.StdCopy(&buf, &buf, hijackedResp.Reader)
@@ -300,6 +302,11 @@ func (rs *ReplicaSetService) PatchContainer(name string, spec *models.PatchReque
 		return id, newContainerName, errors.WithMessage(err, "runContainer failed")
 	}
 
+	err = rs.containerRemoveBallastStone(ctrVersionName)
+	if err != nil {
+		return id, newContainerName, errors.WithMessage(err, "removeContainerBallastStone failed")
+	}
+
 	// copy the old container's merged files to the new container
 	err = utils.CopyOldMergedToNewContainerMerged(ctrVersionName, newContainerName)
 	if err != nil {
@@ -791,6 +798,11 @@ func (rs *ReplicaSetService) RestartContainer(name string) (id, newContainerName
 		return id, newContainerName, errors.WithMessage(err, "services.runContainer failed")
 	}
 
+	err = rs.containerRemoveBallastStone(ctrVersionName)
+	if err != nil {
+		return id, newContainerName, errors.WithMessage(err, "removeContainerBallastStone failed")
+	}
+
 	// copy the old container's merged files to the new container
 	err = utils.CopyOldMergedToNewContainerMerged(ctrVersionName, newContainerName)
 	if err != nil {
@@ -895,6 +907,14 @@ func (rs *ReplicaSetService) startContainer(ctx context.Context, respId, ctrVers
 		return errors.Wrapf(err, "docker.ContainerStart failed, id: %s, name: %s", respId, ctrVersionName)
 	}
 
+	go func(name string) {
+		time.Sleep(5 * time.Second)
+		err := rs.containerCreateBallastStone(name)
+		if err != nil {
+			log.Errorf("services.containerCreateBallastStone failed, name: %s, err: %v", name, err)
+		}
+	}(ctrVersionName)
+
 	return nil
 }
 
@@ -962,3 +982,38 @@ func (rs *ReplicaSetService) containerMemory(name string) (int64, error) {
 	}
 	return resp.HostConfig.Resources.Memory, nil
 }
+
+func (rs *ReplicaSetService) containerCreateBallastStone(name string) error {
+	containerName := strings.Split(name, "-")[0]
+
+	cmds := models.ContainerExecute{
+		Cmd: []string{
+			"dd",
+			"if=/dev/zero",
+			"of=/" + ballastStone,
+			"bs=1M",
+			"count=5", // 5MB
+		},
+	}
+
+	_, err := rs.ExecuteContainer(containerName, &cmds)
+	if err != nil {
+		return errors.WithMessagef(err, "services.ExecuteContainer failed, container: %s", containerName)
+	}
+
+	return nil
+}
+
+func (rs *ReplicaSetService) containerRemoveBallastStone(name string) error {
+	mergedLayer, err := utils.GetContainerMergedLayer(name)
+	if err != nil {
+		return errors.WithMessagef(err, "utils.GetContainerMergedLayer failed, container: %s", name)
+	}
+	err = os.Remove(mergedLayer + "/" + ballastStone)
+	if err != nil {
+		if err != os.ErrExist {
+			return errors.WithMessagef(err, "remove container ballast stone failed, path: %s", mergedLayer+"/"+ballastStone)
+		}
+	}
+	return nil
+}

internal/services/replicaset_mock.go

@@ -98,11 +98,7 @@ func (rs *ReplicaSetService) runContainer(ctx context.Context, name string, info
 
 	if !onlyCreate {
 		// start container
-		if err = docker.Cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {
-			_ = docker.Cli.ContainerRemove(ctx,
-				resp.ID,
-				container.RemoveOptions{Force: true})
-			info.HostConfig.Resources.DeviceRequests = deviceRequest
+		if err = rs.startContainer(ctx, resp.ID, ctrVersionName); err != nil {
 			return "", "", etcd.PutKeyValue{}, errors.Wrapf(err, "docker.ContainerStart failed, id: %s, name: %s", resp.ID, ctrVersionName)
 		}
 	}

internal/services/replicaset_nomock.go

@@ -79,10 +79,7 @@ func (rs *ReplicaSetService) runContainer(ctx context.Context, name string, info
 
 	if !onlyCreate {
 		// start container
-		if err = docker.Cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {
-			_ = docker.Cli.ContainerRemove(ctx,
-				resp.ID,
-				container.RemoveOptions{Force: true})
+		if err = rs.startContainer(ctx, resp.ID, ctrVersionName); err != nil {
 			return "", "", etcd.PutKeyValue{}, errors.Wrapf(err, "docker.ContainerStart failed, id: %s, name: %s", resp.ID, ctrVersionName)
 		}
 	}