add 'state' test; using WaitingForStatus in insideValidation

validation/create.go

@@ -56,10 +56,10 @@ func main() {
 		}
 		if err != nil {
 			diagnostic["error"] = err.Error()
-		}
-		if e, ok := err.(*exec.ExitError); ok {
-			if len(e.Stderr) > 0 {
-				diagnostic["stderr"] = string(e.Stderr)
+			if e, ok := err.(*exec.ExitError); ok {
+				if len(e.Stderr) > 0 {
+					diagnostic["stderr"] = string(e.Stderr)
+				}
 			}
 		}
 		t.YAML(diagnostic)

validation/pidfile.go

@@ -1,14 +1,16 @@
 package main
 
 import (
-	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 
 	tap "github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/validation/util"
+	uuid "github.com/satori/go.uuid"
 )
 
 func main() {
@@ -25,6 +27,7 @@ func main() {
 	config := util.LifecycleConfig{
 		Actions: util.LifecycleActionCreate | util.LifecycleActionDelete,
 		PreCreate: func(r *util.Runtime) error {
+			r.SetID(uuid.NewV4().String())
 			r.PidFile = tempPidFile
 			return nil
 		},
@@ -42,7 +45,7 @@ func main() {
 				return err
 			}
 			if state.Pid != pid {
-				return errors.New("Wrong pid in the pidfile")
+				return fmt.Errorf("wrong pid %d, expected %d", pid, state.Pid)
 			}
 			return nil
 		},
@@ -51,8 +54,17 @@ func main() {
 	g := util.GetDefaultGenerator()
 	g.SetProcessArgs([]string{"true"})
 	err = util.RuntimeLifecycleValidate(g, config)
+	t.Ok(err == nil, "create with '--pid-file' option works")
 	if err != nil {
-		util.Fatal(err)
+		diagnostic := map[string]string{
+			"error": err.Error(),
+		}
+		if e, ok := err.(*exec.ExitError); ok {
+			if len(e.Stderr) > 0 {
+				diagnostic["stderr"] = string(e.Stderr)
+			}
+		}
+		t.YAML(diagnostic)
 	}
 
 	t.AutoPlan()

validation/state.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/mndrix/tap-go"
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	uuid "github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+	t.Header(0)
+
+	g := util.GetDefaultGenerator()
+	g.SetProcessArgs([]string{"true"})
+	containerID := uuid.NewV4().String()
+
+	cases := []struct {
+		id          string
+		action      util.LifecycleAction
+		errExpected bool
+		err         error
+	}{
+		{"", util.LifecycleActionNone, false, specerror.NewError(specerror.QueryWithoutIDGenError, fmt.Errorf("state MUST generate an error if it is not provided the ID of a container"), rspecs.Version)},
+		{containerID, util.LifecycleActionNone, false, specerror.NewError(specerror.QueryNonExistGenError, fmt.Errorf("state MUST generate an error if a container that does not exist"), rspecs.Version)},
+		{containerID, util.LifecycleActionCreate | util.LifecycleActionDelete, true, specerror.NewError(specerror.QueryStateImplement, fmt.Errorf("state MUST return the state of a container as specified in the State section"), rspecs.Version)},
+	}
+
+	for _, c := range cases {
+		config := util.LifecycleConfig{
+			Actions: c.action,
+			PreCreate: func(r *util.Runtime) error {
+				r.SetID(c.id)
+				return nil
+			},
+			PostCreate: func(r *util.Runtime) error {
+				_, err := r.State()
+				return err
+			},
+		}
+		err := util.RuntimeLifecycleValidate(g, config)
+		t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
+		diagnostic := map[string]string{
+			"reference": c.err.(*specerror.Error).Err.Reference,
+		}
+		if err != nil {
+			diagnostic["error"] = err.Error()
+			if e, ok := err.(*exec.ExitError); ok {
+				if len(e.Stderr) > 0 {
+					diagnostic["stderr"] = string(e.Stderr)
+				}
+			}
+		}
+		t.YAML(diagnostic)
+	}
+
+	t.AutoPlan()
+}

validation/util/test.go

@@ -26,6 +26,8 @@ var (
 type LifecycleAction int
 
 const (
+	// LifecycleActionNone does nothing
+	LifecycleActionNone = 0
 	// LifecycleActionCreate creates a container
 	LifecycleActionCreate = 1 << iota
 	// LifecycleActionStart starts a container
@@ -192,8 +194,10 @@ func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) {
 		return err
 	}
 
-	// FIXME: wait until the container exits and collect its exit code.
-	time.Sleep(1 * time.Second)
+	err = WaitingForStatus(r, LifecycleStatusStopped, 10*time.Second, 1*time.Second)
+	if err != nil {
+		return err
+	}
 
 	stdout, stderr, err := r.ReadStandardStreams()
 	if err != nil {
@@ -259,17 +263,15 @@ func RuntimeLifecycleValidate(g *generate.Generator, config LifecycleConfig) err
 	if err != nil {
 		return err
 	}
+	defer os.RemoveAll(bundleDir)
 	r, err := NewRuntime(RuntimeCommand, bundleDir)
 	if err != nil {
-		os.RemoveAll(bundleDir)
 		return err
 	}
-	defer r.Clean(true, true)
 	err = r.SetConfig(g)
 	if err != nil {
 		return err
 	}
-	r.SetID(uuid.NewV4().String())
 
 	if config.PreCreate != nil {
 		if err := config.PreCreate(&r); err != nil {
@@ -286,6 +288,17 @@ func RuntimeLifecycleValidate(g *generate.Generator, config LifecycleConfig) err
 			}
 			return err
 		}
+		if config.Actions&LifecycleActionDelete != 0 {
+			defer func() {
+				// runtime error or the container is already deleted
+				if _, err := r.State(); err != nil {
+					return
+				}
+				if err := WaitingForStatus(r, LifecycleStatusCreated|LifecycleStatusStopped, time.Second*10, time.Second*1); err != nil {
+					r.Delete()
+				}
+			}()
+		}
 	}
 
 	if config.PostCreate != nil {