api, adaptation: implement pluggable validation API and a default validator plugin.

Makefile

@@ -56,6 +56,7 @@ PLUGINS := \
 	$(BIN_PATH)/ulimit-adjuster \
 	$(BIN_PATH)/v010-adapter \
 	$(BIN_PATH)/template \
+	$(BIN_PATH)/default-validator \
 	$(BIN_PATH)/wasm
 
 
@@ -134,6 +135,11 @@ $(BIN_PATH)/template: $(wildcard plugins/template/*.go)
 	$(Q)echo "Building $@..."; \
 	cd $(dir $<) && $(GO_BUILD) -o $@ .
 
+$(BIN_PATH)/default-validator: $(wildcard plugins/default-validator/external/*.go) $(wildcard plugins/default-validator/*.go)
+	$(Q)echo "Building $@..."; \
+	cd $(dir $<) && $(GO_BUILD) -o $@ .
+
+
 ifneq ($(TINYGO_DOCKER),1)
 $(BIN_PATH)/wasm: $(wildcard plugins/wasm/*.go)
 	$(Q)echo "Building $@..."; \

go.mod

@@ -17,6 +17,7 @@ require (
 	golang.org/x/sys v0.21.0
 	google.golang.org/grpc v1.57.1
 	google.golang.org/protobuf v1.34.1
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -35,7 +36,6 @@ require (
 	golang.org/x/tools v0.21.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 replace github.com/opencontainers/runtime-tools v0.9.0 => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809

pkg/adaptation/adaptation.go

@@ -27,11 +27,15 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/containerd/nri/pkg/adaptation/builtin"
 	"github.com/containerd/nri/pkg/api"
 	"github.com/containerd/nri/pkg/log"
+	validator "github.com/containerd/nri/plugins/default-validator/builtin"
 	"github.com/containerd/ttrpc"
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
+
+	"google.golang.org/protobuf/proto"
 )
 
 const (
@@ -67,6 +71,8 @@ type Adaptation struct {
 	serverOpts  []ttrpc.ServerOpt
 	listener    net.Listener
 	plugins     []*plugin
+	validators  []*plugin
+	builtin     []*builtin.BuiltinPlugin
 	syncLock    sync.RWMutex
 	wasmService *api.PluginPlugin
 }
@@ -120,6 +126,24 @@ func WithTTRPCOptions(clientOpts []ttrpc.ClientOpts, serverOpts []ttrpc.ServerOp
 	}
 }
 
+// WithBuiltinPlugins sets extra builtin plugins to register.
+func WithBuiltinPlugins(plugins ...*builtin.BuiltinPlugin) Option {
+	return func(r *Adaptation) error {
+		r.builtin = append(r.builtin, plugins...)
+		return nil
+	}
+}
+
+// WithDefaultValidator sets up builtin validator plugin if it is configured.
+func WithDefaultValidator(cfg *validator.DefaultValidatorConfig) Option {
+	return func(r *Adaptation) error {
+		if plugin := validator.GetDefaultValidator(cfg); plugin != nil {
+			r.builtin = append([]*builtin.BuiltinPlugin{plugin}, r.builtin...)
+		}
+		return nil
+	}
+}
+
 // New creates a new NRI Runtime.
 func New(name, version string, syncFn SyncFn, updateFn UpdateFn, opts ...Option) (*Adaptation, error) {
 	var err error
@@ -248,8 +272,22 @@ func (r *Adaptation) CreateContainer(ctx context.Context, req *CreateContainerRe
 	defer r.Unlock()
 	defer r.removeClosedPlugins()
 
-	result := collectCreateContainerResult(req)
+	var (
+		result   = collectCreateContainerResult(req)
+		validate *ValidateContainerAdjustmentRequest
+	)
+
+	if r.hasValidators() {
+		validate = &ValidateContainerAdjustmentRequest{
+			Pod:       req.Pod,
+			Container: proto.Clone(req.Container).(*Container),
+		}
+	}
+
 	for _, plugin := range r.plugins {
+		if validate != nil {
+			validate.AddPlugin(plugin.base, plugin.idx)
+		}
 		rpl, err := plugin.createContainer(ctx, req)
 		if err != nil {
 			return nil, err
@@ -260,7 +298,7 @@ func (r *Adaptation) CreateContainer(ctx context.Context, req *CreateContainerRe
 		}
 	}
 
-	return result.createContainerResponse(), nil
+	return r.validateContainerAdjustment(ctx, validate, result)
 }
 
 // PostCreateContainer relays the corresponding CRI event to plugins.
@@ -363,6 +401,40 @@ func (r *Adaptation) updateContainers(ctx context.Context, req []*ContainerUpdat
 	return r.updateFn(ctx, req)
 }
 
+// Validate requested container adjustments.
+func (r *Adaptation) validateContainerAdjustment(ctx context.Context, req *ValidateContainerAdjustmentRequest, result *result) (*CreateContainerResponse, error) {
+	rpl := result.createContainerResponse()
+
+	if req == nil || len(r.validators) == 0 {
+		return rpl, nil
+	}
+
+	req.AddResponse(rpl)
+	req.AddOwners(result.owners)
+
+	errors := make(chan error, len(r.validators))
+	wg := sync.WaitGroup{}
+
+	for _, p := range r.validators {
+		wg.Add(1)
+		go func(p *plugin) {
+			defer wg.Done()
+			errors <- p.ValidateContainerAdjustment(ctx, req)
+		}(p)
+	}
+
+	wg.Wait()
+	close(errors)
+
+	for err := range errors {
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return rpl, nil
+}
+
 // Start up pre-installed plugins.
 func (r *Adaptation) startPlugins() (retErr error) {
 	var plugins []*plugin
@@ -382,6 +454,20 @@ func (r *Adaptation) startPlugins() (retErr error) {
 		}
 	}()
 
+	for _, b := range r.builtin {
+		log.Infof(noCtx, "starting builtin NRI plugin %q...", b.Index+"-"+b.Base)
+		p, err := r.newBuiltinPlugin(b)
+		if err != nil {
+			return fmt.Errorf("failed to initialize builtin NRI plugin %q: %v", b.Base, err)
+		}
+
+		if err := p.start(r.name, r.version); err != nil {
+			return fmt.Errorf("failed to start builtin NRI plugin %q: %v", b.Base, err)
+		}
+
+		plugins = append(plugins, p)
+	}
+
 	for i, name := range names {
 		log.Infof(noCtx, "starting pre-installed NRI plugin %q...", name)
 
@@ -439,12 +525,15 @@ func (r *Adaptation) stopPlugins() {
 }
 
 func (r *Adaptation) removeClosedPlugins() {
-	var active, closed []*plugin
+	var active, closed, validators []*plugin
 	for _, p := range r.plugins {
 		if p.isClosed() {
 			closed = append(closed, p)
 		} else {
 			active = append(active, p)
+			if p.isContainerAdjustmentValidator() {
+				validators = append(validators, p)
+			}
 		}
 	}
 
@@ -455,7 +544,9 @@ func (r *Adaptation) removeClosedPlugins() {
 			}
 		}()
 	}
+
 	r.plugins = active
+	r.validators = validators
 }
 
 func (r *Adaptation) startListener() error {
@@ -519,6 +610,9 @@ func (r *Adaptation) acceptPluginConnections(l net.Listener) error {
 			} else {
 				r.Lock()
 				r.plugins = append(r.plugins, p)
+				if p.isContainerAdjustmentValidator() {
+					r.validators = append(r.validators, p)
+				}
 				r.sortPlugins()
 				r.Unlock()
 				log.Infof(ctx, "plugin %q connected and synchronized", p.name())
@@ -588,12 +682,25 @@ func (r *Adaptation) sortPlugins() {
 	sort.Slice(r.plugins, func(i, j int) bool {
 		return r.plugins[i].idx < r.plugins[j].idx
 	})
+	sort.Slice(r.validators, func(i, j int) bool {
+		return r.validators[i].idx < r.validators[j].idx
+	})
 	if len(r.plugins) > 0 {
 		log.Infof(noCtx, "plugin invocation order")
 		for i, p := range r.plugins {
 			log.Infof(noCtx, "  #%d: %q (%s)", i+1, p.name(), p.qualifiedName())
 		}
 	}
+	if len(r.validators) > 0 {
+		log.Infof(noCtx, "validator plugins")
+		for _, p := range r.validators {
+			log.Infof(noCtx, "  %q (%s)", p.name(), p.qualifiedName())
+		}
+	}
+}
+
+func (r *Adaptation) hasValidators() bool {
+	return len(r.validators) > 0
 }
 
 func (r *Adaptation) requestPluginSync() {

pkg/adaptation/adaptation_suite_test.go

@@ -18,6 +18,7 @@ package adaptation_test
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
@@ -921,6 +922,99 @@ var _ = Describe("Plugin container creation adjustments", func() {
 		)
 	})
 
+	When("there are validating plugins", func() {
+		BeforeEach(func() {
+			s.Prepare(
+				&mockRuntime{},
+				&mockPlugin{idx: "00", name: "foo"},
+				&mockPlugin{idx: "00", name: "validator"},
+			)
+		})
+
+		DescribeTable("validation result should be honored",
+			func(subject string, shouldFail bool, expected *api.ContainerAdjustment) {
+				var (
+					runtime = s.runtime
+					plugins = s.plugins
+					ctx     = context.Background()
+
+					pod = &api.PodSandbox{
+						Id:        "pod0",
+						Name:      "pod0",
+						Uid:       "uid0",
+						Namespace: "default",
+					}
+					ctr = &api.Container{
+						Id:           "ctr0",
+						PodSandboxId: "pod0",
+						Name:         "ctr0",
+						State:        api.ContainerState_CONTAINER_CREATED, // XXX FIXME-kludge
+						Args: []string{
+							"echo",
+							"original",
+							"argument",
+							"list",
+						},
+					}
+
+					forbidden = "no-no"
+				)
+
+				create := func(p *mockPlugin, _ *api.PodSandbox, _ *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
+					plugin := p.idx + "-" + p.name
+					a := &api.ContainerAdjustment{}
+					switch subject {
+					case "annotation":
+						key := "key"
+						if shouldFail {
+							key = forbidden
+						}
+						a.AddAnnotation(key, plugin)
+					}
+
+					return a, nil, nil
+				}
+
+				validate := func(_ *mockPlugin, req *api.ValidateContainerAdjustmentRequest) error {
+					_, ok := req.Owners.AnnotationOwner(req.Container.Id, forbidden)
+					if ok {
+						return fmt.Errorf("forbidden annotation %q adjusted", forbidden)
+					}
+					return nil
+				}
+
+				plugins[0].createContainer = create
+				plugins[1].validateAdjustment = validate
+				s.Startup()
+
+				podReq := &api.RunPodSandboxRequest{Pod: pod}
+				Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed())
+				ctrReq := &api.CreateContainerRequest{
+					Pod:       pod,
+					Container: ctr,
+				}
+				reply, err := runtime.CreateContainer(ctx, ctrReq)
+				if shouldFail {
+					Expect(err).ToNot(BeNil())
+				} else {
+					Expect(err).To(BeNil())
+					Expect(protoEqual(reply.Adjust.Strip(), expected.Strip())).Should(BeTrue(),
+						protoDiff(reply.Adjust, expected))
+				}
+			},
+
+			Entry("adjust allowed annotation", "annotation", false,
+				&api.ContainerAdjustment{
+					Annotations: map[string]string{
+						"key": "00-foo",
+					},
+				},
+			),
+
+			Entry("adjust forbidden annotation", "annotation", true, nil),
+		)
+	})
+
 })
 
 // --------------------------------------------