WIP: MCO NRI plugin experimentation

cmd/machine-config-daemon/start.go

@@ -19,7 +19,9 @@ import (
 	"github.com/openshift/machine-config-operator/pkg/daemon"
 	"github.com/openshift/machine-config-operator/pkg/daemon/constants"
 	"github.com/openshift/machine-config-operator/pkg/daemon/cri"
+	"github.com/openshift/machine-config-operator/pkg/daemon/nri"
 	"github.com/openshift/machine-config-operator/pkg/version"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 )
@@ -246,6 +248,24 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 		ctrlctx.InformerFactory.Start(stopCh)
 	}
 
+	// Initialize and start NRI plugin for mutation control
+	nriLogger := logrus.New()
+	nriLogger.SetFormatter(&logrus.TextFormatter{
+		FullTimestamp: true,
+	})
+
+	nriPlugin, err := nri.NewAllowMutationsPlugin(nriLogger, nri.DefaultConfigPath)
+	if err != nil {
+		klog.Warningf("Failed to initialize NRI plugin: %v. NRI mutation control will not be available.", err)
+	} else {
+		klog.Infof("Starting NRI AllowMutations plugin")
+		go func() {
+			if err := nriPlugin.Start(ctx); err != nil {
+				klog.Errorf("NRI plugin exited with error: %v", err)
+			}
+		}()
+	}
+
 	if err := dn.Run(stopCh, exitCh, errCh); err != nil {
 		ctrlcommon.WriteTerminationError(err)
 		if errors.Is(err, daemon.ErrAuxiliary) {

go.mod

@@ -1,6 +1,8 @@
 module github.com/openshift/machine-config-operator
 
-go 1.24.0
+go 1.24.3
+
+toolchain go1.24.5
 
 require (
 	github.com/Azure/ARO-RP v0.0.0-20250602035759-0693f32d5ccc
@@ -11,6 +13,7 @@ require (
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c
 	github.com/clarketm/json v1.17.1
+	github.com/containerd/nri v0.10.0
 	github.com/containers/common v0.62.1
 	github.com/containers/image/v5 v5.35.0
 	github.com/containers/kubensmnt v1.2.0
@@ -102,7 +105,7 @@ require (
 	github.com/containerd/errdefs v1.0.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/ttrpc v1.2.6 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect
 	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
@@ -142,6 +145,7 @@ require (
 	github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect
 	github.com/karrick/godirwalk v1.17.0 // indirect
 	github.com/kkHAIKE/contextcheck v1.1.5 // indirect
+	github.com/knqyf263/go-plugin v0.9.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lasiar/canonicalheader v1.1.2 // indirect
 	github.com/ldez/grignotin v0.7.0 // indirect
@@ -180,6 +184,7 @@ require (
 	github.com/sigstore/rekor v1.3.10 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/tetratelabs/wazero v1.9.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/timonwong/loggercheck v0.10.1 // indirect
@@ -366,7 +371,7 @@ require (
 	github.com/securego/gosec/v2 v2.21.4 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
 	github.com/sigstore/sigstore v1.9.3 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sirupsen/logrus v1.9.3
 	github.com/sivchari/containedctx v1.0.3 // indirect
 	github.com/sivchari/tenv v1.12.1 // indirect
 	github.com/sonatard/noctx v0.1.0 // indirect

go.sum

@@ -136,8 +136,10 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X
 github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC4=
-github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
+github.com/containerd/nri v0.10.0 h1:bt2NzfvlY6OJE0i+fB5WVeGQEycxY7iFVQpEbh7J3Go=
+github.com/containerd/nri v0.10.0/go.mod h1:5VyvLa/4uL8FjyO8nis1UjbCutXDpngil17KvBSL6BU=
+github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ=
+github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
 github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40=
 github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk=
 github.com/containers/common v0.62.1 h1:durvu7Kelb8PYgX7bwuAg/d5LKj2hs3cAaqcU7Vnqus=
@@ -476,6 +478,8 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI=
+github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -823,6 +827,8 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR
 github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
 github.com/tetafro/godot v1.4.20 h1:z/p8Ek55UdNvzt4TFn2zx2KscpW4rWqcnUrdmvWJj7E=
 github.com/tetafro/godot v1.4.20/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio=
+github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
+github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
 github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=

manifests/machineconfigdaemon/daemonset.yaml

@@ -43,6 +43,10 @@ spec:
           - mountPath: /rootfs
             name: rootfs
             mountPropagation: HostToContainer
+          - mountPath: /var/run/nri
+            name: nri-socket
+          - mountPath: /etc/crio/nri_plugins
+            name: nri-plugins-config
         livenessProbe:
           initialDelaySeconds: 120
           periodSeconds: 30
@@ -107,6 +111,14 @@ spec:
         - name: rootfs
           hostPath:
             path: /
+        - name: nri-socket
+          hostPath:
+            path: /var/run/nri
+            type: DirectoryOrCreate
+        - name: nri-plugins-config
+          hostPath:
+            path: /etc/crio/nri_plugins
+            type: DirectoryOrCreate
         - name: proxy-tls
           secret:
             secretName: proxy-tls

pkg/daemon/nri/allow_mutations.go

@@ -0,0 +1,192 @@
+package nri
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/containerd/nri/pkg/api"
+	"github.com/containerd/nri/pkg/stub"
+	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	// PluginName is the name registered with NRI
+	PluginName = "AllowMutations"
+	// PluginIdx is the plugin index - high value ensures it runs last
+	// NRI plugins are invoked in index order, so 99 ensures this runs after all other plugins
+	PluginIdx = "99"
+	// DefaultConfigPath is the default location for plugin configuration
+	DefaultConfigPath = "/etc/crio/nri_plugins/AllowMutations/config.yaml"
+	// DefaultSocketPath is the default NRI socket path
+	DefaultSocketPath = "/var/run/nri/nri.sock"
+)
+
+// Config represents the plugin configuration
+type Config struct {
+	AllowedNamespaces []string `json:"allowed_namespaces" yaml:"allowed_namespaces"`
+}
+
+// AllowMutationsPlugin is the NRI plugin implementation
+type AllowMutationsPlugin struct {
+	stub   stub.Stub
+	config Config
+	mu     sync.RWMutex
+	log    *logrus.Logger
+}
+
+// NewAllowMutationsPlugin creates a new instance of the AllowMutations NRI plugin
+func NewAllowMutationsPlugin(logger *logrus.Logger, configPath string) (*AllowMutationsPlugin, error) {
+	if logger == nil {
+		logger = logrus.StandardLogger()
+	}
+
+	p := &AllowMutationsPlugin{
+		log: logger,
+		config: Config{
+			AllowedNamespaces: []string{},
+		},
+	}
+
+	// Load configuration (never fails - defaults to deny all on error)
+	p.loadConfig(configPath)
+
+	// Create NRI stub
+	opts := []stub.Option{
+		stub.WithOnClose(p.onClose),
+		stub.WithPluginName(PluginName),
+		stub.WithPluginIdx(PluginIdx),
+	}
+
+	var err error
+	p.stub, err = stub.New(p, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create NRI stub: %w", err)
+	}
+
+	return p, nil
+}
+
+// Start starts the NRI plugin
+func (p *AllowMutationsPlugin) Start(ctx context.Context) error {
+	p.log.Infof("Starting %s NRI plugin", PluginName)
+	return p.stub.Run(ctx)
+}
+
+// Stop stops the NRI plugin
+func (p *AllowMutationsPlugin) Stop() {
+	if p.stub != nil {
+		p.stub.Stop()
+	}
+}
+
+// Configure is called when the plugin is configured by the runtime
+func (p *AllowMutationsPlugin) Configure(_ context.Context, _ /* config */, runtime, version string) (stub.EventMask, error) {
+	p.log.Infof("Connected to %s/%s", runtime, version)
+
+	// We want to intercept container creation to enforce mutation policy
+	mask := api.MustParseEventMask("RunPodSandbox,CreateContainer")
+
+	p.log.Infof("Configured %s plugin with event mask: %v", PluginName, mask)
+	return mask, nil
+}
+
+// Synchronize is called when the plugin needs to synchronize state
+func (p *AllowMutationsPlugin) Synchronize(_ context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) {
+	p.log.Infof("Synchronized state with runtime (%d pods, %d containers)", len(pods), len(containers))
+	return nil, nil
+}
+
+// Shutdown is called when the runtime is shutting down
+func (p *AllowMutationsPlugin) Shutdown(_ context.Context) {
+	p.log.Info("Runtime shutting down")
+}
+
+// RunPodSandbox is called when a pod sandbox is being created
+func (p *AllowMutationsPlugin) RunPodSandbox(_ context.Context, pod *api.PodSandbox) error {
+	namespace := pod.GetNamespace()
+	podName := pod.GetName()
+
+	if !p.isNamespaceAllowed(namespace) {
+		p.log.Infof("Denying mutations for pod %s/%s (namespace not in allowed list)", namespace, podName)
+	} else {
+		p.log.Debugf("Allowing mutations for pod %s/%s", namespace, podName)
+	}
+
+	return nil
+}
+
+// CreateContainer is called when a container is being created
+// This is where we enforce the mutation policy by returning nil adjustments
+// for namespaces not in the allowed list
+func (p *AllowMutationsPlugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
+	namespace := pod.GetNamespace()
+	podName := pod.GetName()
+	containerName := container.GetName()
+
+	if !p.isNamespaceAllowed(namespace) {
+		p.log.Infof("Denying mutations for container %s in pod %s/%s (namespace not in allowed list)",
+			containerName, namespace, podName)
+		// Return nil adjustment - this prevents any mutations from being applied
+		return nil, nil, nil
+	}
+
+	p.log.Debugf("Allowing mutations for container %s in pod %s/%s",
+		containerName, namespace, podName)
+
+	// For allowed namespaces, return empty adjustments to allow other plugins to make changes
+	return &api.ContainerAdjustment{}, nil, nil
+}
+
+// isNamespaceAllowed checks if mutations are allowed in the given namespace
+func (p *AllowMutationsPlugin) isNamespaceAllowed(namespace string) bool {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+
+	for _, ns := range p.config.AllowedNamespaces {
+		if ns == namespace {
+			return true
+		}
+	}
+	return false
+}
+
+// loadConfig loads the plugin configuration from the specified path.
+// All errors are treated as non-fatal to ensure the plugin always starts.
+// If configuration cannot be loaded, the plugin defaults to denying all namespaces.
+func (p *AllowMutationsPlugin) loadConfig(configPath string) {
+	if configPath == "" {
+		configPath = DefaultConfigPath
+	}
+
+	data, err := os.ReadFile(configPath)
+	if err != nil {
+		// Treat all read errors as non-fatal - plugin starts with empty allowed list
+		if os.IsNotExist(err) {
+			p.log.Warnf("Configuration file not found at %s, using default (deny all namespaces)", configPath)
+		} else {
+			p.log.Warnf("Failed to read configuration file at %s: %v. Using default (deny all namespaces)", configPath, err)
+		}
+		return
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if err := yaml.Unmarshal(data, &p.config); err != nil {
+		// Invalid YAML is also non-fatal - plugin starts with empty allowed list
+		p.log.Warnf("Failed to parse configuration YAML at %s: %v. Using default (deny all namespaces)", configPath, err)
+		p.config.AllowedNamespaces = []string{}
+		return
+	}
+
+	p.log.Infof("Loaded configuration: allowed_namespaces=%v", p.config.AllowedNamespaces)
+}
+
+// onClose is called when the connection to the runtime is closed
+func (p *AllowMutationsPlugin) onClose() {
+	p.log.Warnf("Connection to runtime lost, plugin will exit")
+	os.Exit(1)
+}