Merge branch 'main' into fix/go-system-prompt

Author: supreme-gg-gg

.github/workflows/ci.yaml

@@ -2,11 +2,11 @@ name: CI Build
 
 on:
   push:
-    branches: [main, release/v0.7.x]
+    branches: [main, "release/**"]
     paths-ignore:
       - "**/*.md"
   pull_request:
-    branches: [main, release/v0.7.x]
+    branches: [main, "release/**"]
     paths-ignore:
       - "**/*.md"
   workflow_dispatch:

.github/workflows/migration-immutability.yaml

@@ -2,7 +2,7 @@ name: Migration Immutability
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, "release/**"]
     paths:
       - "go/core/pkg/migrations/**"
 

.github/workflows/sqlc-generate-check.yaml

@@ -2,7 +2,7 @@ name: sqlc Generate Check
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, "release/**"]
     paths:
       - "go/core/internal/database/queries/**"
       - "go/core/internal/database/sqlc.yaml"

.github/workflows/ui-chromatic.yaml

@@ -5,11 +5,11 @@ name: UI Storybook & Chromatic
 
 on:
   push:
-    branches: [main, release/v0.7.x]
+    branches: [main, "release/**"]
     paths:
       - "ui/**"
   pull_request:
-    branches: [main, release/v0.7.x]
+    branches: [main, "release/**"]
     paths:
       - "ui/**"
   workflow_dispatch:

examples/substrate-openclaw/README.md

@@ -1,30 +1,31 @@
-# kagent agents and agentharness on substrate
-
-Follow these instructions to install substrate on a kind cluster. This feature allows you to run AgentHarness (OpenClaw) and declerative Go agents in substrate.
+# kagent agents and AgentHarness on Substrate
 
+Follow these instructions to install Substrate on a kind cluster. This feature allows you to run AgentHarness (OpenClaw) and declarative Go SandboxAgents on Agent Substrate.
 
 ## 1. Install Substrate on your Kind cluster
 
-This assumes you've configured kind cluster using `make create-kind-cluster`.
+This assumes you've configured a kind cluster using `make create-kind-cluster`.
 
-Create the substrate-values.yaml file:
+Create a `substrate-values.yaml` file:
 
 ```yaml
 atelet:
   extraArgs:
     - --localhost-registry-replacement=kind-registry:5000
 ```
 
-Then install substrate and kagent:
+Then install the Substrate platform and kagent:
 
 ```bash
-export ATEOM_VERSION=v0.0.6
+export SUBSTRATE_VERSION=0.0.7
 
 helm upgrade --install substrate-crds \
-  oci://ghcr.io/kagent-dev/substrate/helm/substrate-crds
+  oci://ghcr.io/kagent-dev/substrate/helm/substrate-crds \
+  --version "${SUBSTRATE_VERSION}"
 
 helm upgrade --install substrate \
   oci://ghcr.io/kagent-dev/substrate/helm/substrate \
+  --version "${SUBSTRATE_VERSION}" \
   --namespace ate-system \
   --create-namespace -f substrate-values.yaml
 
@@ -33,18 +34,24 @@ make helm-install KAGENT_HELM_EXTRA_ARGS="\
   --set controller.substrate.ateApiEndpoint=dns:///api.ate-system.svc:443 \
   --set controller.substrate.ateApiInsecure=true \
   --set substrateWorkerPool.create=true \
-  --set substrateWorkerPool.ateomImage=ghcr.io/kagent-dev/substrate/ateom-gvisor:${ATEOM_VERSION}"
+  --set substrateWorkerPool.ateomImage=ghcr.io/kagent-dev/substrate/ateom-gvisor:v${SUBSTRATE_VERSION}"
 ```
 
-## kagent AgentHarness with substrate runtime
+When `substrateWorkerPool.create=true`, the kagent chart installs a namespace-scoped `WorkerPool` with:
+
+- `spec.sandboxClass: gvisor`
+- label `kagent.dev/worker-pool: kagent-default` (matches generated `ActorTemplate` selectors)
+- controller default `workerPool` name set to that pool when `create=true`
+
+**Zero-downtime rollouts:** SandboxAgent config and image rollouts retain the previous `ActorTemplate` until the new golden is Ready (blue-green). Use `substrateWorkerPool.replicas: 2` or higher so a spare worker can build the new golden while the current one keeps serving chat.
 
-kagent generates a per-harness `ActorTemplate` and uses an existing `WorkerPool`.
+## 2. AgentHarness with Substrate runtime
 
-The generated `ActorTemplate` uses `controller.substrate.pauseImage`, `controller.substrate.runscAMD64URL`, `controller.substrate.runscAMD64SHA256`, `controller.substrate.runscARM64URL`, and `controller.substrate.runscARM64SHA256` from the Helm values Override them with `--set` or a values file when you need to pin a different gVisor build.
+kagent generates a per-harness `ActorTemplate` and schedules actors onto an existing `WorkerPool`.
 
 Create a harness. If `snapshotsConfig` is omitted, kagent defaults it to `gs://ate-snapshots/<namespace>/<agentharnessname>`.
 
-- **Worker pool** — reference an existing pool (`workerPoolRef`) or configure a controller default WorkerPool
+- **Worker pool** — reference an existing pool (`workerPoolRef`) or configure a controller default WorkerPool. The target pool must carry label `kagent.dev/worker-pool: <pool-name>`. The kagent Helm-managed pool gets this label automatically; externally owned pools must add it manually.
 
 ```yaml
 apiVersion: kagent.dev/v1alpha2
@@ -83,23 +90,15 @@ metadata:
     kagent.dev/agent-harness: peterj-claw
 spec:
   pauseImage: gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da
-  runsc:
-    amd64:
-      url: gs://gvisor/releases/nightly/2026-06-02/x86_64/runsc
-      sha256Hash: efd12935f6654c91a1389710eb8dfa4d12b6b9be00db87526dc2eb584ad00119
-    arm64:
-      url: gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc
-      sha256Hash: 1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9
-  workerPoolRef:
-    name: peterj-claw-wp
-    namespace: kagent
+  sandboxClass: gvisor
+  workerSelector:
+    matchLabels:
+      kagent.dev/worker-pool: kagent-default
   snapshotsConfig:
     location: gs://ate-snapshots/kagent/peterj-claw
   containers:
   - name: openclaw
     image: ghcr.io/kagent-dev/nemoclaw/sandbox-base@sha256:d52bee415dc4c0dba7164f9eabe727574c056d4f211781f20af249707883a3b4
-    ports:
-    - containerPort: 80
     command:
     - /bin/sh
     - -c

go/api/httpapi/substrate.go

@@ -28,7 +28,8 @@ type SubstrateActorTemplateEntry struct {
 	Phase           string `json:"phase,omitempty"`
 	GoldenActorID   string `json:"goldenActorId,omitempty"`
 	GoldenSnapshot  string `json:"goldenSnapshot,omitempty"`
-	WorkerPoolRef   string `json:"workerPoolRef,omitempty"`
+	SandboxClass    string `json:"sandboxClass,omitempty"`
+	WorkerSelector  string `json:"workerSelector,omitempty"`
 	HarnessName     string `json:"harnessName,omitempty"`
 	ManagedByKagent bool   `json:"managedByKagent"`
 }
@@ -42,7 +43,8 @@ type SubstrateActorEntry struct {
 	AteomPodNamespace      string `json:"ateomPodNamespace,omitempty"`
 	AteomPodName           string `json:"ateomPodName,omitempty"`
 	AteomPodIP             string `json:"ateomPodIp,omitempty"`
-	LastSnapshot           string `json:"lastSnapshot,omitempty"`
+	LatestSnapshot         string `json:"latestSnapshot,omitempty"`
+	WorkerPoolName         string `json:"workerPoolName,omitempty"`
 	InProgressSnapshot     string `json:"inProgressSnapshot,omitempty"`
 	Version                int64  `json:"version,omitempty"`
 }

go/core/internal/controller/reconciler/reconciler.go

@@ -14,11 +14,13 @@ import (
 	"strings"
 	"time"
 
+	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 	"github.com/hashicorp/go-multierror"
 	reconcilerutils "github.com/kagent-dev/kagent/go/core/internal/controller/reconciler/utils"
 	"github.com/kagent-dev/kagent/go/core/internal/controller/translator"
 	"github.com/kagent-dev/kagent/go/core/pkg/egress"
 	"github.com/kagent-dev/kagent/go/core/pkg/sandboxbackend"
+	"github.com/kagent-dev/kagent/go/core/pkg/sandboxbackend/substrate"
 	"github.com/kagent-dev/kmcp/api/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -150,11 +152,19 @@ func (a *kagentReconciler) ReconcileKagentSandboxAgent(ctx context.Context, req
 	}
 
 	err := a.reconcileSandboxAgent(ctx, sandboxAgent)
+	if errors.Is(err, substrate.ErrActorTemplateReconcilePending) {
+		// A spec-drift recreate is mid-flight; report not-ready without failing
+		// the resource and return the sentinel so the controller requeues.
+		if statusErr := a.reconcileSandboxAgentStatus(ctx, sandboxAgent, nil, true); statusErr != nil {
+			return statusErr
+		}
+		return err
+	}
 	if err != nil {
 		reconcileLog.Error(err, "failed to reconcile sandboxagent", "sandboxagent", req.NamespacedName)
 	}
 
-	return a.reconcileSandboxAgentStatus(ctx, sandboxAgent, err)
+	return a.reconcileSandboxAgentStatus(ctx, sandboxAgent, err, false)
 }
 
 func (a *kagentReconciler) handleDeletedAgentResource(ctx context.Context, req ctrl.Request, resourceName string) error {
@@ -239,14 +249,18 @@ func (a *kagentReconciler) reconcileSandboxAgent(ctx context.Context, sa *v1alph
 	})
 }
 
-func (a *kagentReconciler) reconcileSandboxAgentStatus(ctx context.Context, sa *v1alpha2.SandboxAgent, reconcileErr error) error {
+func (a *kagentReconciler) reconcileSandboxAgentStatus(ctx context.Context, sa *v1alpha2.SandboxAgent, reconcileErr error, actorTemplatePending bool) error {
 	deployedCondition := metav1.Condition{
 		Type:               v1alpha2.AgentConditionTypeReady,
 		Status:             metav1.ConditionUnknown,
 		ObservedGeneration: sa.Generation,
 	}
 
-	if a.sandboxBackend == nil {
+	if actorTemplatePending {
+		deployedCondition.Status = metav1.ConditionFalse
+		deployedCondition.Reason = "ActorTemplateRecreating"
+		deployedCondition.Message = "waiting for ActorTemplate golden actor deletion and recreate"
+	} else if a.sandboxBackend == nil {
 		deployedCondition.Status = metav1.ConditionUnknown
 		deployedCondition.Reason = "SandboxBackendNotConfigured"
 		deployedCondition.Message = "Sandbox backend is not configured"
@@ -924,12 +938,31 @@ func (r *kagentReconciler) GetOwnedResourceTypes() []client.Object {
 // Function initially copied from https://github.com/open-telemetry/opentelemetry-operator/blob/e6d96f006f05cff0bc3808da1af69b6b636fbe88/internal/controllers/common.go#L141-L192
 func (a *kagentReconciler) reconcileDesiredObjects(ctx context.Context, owner metav1.Object, desiredObjects []client.Object, ownedObjects map[types.UID]client.Object) error {
 	var errs []error
+	actorTemplatePending := false
 	for _, desired := range desiredObjects {
 		l := reconcileLog.WithValues(
 			"object_name", desired.GetName(),
 			"object_kind", desired.GetObjectKind(),
 		)
 
+		// Substrate ActorTemplate.spec is immutable, delegate to the sandbox backend to handle spec drift.
+		if _, ok := desired.(*atev1alpha1.ActorTemplate); ok {
+			if r, ok := a.sandboxBackend.(actorTemplateReconciler); ok {
+				if err := r.ReconcileActorTemplate(ctx, desired); err != nil {
+					if errors.Is(err, substrate.ErrActorTemplateReconcilePending) {
+						actorTemplatePending = true
+						pruneOwnedActorTemplate(ownedObjects, desired)
+						continue
+					}
+					l.Error(err, "failed to reconcile ActorTemplate")
+					errs = append(errs, err)
+					continue
+				}
+				pruneOwnedActorTemplate(ownedObjects, desired)
+				continue
+			}
+		}
+
 		// existing is an object the controller runtime will hydrate for us
 		// we obtain the existing object by deep copying the desired object because it's the most convenient way
 		existing := desired.DeepCopyObject().(client.Object)
@@ -951,6 +984,9 @@ func (a *kagentReconciler) reconcileDesiredObjects(ctx context.Context, owner me
 	if len(errs) > 0 {
 		return fmt.Errorf("failed to create objects for %s: %w", owner.GetName(), errors.Join(errs...))
 	}
+	if actorTemplatePending {
+		return substrate.ErrActorTemplateReconcilePending
+	}
 
 	// Pruning owned objects in the cluster which are not should not be present after the reconciliation.
 	err := a.deleteObjects(ctx, ownedObjects)
@@ -961,6 +997,25 @@ func (a *kagentReconciler) reconcileDesiredObjects(ctx context.Context, owner me
 	return nil
 }
 
+// actorTemplateReconciler is implemented by sandbox backends that own substrate
+// ActorTemplate objects and need immutable-spec aware create/recreate semantics.
+type actorTemplateReconciler interface {
+	ReconcileActorTemplate(ctx context.Context, desired client.Object) error
+}
+
+// pruneOwnedActorTemplate removes the live ActorTemplate matching desired from the
+// prune set so it is not garbage collected.
+func pruneOwnedActorTemplate(owned map[types.UID]client.Object, desired client.Object) {
+	for uid, obj := range owned {
+		if _, ok := obj.(*atev1alpha1.ActorTemplate); !ok {
+			continue
+		}
+		if obj.GetName() == desired.GetName() && obj.GetNamespace() == desired.GetNamespace() {
+			delete(owned, uid)
+		}
+	}
+}
+
 // modified version of controllerutil.CreateOrUpdate to support proto based objects like istio
 func createOrUpdate(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) {
 	key := client.ObjectKeyFromObject(obj)

go/core/internal/controller/sandboxagent_controller.go

@@ -18,6 +18,7 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
 
@@ -86,6 +87,9 @@ func (r *SandboxAgentController) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	if err := r.Reconciler.ReconcileKagentSandboxAgent(ctx, req); err != nil {
+		if errors.Is(err, substrate.ErrActorTemplateReconcilePending) {
+			return ctrl.Result{RequeueAfter: agentHarnessNotReadyRequeue}, nil
+		}
 		return ctrl.Result{}, err
 	}
 	return ctrl.Result{}, nil

go/core/internal/httpserver/handlers/substrate.go

@@ -13,6 +13,7 @@ import (
 	"github.com/kagent-dev/kagent/go/core/internal/httpserver/errors"
 	"github.com/kagent-dev/kagent/go/core/pkg/auth"
 	"github.com/kagent-dev/kagent/go/core/pkg/sandboxbackend/substrate"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
@@ -137,20 +138,15 @@ func (h *SubstrateHandler) listSubstrateCRs(ctx context.Context, namespace strin
 			Phase:           string(tmpl.Status.Phase),
 			GoldenActorID:   tmpl.Status.GoldenActorID,
 			GoldenSnapshot:  tmpl.Status.GoldenSnapshot,
+			SandboxClass:    string(tmpl.Spec.SandboxClass),
+			WorkerSelector:  labelSelectorString(ctx, tmpl.Spec.WorkerSelector),
 			ManagedByKagent: tmpl.Labels["app.kubernetes.io/managed-by"] == "kagent",
 		}
 		if harness := strings.TrimSpace(tmpl.Labels[substrate.HarnessLabelKey]); harness != "" {
 			entry.HarnessName = harness
 		} else if agentName := substrate.SandboxAgentNameFromLabels(tmpl.Labels); agentName != "" {
 			entry.HarnessName = agentName
 		}
-		if ref := tmpl.Spec.WorkerPoolRef; ref.Name != "" {
-			wpNS := ref.Namespace
-			if wpNS == "" {
-				wpNS = tmpl.Namespace
-			}
-			entry.WorkerPoolRef = wpNS + "/" + ref.Name
-		}
 		templates = append(templates, entry)
 	}
 
@@ -215,12 +211,41 @@ func actorEntryFromPB(a *ateapipb.Actor) api.SubstrateActorEntry {
 		AteomPodNamespace:      a.GetAteomPodNamespace(),
 		AteomPodName:           a.GetAteomPodName(),
 		AteomPodIP:             a.GetAteomPodIp(),
-		LastSnapshot:           a.GetLastSnapshot(),
+		LatestSnapshot:         snapshotInfoString(a.GetLatestSnapshotInfo()),
+		WorkerPoolName:         a.GetWorkerPoolName(),
 		InProgressSnapshot:     a.GetInProgressSnapshot(),
 		Version:                a.GetVersion(),
 	}
 }
 
+// snapshotInfoString renders a SnapshotInfo as a single location string.
+func snapshotInfoString(s *ateapipb.SnapshotInfo) string {
+	if s == nil {
+		return ""
+	}
+	if ext := s.GetExternal(); ext != nil {
+		return ext.GetSnapshotUriPrefix()
+	}
+	if loc := s.GetLocal(); loc != nil {
+		return loc.GetSnapshotPrefix()
+	}
+	return ""
+}
+
+// labelSelectorString renders a metav1.LabelSelector as a compact human-readable
+// string (e.g. "kagent.dev/worker-pool=kagent-default") for UI display.
+func labelSelectorString(ctx context.Context, sel *metav1.LabelSelector) string {
+	if sel == nil {
+		return ""
+	}
+	s, err := metav1.LabelSelectorAsSelector(sel)
+	if err != nil {
+		ctrllog.FromContext(ctx).Info("invalid ActorTemplate workerSelector", "error", err)
+		return "<invalid selector>"
+	}
+	return s.String()
+}
+
 func workerEntryFromPB(w *ateapipb.Worker) api.SubstrateWorkerEntry {
 	return api.SubstrateWorkerEntry{
 		WorkerNamespace: w.GetWorkerNamespace(),