diff --git a/.gitignore b/.gitignore index 3cbd627..7f46e29 100644 --- a/.gitignore +++ b/.gitignore @@ -50,5 +50,9 @@ cover.out # Local config config.toml !config.example.toml +.env +deploy/litellm/.env +deploy/litellm/secret.env +deploy/docker-desktop/secret.env cmd/kubectl-gt/kubectl-gt cmd/kubectl-gt/bin/kubectl-gt-* diff --git a/Makefile b/Makefile index 641b824..2b0aa73 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,10 @@ demo: setup-test-e2e docker-build-e2e ## Deploy to Kind and show a sample Poleca @echo "Cleanup:" @echo " make cleanup-test-e2e" +.PHONY: demo-docker-desktop +demo-docker-desktop: ## Deploy full stack (operator + LiteLLM/MadEye + smoke Polecat) to Docker Desktop K8s. + ./deploy/docker-desktop/apply-full-stack.sh + .PHONY: smoke-test smoke-test: ## Smoke test published release artifacts in Kind (requires VERSION). ./scripts/smoke-test.sh --version $(VERSION) diff --git a/api/v1alpha1/polecat_webhook.go b/api/v1alpha1/polecat_webhook.go index a26018d..89ddaa4 100644 --- a/api/v1alpha1/polecat_webhook.go +++ b/api/v1alpha1/polecat_webhook.go @@ -94,7 +94,7 @@ func (v *PolecatCustomValidator) validatePolecat(polecat *Polecat) (admission.Wa if polecat.Spec.Kubernetes == nil { allErrs = append(allErrs, "spec.kubernetes: is required when executionMode is 'kubernetes'") } else { - errs := validateKubernetesSpec(polecat.Spec.Kubernetes) + errs := validateKubernetesSpec(polecat.Spec.Kubernetes, polecat.Spec.AgentConfig) allErrs = append(allErrs, errs...) } } @@ -163,7 +163,7 @@ func containsPathTraversal(s string) bool { } // validateKubernetesSpec validates the kubernetes execution spec. -func validateKubernetesSpec(k *KubernetesSpec) []string { +func validateKubernetesSpec(k *KubernetesSpec, agentConfig *AgentConfig) []string { var errs []string // GitRepository is required (validated by CRD, but double-check) @@ -186,11 +186,15 @@ func validateKubernetesSpec(k *KubernetesSpec) []string { errs = append(errs, "spec.kubernetes.gitSecretRef.name: is required") } - // Either ClaudeCredsSecretRef or ApiKeySecretRef is required for authentication + // Auth: OAuth creds, direct API key, or LiteLLM/gateway key via agentConfig hasOAuth := k.ClaudeCredsSecretRef != nil && k.ClaudeCredsSecretRef.Name != "" hasAPIKey := k.ApiKeySecretRef != nil && k.ApiKeySecretRef.Name != "" - if !hasOAuth && !hasAPIKey { - errs = append(errs, "spec.kubernetes: either claudeCredsSecretRef or apiKeySecretRef is required") + hasGateway := agentConfig != nil && + agentConfig.ModelProvider != nil && + agentConfig.ModelProvider.APIKeySecretRef != nil && + agentConfig.ModelProvider.APIKeySecretRef.Name != "" + if !hasOAuth && !hasAPIKey && !hasGateway { + errs = append(errs, "spec.kubernetes: either claudeCredsSecretRef, apiKeySecretRef, or agentConfig.modelProvider.apiKeySecretRef is required") } // Validate ActiveDeadlineSeconds @@ -210,11 +214,13 @@ func validateKubernetesSpec(k *KubernetesSpec) []string { // validateImageRegistry checks that the image is from an allowed registry. // This prevents attackers from running arbitrary container images in the cluster. +// Local Docker Desktop builds (polecat-agent:local) are allowed in addition to published registries. var allowedRegistries = []string{ "ghcr.io/boshu2/", // Official Gas Town images "docker.io/boshu2/", // Docker Hub official images "registry.access.redhat.com/", // Red Hat UBI images for FIPS "quay.io/boshu2/", // Quay official images + "polecat-agent:", // Local Docker Desktop image (name:tag, no registry) } func validateImageRegistry(image string) error { diff --git a/api/v1alpha1/polecat_webhook_test.go b/api/v1alpha1/polecat_webhook_test.go index 5548988..f820187 100644 --- a/api/v1alpha1/polecat_webhook_test.go +++ b/api/v1alpha1/polecat_webhook_test.go @@ -136,7 +136,7 @@ func TestPolecatCustomValidator_ValidateCreate(t *testing.T) { }, }, wantErr: true, - errContains: "spec.kubernetes: either claudeCredsSecretRef or apiKeySecretRef is required", + errContains: "spec.kubernetes: either claudeCredsSecretRef, apiKeySecretRef, or agentConfig.modelProvider.apiKeySecretRef is required", }, { name: "high resource usage warning", @@ -537,7 +537,7 @@ func TestValidateKubernetesSpec(t *testing.T) { errContains: []string{ "spec.kubernetes.gitRepository: is required", "spec.kubernetes.gitSecretRef.name: is required", - "spec.kubernetes: either claudeCredsSecretRef or apiKeySecretRef is required", + "spec.kubernetes: either claudeCredsSecretRef, apiKeySecretRef, or agentConfig.modelProvider.apiKeySecretRef is required", }, }, { @@ -586,7 +586,7 @@ func TestValidateKubernetesSpec(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - errs := validateKubernetesSpec(tt.spec) + errs := validateKubernetesSpec(tt.spec, nil) assert.Len(t, errs, tt.wantErrs) for _, expected := range tt.errContains { found := false diff --git a/cmd/promo-api/Dockerfile b/cmd/promo-api/Dockerfile new file mode 100644 index 0000000..7b815d2 --- /dev/null +++ b/cmd/promo-api/Dockerfile @@ -0,0 +1,14 @@ +# Promo API - thin HTTP front door for Gastown promo script jobs +FROM golang:1.25-alpine AS builder +WORKDIR /src +RUN apk add --no-cache git ca-certificates +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/promo-api ./cmd/promo-api + +FROM gcr.io/distroless/static:nonroot +COPY --from=builder /out/promo-api /promo-api +USER 65532:65532 +EXPOSE 8080 +ENTRYPOINT ["/promo-api"] diff --git a/cmd/promo-api/kubeconfig.go b/cmd/promo-api/kubeconfig.go new file mode 100644 index 0000000..7b15144 --- /dev/null +++ b/cmd/promo-api/kubeconfig.go @@ -0,0 +1,19 @@ +package main + +import ( + "os" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func restConfigFromKubeconfig() (*rest.Config, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + if kc := os.Getenv("KUBECONFIG"); kc != "" { + loadingRules.ExplicitPath = kc + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + loadingRules, + &clientcmd.ConfigOverrides{}, + ).ClientConfig() +} diff --git a/cmd/promo-api/main.go b/cmd/promo-api/main.go new file mode 100644 index 0000000..2ed741f --- /dev/null +++ b/cmd/promo-api/main.go @@ -0,0 +1,422 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + gastownv1alpha1 "github.com/org/gastown-operator/api/v1alpha1" +) + +func main() { + cfg := loadConfig() + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(gastownv1alpha1.AddToScheme(scheme)) + + restCfg, err := rest.InClusterConfig() + if err != nil { + // Local/dev fallback (Docker Desktop kubectl proxy / kubeconfig) + restCfg, err = restConfigFromKubeconfig() + if err != nil { + log.Fatalf("kubernetes config: %v", err) + } + } + + k8s, err := client.New(restCfg, client.Options{Scheme: scheme}) + if err != nil { + log.Fatalf("kubernetes client: %v", err) + } + + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.Port), + Handler: newRouter(cfg, k8s), + ReadHeaderTimeout: 10 * time.Second, + } + + go func() { + log.Printf("promo-api listening on :%d (namespace=%s rig=%s)", cfg.Port, cfg.Namespace, cfg.RigName) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("listen: %v", err) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + <-stop + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = srv.Shutdown(ctx) +} + +type config struct { + Port int + Namespace string + RigName string + GitRepo string + GitBranch string + GitSecret string + LiteLLMURL string + LiteLLMAuthSec string + AgentImage string +} + +func loadConfig() config { + return config{ + Port: envInt("PORT", 8080), + Namespace: envOr("NAMESPACE", "gastown-system"), + RigName: envOr("RIG_NAME", "local-smoke"), + GitRepo: envOr("GIT_REPO", "git@github.com:priyanshur01/gastown-static.git"), + GitBranch: envOr("GIT_BRANCH", "main"), + GitSecret: envOr("GIT_SECRET", "git-creds"), + LiteLLMURL: envOr("LITELLM_URL", "http://litellm.gastown-system.svc:4000"), + LiteLLMAuthSec: envOr("LITELLM_AUTH_SECRET", "litellm-auth"), + AgentImage: envOr("AGENT_IMAGE", "polecat-agent:local"), + } +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func envInt(k string, def int) int { + v := os.Getenv(k) + if v == "" { + return def + } + var n int + if _, err := fmt.Sscanf(v, "%d", &n); err != nil { + return def + } + return n +} + +func newRouter(cfg config, k8s client.Client) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health", handleHealth(cfg, k8s)) + mux.HandleFunc("GET /healthz", handleHealth(cfg, k8s)) + mux.HandleFunc("POST /v1/promo/generate", handleGenerate(cfg, k8s)) + mux.HandleFunc("GET /v1/promo/jobs/{name}", handleJobStatus(cfg, k8s)) + return mux +} + +type healthResponse struct { + Status string `json:"status"` + Checks map[string]string `json:"checks"` + Ready bool `json:"ready"` + TimeUTC string `json:"time"` +} + +func handleHealth(cfg config, k8s client.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + checks := map[string]string{} + ready := true + + // Kubernetes API + Rig exists + var rig gastownv1alpha1.Rig + if err := k8s.Get(ctx, types.NamespacedName{Name: cfg.RigName}, &rig); err != nil { + checks["rig"] = "error: " + err.Error() + ready = false + } else { + checks["rig"] = string(rig.Status.Phase) + if rig.Status.Phase != gastownv1alpha1.RigPhaseReady && rig.Status.Phase != "" { + // Still acceptable if Ready condition is true + for _, c := range rig.Status.Conditions { + if c.Type == "Ready" && c.Status == metav1.ConditionTrue { + checks["rig"] = "Ready" + break + } + } + } + } + + // Required secrets + for _, name := range []string{cfg.GitSecret, cfg.LiteLLMAuthSec} { + var sec corev1.Secret + if err := k8s.Get(ctx, types.NamespacedName{Namespace: cfg.Namespace, Name: name}, &sec); err != nil { + checks["secret:"+name] = "missing" + ready = false + } else { + checks["secret:"+name] = "ok" + } + } + + // LiteLLM liveliness + llURL := strings.TrimRight(cfg.LiteLLMURL, "/") + "/health/liveliness" + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, llURL, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + checks["litellm"] = "unreachable: " + err.Error() + ready = false + } else { + _ = resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + checks["litellm"] = "ok" + } else { + checks["litellm"] = fmt.Sprintf("http_%d", resp.StatusCode) + ready = false + } + } + + status := "ok" + code := http.StatusOK + if !ready { + status = "degraded" + code = http.StatusServiceUnavailable + } + writeJSON(w, code, healthResponse{ + Status: status, + Checks: checks, + Ready: ready, + TimeUTC: time.Now().UTC().Format(time.RFC3339), + }) + } +} + +type generateRequest struct { + Prompt string `json:"prompt"` + Name string `json:"name,omitempty"` + Branch string `json:"branch,omitempty"` +} + +type generateResponse struct { + JobName string `json:"job_name"` + Namespace string `json:"namespace"` + Rig string `json:"rig"` + StatusURL string `json:"status_url"` + Message string `json:"message"` +} + +func handleGenerate(cfg config, k8s client.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req generateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeErr(w, http.StatusBadRequest, "invalid JSON body") + return + } + prompt := strings.TrimSpace(req.Prompt) + if prompt == "" { + writeErr(w, http.StatusBadRequest, "prompt is required") + return + } + if len(prompt) > 4000 { + writeErr(w, http.StatusBadRequest, "prompt too long (max 4000 chars)") + return + } + + name := sanitizeName(req.Name) + if name == "" { + name = "promo-" + shortID() + } + + branch := req.Branch + if branch == "" { + branch = cfg.GitBranch + } + workBranch := fmt.Sprintf("feature/%s", name) + deadline := int64(3600) + + task := fmt.Sprintf(`You are generating promo scripts in this repository. + +USER REQUEST: +%s + +INSTRUCTIONS: +1. Explore the repo structure and existing promo/script conventions. +2. Generate or update the promo script(s) needed for the request. +3. Keep changes focused; do not refactor unrelated code. +4. After finishing: + - git add relevant files + - git commit -m 'feat(%s): promo script generation' + - git push origin HEAD + - create a PR if gh is available (gh pr create --fill), otherwise leave the branch pushed +`, prompt, name) + + polecat := &gastownv1alpha1.Polecat{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: cfg.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "promo-api", + "gastown.io/flow": "promo-script", + "gastown.io/rig": cfg.RigName, + }, + }, + Spec: gastownv1alpha1.PolecatSpec{ + Rig: cfg.RigName, + DesiredState: gastownv1alpha1.PolecatDesiredWorking, + BeadID: "ls-" + name, + TaskDescription: task, + ExecutionMode: gastownv1alpha1.ExecutionModeKubernetes, + Agent: gastownv1alpha1.AgentTypeClaudeCode, + AgentConfig: &gastownv1alpha1.AgentConfig{ + Provider: gastownv1alpha1.LLMProviderLiteLLM, + Model: "claude-opus-4-8", + ModelProvider: &gastownv1alpha1.ModelProviderConfig{ + Endpoint: cfg.LiteLLMURL, + APIKeySecretRef: &gastownv1alpha1.SecretKeyRef{ + Name: cfg.LiteLLMAuthSec, + Key: "master-key", + }, + }, + // Force every Claude Code model tier onto MadEye opus-only access. + Env: []corev1.EnvVar{ + {Name: "ANTHROPIC_MODEL", Value: "claude-opus-4-8"}, + {Name: "ANTHROPIC_DEFAULT_OPUS_MODEL", Value: "claude-opus-4-8"}, + {Name: "ANTHROPIC_DEFAULT_SONNET_MODEL", Value: "claude-opus-4-8"}, + {Name: "ANTHROPIC_DEFAULT_HAIKU_MODEL", Value: "claude-opus-4-8"}, + }, + }, + Kubernetes: &gastownv1alpha1.KubernetesSpec{ + GitRepository: cfg.GitRepo, + GitBranch: branch, + WorkBranch: workBranch, + GitSecretRef: gastownv1alpha1.SecretReference{Name: cfg.GitSecret}, + ApiKeySecretRef: &gastownv1alpha1.SecretKeyRef{ + Name: cfg.LiteLLMAuthSec, + Key: "master-key", + }, + Image: cfg.AgentImage, + ActiveDeadlineSeconds: &deadline, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + + if err := k8s.Create(ctx, polecat); err != nil { + if apierrors.IsAlreadyExists(err) { + writeErr(w, http.StatusConflict, "job name already exists: "+name) + return + } + writeErr(w, http.StatusInternalServerError, "failed to create polecat: "+err.Error()) + return + } + + writeJSON(w, http.StatusAccepted, generateResponse{ + JobName: name, + Namespace: cfg.Namespace, + Rig: cfg.RigName, + StatusURL: fmt.Sprintf("/v1/promo/jobs/%s", name), + Message: "promo script generation started", + }) + } +} + +type jobStatusResponse struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Phase string `json:"phase"` + PodName string `json:"pod_name,omitempty"` + PodActive bool `json:"pod_active"` + BeadID string `json:"bead_id,omitempty"` + Message string `json:"message,omitempty"` +} + +func handleJobStatus(cfg config, k8s client.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if name == "" { + writeErr(w, http.StatusBadRequest, "job name required") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + var polecat gastownv1alpha1.Polecat + if err := k8s.Get(ctx, types.NamespacedName{Namespace: cfg.Namespace, Name: name}, &polecat); err != nil { + if apierrors.IsNotFound(err) { + writeErr(w, http.StatusNotFound, "job not found") + return + } + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + + msg := "" + for _, c := range polecat.Status.Conditions { + if c.Status == metav1.ConditionTrue { + msg = c.Message + break + } + } + + writeJSON(w, http.StatusOK, jobStatusResponse{ + Name: polecat.Name, + Namespace: polecat.Namespace, + Phase: string(polecat.Status.Phase), + PodName: polecat.Status.PodName, + PodActive: polecat.Status.PodActive, + BeadID: polecat.Status.AssignedBead, + Message: msg, + }) + } +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} + +func writeErr(w http.ResponseWriter, code int, msg string) { + writeJSON(w, code, map[string]string{"error": msg}) +} + +func sanitizeName(in string) string { + in = strings.ToLower(strings.TrimSpace(in)) + var b strings.Builder + for _, r := range in { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + b.WriteRune(r) + } + } + out := strings.Trim(b.String(), "-") + if len(out) > 40 { + out = out[:40] + } + return out +} + +func shortID() string { + var b [4]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} diff --git a/config/samples/litellm/polecat-madeye.yaml b/config/samples/litellm/polecat-madeye.yaml new file mode 100644 index 0000000..871aa89 --- /dev/null +++ b/config/samples/litellm/polecat-madeye.yaml @@ -0,0 +1,52 @@ +# Example Polecat routed through in-cluster LiteLLM → MadEye +apiVersion: gastown.gastown.io/v1alpha1 +kind: Polecat +metadata: + name: madeye-smoke + namespace: gastown-system +spec: + rig: smoke + desiredState: Working + beadID: smoke-001 + taskDescription: | + Reply with a short hello and confirm which model you are using. + Do not modify any files. Exit after the reply. + executionMode: kubernetes + agent: claude-code + agentConfig: + provider: litellm + model: claude-opus-4-8 + modelProvider: + # Claude Code Anthropic-compatible endpoint (LiteLLM Service) + endpoint: http://litellm.gastown-system.svc:4000 + apiKeySecretRef: + name: litellm-auth + key: master-key + env: + - name: CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY + value: "1" + kubernetes: + gitRepository: git@github.com:example/smoke-repo.git + gitBranch: main + gitSecretRef: + name: git-creds + # No direct Anthropic key — gateway auth comes from agentConfig + activeDeadlineSeconds: 600 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi +--- +# Secret referenced by agentConfig.modelProvider.apiKeySecretRef +# Value must match LITELLM_MASTER_KEY in litellm-secrets +apiVersion: v1 +kind: Secret +metadata: + name: litellm-auth + namespace: gastown-system +type: Opaque +stringData: + master-key: sk-local-litellm diff --git a/deploy/docker-desktop/PROMO_API.md b/deploy/docker-desktop/PROMO_API.md new file mode 100644 index 0000000..0535e9f --- /dev/null +++ b/deploy/docker-desktop/PROMO_API.md @@ -0,0 +1,53 @@ +# Promo API + +Small HTTP front door for Gastown promo-script jobs on Docker Desktop. + +## Endpoints + +| Method | Path | Purpose | +|--------|------|---------| +| `GET` | `/health` or `/healthz` | Stack health (Rig + secrets + LiteLLM) | +| `POST` | `/v1/promo/generate` | Start promo script generation (creates a Polecat) | +| `GET` | `/v1/promo/jobs/{name}` | Job / Polecat status | + +## Deploy + +```bash +cd deploy/docker-desktop +./deploy-promo-api.sh +``` + +Service is exposed on **NodePort 30080**. + +## Examples + +```bash +# Health +curl -s http://localhost:30080/health | jq + +# Invoke promo script generation against gastown-static Rig +curl -s -X POST http://localhost:30080/v1/promo/generate \ + -H 'Content-Type: application/json' \ + -d '{ + "prompt": "Create a 30-second promo script for a thriller podcast launch", + "name": "thriller-promo-1" + }' | jq + +# Poll status +curl -s http://localhost:30080/v1/promo/jobs/thriller-promo-1 | jq + +# Watch agent logs +kubectl -n gastown-system logs -f -l gastown.io/polecat=thriller-promo-1 -c claude +``` + +## Request body (`POST /v1/promo/generate`) + +```json +{ + "prompt": "required user request for script generation", + "name": "optional-job-name", + "branch": "optional base branch (default main)" +} +``` + +Async: returns `202` with `job_name` + `status_url`. The operator runs the Polecat against Rig `local-smoke` (`gastown-static`) via LiteLLM → MadEye. diff --git a/deploy/docker-desktop/README.md b/deploy/docker-desktop/README.md new file mode 100644 index 0000000..767ccd4 --- /dev/null +++ b/deploy/docker-desktop/README.md @@ -0,0 +1,70 @@ +# Full Gas Town on Docker Desktop Kubernetes + +Deploys: +1. Operator (built from this repo — includes LiteLLM/`agentConfig` wiring) +2. LiteLLM → MadEye adapter +3. Sample Rig + smoke Polecat + +## Prerequisites + +- Docker Desktop running with **Kubernetes enabled** +- `kubectl config use-context docker-desktop` works +- MadEye reachable (VPN) + Bearer token +- SSH private key with access to a git repo + +## Steps + +```bash +kubectl config use-context docker-desktop +kubectl get nodes + +cd deploy/docker-desktop +cp secret.env.example secret.env +``` + +Edit `secret.env`: + +| Variable | Purpose | +|----------|---------| +| `MADEYE_API_KEY` | MadEye Bearer token | +| `MADEYE_USER_EMAIL` | Your PocketFM email | +| `GIT_SSH_PRIVATE_KEY_PATH` | Path to SSH key | +| `GIT_REPO` | e.g. `git@github.com:org/repo.git` | + +Then: + +```bash +./apply-full-stack.sh +``` + +First run builds the operator image (needs network; several minutes). + +## Verify + +```bash +kubectl -n gastown-operator-system get pods +kubectl -n gastown-system get pods +kubectl get rig +kubectl -n gastown-system get polecat + +# LiteLLM +curl -s http://localhost:30040/v1/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-local-litellm" \ + -H "anthropic-version: 2023-06-01" \ + -d '{"model":"claude-opus-4-8","max_tokens":32,"messages":[{"role":"user","content":"ping"}]}' + +# Polecat agent logs +kubectl -n gastown-system logs -f -l gastown.io/polecat=madeye-smoke -c claude +``` + +## Cleanup + +```bash +kubectl -n gastown-system delete polecat --all --ignore-not-found +kubectl delete rig --all --ignore-not-found +kubectl -n gastown-system delete deploy,svc,cm,secret -l app=litellm --ignore-not-found +kubectl delete -f ../litellm/k8s-docker-desktop.yaml --ignore-not-found +make undeploy +make uninstall +``` diff --git a/deploy/docker-desktop/apply-full-stack.sh b/deploy/docker-desktop/apply-full-stack.sh new file mode 100755 index 0000000..253df19 --- /dev/null +++ b/deploy/docker-desktop/apply-full-stack.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Deploy full Gas Town stack on Docker Desktop Kubernetes: +# operator (local build with MadEye agentConfig wiring) +# + LiteLLM → MadEye +# + Rig + smoke Polecat +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +LITELLM_DIR="${ROOT}/deploy/litellm" +SECRET_ENV="$(cd "$(dirname "$0")" && pwd)/secret.env" +IMG="${IMG:-gastown-operator:dev}" +CONTEXT="${KUBE_CONTEXT:-docker-desktop}" +NS_WORKLOAD="gastown-system" + +cd "${ROOT}" + +echo "==> Checking Docker Desktop Kubernetes (${CONTEXT})" +kubectl config use-context "${CONTEXT}" >/dev/null +kubectl get nodes >/dev/null + +if [[ ! -f "${SECRET_ENV}" ]]; then + cp "$(dirname "$0")/secret.env.example" "${SECRET_ENV}" + echo "Created ${SECRET_ENV}" + echo "Edit MADEYE_API_KEY, GIT_SSH_PRIVATE_KEY_PATH, and GIT_REPO, then re-run." + exit 1 +fi + +# shellcheck disable=SC1090 +source "${SECRET_ENV}" + +: "${MADEYE_API_KEY:?set MADEYE_API_KEY in secret.env}" +: "${MADEYE_USER_EMAIL:?set MADEYE_USER_EMAIL in secret.env}" +: "${GIT_SSH_PRIVATE_KEY_PATH:?set GIT_SSH_PRIVATE_KEY_PATH in secret.env}" +: "${GIT_REPO:?set GIT_REPO in secret.env}" +LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-local-litellm}" +GIT_BRANCH="${GIT_BRANCH:-main}" +RIG_NAME="${RIG_NAME:-local-smoke}" +POLECAT_NAME="${POLECAT_NAME:-madeye-smoke}" + +if [[ ! -f "${GIT_SSH_PRIVATE_KEY_PATH}" ]]; then + echo "ERROR: SSH key not found: ${GIT_SSH_PRIVATE_KEY_PATH}" + exit 1 +fi + +echo "==> Building operator image ${IMG} (includes agentConfig → LiteLLM wiring)" +# docker-build-e2e tags the exact IMG reference (no VERSION suffix) +make docker-build-e2e IMG="${IMG}" + +echo "==> Installing CRDs" +make install + +echo "==> Deploying operator" +make deploy IMG="${IMG}" + +OP_NS="gastown-operator-system" +echo "==> Forcing imagePullPolicy=Never for local Docker Desktop image" +kubectl -n "${OP_NS}" set image deploy/gastown-operator-controller-manager manager="${IMG}" +kubectl -n "${OP_NS}" patch deploy gastown-operator-controller-manager --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"Never"}]' \ + >/dev/null || true +kubectl -n "${OP_NS}" rollout status deploy/gastown-operator-controller-manager --timeout=180s + +echo "==> Ensuring workload namespace ${NS_WORKLOAD}" +kubectl get ns "${NS_WORKLOAD}" >/dev/null 2>&1 || kubectl create ns "${NS_WORKLOAD}" + +echo "==> Deploying LiteLLM (MadEye adapter)" +# Reuse litellm docker-desktop manifests + secrets from secret.env +kubectl -n "${NS_WORKLOAD}" create secret generic litellm-secrets \ + --from-literal=MADEYE_API_KEY="${MADEYE_API_KEY}" \ + --from-literal=MADEYE_USER_EMAIL="${MADEYE_USER_EMAIL}" \ + --from-literal=LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - + +kubectl -n "${NS_WORKLOAD}" create configmap litellm-callbacks \ + --from-file=strip_prefill.py="${LITELLM_DIR}/strip_prefill.py" \ + --from-file=prefill_proxy.py="${LITELLM_DIR}/prefill_proxy.py" \ + --from-file=k8s-entrypoint.sh="${LITELLM_DIR}/k8s-entrypoint.sh" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Render user email into litellm config via apply script assets +kubectl apply -f "${LITELLM_DIR}/k8s-docker-desktop.yaml" +# Patch ConfigMap email if needed — k8s-docker-desktop uses ${MADEYE_USER_EMAIL} placeholder in template; +# deployment entrypoint sed-substitutes from env. Good. + +kubectl -n "${NS_WORKLOAD}" rollout status deploy/litellm --timeout=180s + +echo "==> Creating git + LiteLLM auth secrets for Polecats" +kubectl -n "${NS_WORKLOAD}" create secret generic git-creds \ + --from-file=ssh-privatekey="${GIT_SSH_PRIVATE_KEY_PATH}" \ + --dry-run=client -o yaml | kubectl apply -f - + +kubectl -n "${NS_WORKLOAD}" create secret generic litellm-auth \ + --from-literal=master-key="${LITELLM_MASTER_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - + +echo "==> Applying Rig + smoke Polecat" +TMP_SAMPLES="$(mktemp)" +sed \ + -e "s|GIT_REPO_PLACEHOLDER|${GIT_REPO}|g" \ + -e "s|GIT_BRANCH_PLACEHOLDER|${GIT_BRANCH}|g" \ + -e "s|name: local-smoke|name: ${RIG_NAME}|g" \ + -e "s|rig: local-smoke|rig: ${RIG_NAME}|g" \ + -e "s|name: madeye-smoke|name: ${POLECAT_NAME}|g" \ + "$(dirname "$0")/samples.yaml.tpl" > "${TMP_SAMPLES}" +kubectl apply -f "${TMP_SAMPLES}" +rm -f "${TMP_SAMPLES}" + +echo +echo "============================================" +echo " Gas Town full stack is up (Docker Desktop)" +echo "============================================" +echo +echo "Operator: kubectl -n ${OP_NS} get pods" +echo "LiteLLM: kubectl -n ${NS_WORKLOAD} get pods" +echo " curl http://localhost:30040/v1/messages ..." +echo "Rig: kubectl get rig ${RIG_NAME}" +echo "Polecat: kubectl -n ${NS_WORKLOAD} get polecat ${POLECAT_NAME} -w" +echo "Logs: kubectl -n ${NS_WORKLOAD} logs -f -l gastown.io/polecat=${POLECAT_NAME} -c claude" +echo +echo "LiteLLM smoke:" +echo " curl -s http://localhost:30040/v1/messages \\" +echo " -H 'Content-Type: application/json' \\" +echo " -H 'Authorization: Bearer ${LITELLM_MASTER_KEY}' \\" +echo " -H 'anthropic-version: 2023-06-01' \\" +echo " -d '{\"model\":\"claude-opus-4-8\",\"max_tokens\":64,\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}]}'" diff --git a/deploy/docker-desktop/deploy-promo-api.sh b/deploy/docker-desktop/deploy-promo-api.sh new file mode 100755 index 0000000..2ce59e6 --- /dev/null +++ b/deploy/docker-desktop/deploy-promo-api.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Build and deploy promo-api to Docker Desktop Kubernetes. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +kubectl config use-context docker-desktop >/dev/null + +echo "==> Building promo-api:local (load into Docker Desktop image store)" +docker buildx build --load -t promo-api:local -f cmd/promo-api/Dockerfile . + +echo "==> Applying manifests" +kubectl apply -f deploy/docker-desktop/promo-api.yaml +kubectl -n gastown-system delete pod -l app=promo-api --ignore-not-found +kubectl -n gastown-system rollout status deploy/promo-api --timeout=120s + +echo +echo "promo-api ready" +echo " Health: curl http://localhost:30080/health" +echo " Generate: curl -X POST http://localhost:30080/v1/promo/generate -H 'Content-Type: application/json' -d '{\"prompt\":\"Write a short promo script for a comedy podcast\"}'" +echo " Status: curl http://localhost:30080/v1/promo/jobs/" diff --git a/deploy/docker-desktop/promo-api.yaml b/deploy/docker-desktop/promo-api.yaml new file mode 100644 index 0000000..1aa626a --- /dev/null +++ b/deploy/docker-desktop/promo-api.yaml @@ -0,0 +1,133 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promo-api + namespace: gastown-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: promo-api + namespace: gastown-system +rules: + - apiGroups: ["gastown.gastown.io"] + resources: ["polecats"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: promo-api + namespace: gastown-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: promo-api +subjects: + - kind: ServiceAccount + name: promo-api + namespace: gastown-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promo-api-rig-reader +rules: + - apiGroups: ["gastown.gastown.io"] + resources: ["rigs"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promo-api-rig-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: promo-api-rig-reader +subjects: + - kind: ServiceAccount + name: promo-api + namespace: gastown-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: promo-api + namespace: gastown-system + labels: + app: promo-api +spec: + replicas: 1 + selector: + matchLabels: + app: promo-api + template: + metadata: + labels: + app: promo-api + spec: + serviceAccountName: promo-api + containers: + - name: promo-api + image: promo-api:local + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http + env: + - name: PORT + value: "8080" + - name: NAMESPACE + value: gastown-system + - name: RIG_NAME + value: local-smoke + - name: GIT_REPO + value: git@github.com:priyanshur01/gastown-static.git + - name: GIT_BRANCH + value: main + - name: GIT_SECRET + value: git-creds + - name: LITELLM_URL + value: http://litellm.gastown-system.svc:4000 + - name: LITELLM_AUTH_SECRET + value: litellm-auth + - name: AGENT_IMAGE + value: polecat-agent:local + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: promo-api + namespace: gastown-system +spec: + type: NodePort + selector: + app: promo-api + ports: + - name: http + port: 8080 + targetPort: http + nodePort: 30080 diff --git a/deploy/docker-desktop/samples.yaml.tpl b/deploy/docker-desktop/samples.yaml.tpl new file mode 100644 index 0000000..776d6ef --- /dev/null +++ b/deploy/docker-desktop/samples.yaml.tpl @@ -0,0 +1,54 @@ +apiVersion: gastown.gastown.io/v1alpha1 +kind: Rig +metadata: + name: local-smoke +spec: + gitURL: "GIT_REPO_PLACEHOLDER" + beadsPrefix: "ls" + settings: + namepoolTheme: "mad-max" + maxPolecats: 2 +--- +apiVersion: gastown.gastown.io/v1alpha1 +kind: Polecat +metadata: + name: madeye-smoke + namespace: gastown-system +spec: + rig: local-smoke + desiredState: Working + beadID: ls-001 + taskDescription: | + Smoke test only: reply confirming you can see this task and which model + you appear to be using. Do not modify repository files. Finish quickly. + executionMode: kubernetes + agent: claude-code + agentConfig: + provider: litellm + model: claude-opus-4-8 + modelProvider: + endpoint: http://litellm.gastown-system.svc:4000 + apiKeySecretRef: + name: litellm-auth + key: master-key + env: + - name: CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY + value: "1" + kubernetes: + gitRepository: "GIT_REPO_PLACEHOLDER" + gitBranch: "GIT_BRANCH_PLACEHOLDER" + workBranch: feature/ls-001-smoke + gitSecretRef: + name: git-creds + # Satisfies auth validation; gateway Bearer token comes from agentConfig above + apiKeySecretRef: + name: litellm-auth + key: master-key + activeDeadlineSeconds: 900 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi diff --git a/deploy/docker-desktop/secret.env.example b/deploy/docker-desktop/secret.env.example new file mode 100644 index 0000000..df7a0b0 --- /dev/null +++ b/deploy/docker-desktop/secret.env.example @@ -0,0 +1,16 @@ +# Full-stack local secrets for Docker Desktop Kubernetes. +# Copy to secret.env and edit (secret.env is gitignored). +# +# Required for LiteLLM → MadEye: +MADEYE_API_KEY=replace-with-madeye-bearer-token +MADEYE_USER_EMAIL=priyanshu.rajput@pocketfm.com +LITELLM_MASTER_KEY=sk-local-litellm +# +# Required for Polecat git clone (SSH): +GIT_SSH_PRIVATE_KEY_PATH=/Users/priyansh.rajput/.ssh/id_ed25519 +GIT_REPO=git@github.com:YOUR_ORG/YOUR_REPO.git +GIT_BRANCH=main +# +# Rig / Polecat names +RIG_NAME=local-smoke +POLECAT_NAME=madeye-smoke diff --git a/deploy/litellm/.env.example b/deploy/litellm/.env.example new file mode 100644 index 0000000..4b4dd68 --- /dev/null +++ b/deploy/litellm/.env.example @@ -0,0 +1,5 @@ +# Copy to .env and fill in values for docker compose +MADEYE_API_KEY=replace-with-madeye-bearer-token +MADEYE_USER_EMAIL=priyanshu.rajput@pocketfm.com +MADEYE_API_BASE=https://madeye.internal.pocketfm.org/v1 +LITELLM_MASTER_KEY=sk-local-litellm diff --git a/deploy/litellm/README.md b/deploy/litellm/README.md new file mode 100644 index 0000000..d291d65 --- /dev/null +++ b/deploy/litellm/README.md @@ -0,0 +1,54 @@ +# LiteLLM → MadEye + +Local and in-cluster adapter so Claude Code (Anthropic API) can call MadEye +(OpenAI `/v1/chat/completions` only). + +Full guide: [docs/LITE_LLM.md](../../docs/LITE_LLM.md) + +## Docker Desktop Kubernetes (recommended for local taste) + +1. **Docker Desktop → Settings → Kubernetes → Enable Kubernetes → Apply & Restart** +2. Confirm: + +```bash +kubectl config use-context docker-desktop +kubectl get nodes +``` + +3. Deploy: + +```bash +cp secret.env.example secret.env # edit MADEYE_API_KEY +./apply-docker-desktop.sh +``` + +4. Smoke test: + +```bash +curl -s http://localhost:30040/v1/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-local-litellm" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role":"user","content":"Hello! which model are you using?"}] + }' +``` + +## Docker Compose (no Kubernetes) + +```bash +cp .env.example .env # edit MADEYE_API_KEY + MADEYE_USER_EMAIL +docker compose --env-file .env up -d + +curl -s http://localhost:4000/v1/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-local-litellm" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role":"user","content":"Hello! which model are you using?"}] + }' +``` diff --git a/deploy/litellm/apply-docker-desktop.sh b/deploy/litellm/apply-docker-desktop.sh new file mode 100755 index 0000000..20bfece --- /dev/null +++ b/deploy/litellm/apply-docker-desktop.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Deploy LiteLLM to Docker Desktop Kubernetes. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")" && pwd)" +CONTEXT="${KUBE_CONTEXT:-docker-desktop}" +SECRET_ENV="${ROOT}/secret.env" + +echo "==> Checking kubectl context: ${CONTEXT}" +if ! kubectl config get-contexts "${CONTEXT}" >/dev/null 2>&1; then + cat </dev/null +if ! kubectl get nodes >/dev/null 2>&1; then + echo "ERROR: cannot reach cluster '${CONTEXT}'. Is Docker Desktop Kubernetes running?" + exit 1 +fi + +if [[ ! -f "${SECRET_ENV}" ]]; then + cp "${ROOT}/secret.env.example" "${SECRET_ENV}" + echo "Created ${SECRET_ENV} — edit MADEYE_API_KEY, then re-run this script." + exit 1 +fi + +# shellcheck disable=SC1090 +source "${SECRET_ENV}" + +: "${MADEYE_API_KEY:?set MADEYE_API_KEY in secret.env}" +: "${MADEYE_USER_EMAIL:?set MADEYE_USER_EMAIL in secret.env}" +LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-local-litellm}" + +echo "==> Ensuring namespace gastown-system" +kubectl get ns gastown-system >/dev/null 2>&1 || kubectl create ns gastown-system + +echo "==> Creating/updating Secret litellm-secrets" +kubectl -n gastown-system create secret generic litellm-secrets \ + --from-literal=MADEYE_API_KEY="${MADEYE_API_KEY}" \ + --from-literal=MADEYE_USER_EMAIL="${MADEYE_USER_EMAIL}" \ + --from-literal=LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - + +echo "==> Creating/updating ConfigMap litellm-callbacks (strip MadEye-incompatible prefills)" +kubectl -n gastown-system create configmap litellm-callbacks \ + --from-file=strip_prefill.py="${ROOT}/strip_prefill.py" \ + --from-file=prefill_proxy.py="${ROOT}/prefill_proxy.py" \ + --from-file=k8s-entrypoint.sh="${ROOT}/k8s-entrypoint.sh" \ + --dry-run=client -o yaml | kubectl apply -f - + +echo "==> Applying LiteLLM manifests" +kubectl apply -f "${ROOT}/k8s-docker-desktop.yaml" + +echo "==> Waiting for LiteLLM rollout" +kubectl -n gastown-system rollout status deploy/litellm --timeout=180s + +NODE_PORT="$(kubectl -n gastown-system get svc litellm -o jsonpath='{.spec.ports[0].nodePort}')" +echo +echo "LiteLLM is up on Docker Desktop Kubernetes." +echo " In-cluster: http://litellm.gastown-system.svc:4000" +echo " From host: http://localhost:${NODE_PORT}" +echo +echo "Smoke test:" +echo " curl -s http://localhost:${NODE_PORT}/v1/messages \\" +echo " -H 'Content-Type: application/json' \\" +echo " -H 'Authorization: Bearer ${LITELLM_MASTER_KEY}' \\" +echo " -H 'anthropic-version: 2023-06-01' \\" +echo " -d '{\"model\":\"claude-opus-4-8\",\"max_tokens\":64,\"messages\":[{\"role\":\"user\",\"content\":\"Hello! which model are you using?\"}]}'" diff --git a/deploy/litellm/config.yaml b/deploy/litellm/config.yaml new file mode 100644 index 0000000..0e2eefe --- /dev/null +++ b/deploy/litellm/config.yaml @@ -0,0 +1,37 @@ +# LiteLLM config: Anthropic-compatible front door → MadEye OpenAI backend +# +# Claude Code talks Anthropic /v1/messages to this proxy. +# LiteLLM translates to MadEye OpenAI /v1/chat/completions. +# +# Placeholders replaced by deploy scripts / docker-compose: +# ${MADEYE_API_BASE} default https://madeye.internal.pocketfm.org/v1 +# ${MADEYE_USER_EMAIL} required PocketFM user email for MadEye metadata + +model_list: + - model_name: claude-opus-4-8 + litellm_params: + model: openai/claude-opus-4-8 + api_base: ${MADEYE_API_BASE} + api_key: os.environ/MADEYE_API_KEY + extra_body: + metadata: + user_email: ${MADEYE_USER_EMAIL} + + - model_name: claude-sonnet-4 + litellm_params: + model: openai/claude-opus-4-8 + api_base: ${MADEYE_API_BASE} + api_key: os.environ/MADEYE_API_KEY + extra_body: + metadata: + user_email: ${MADEYE_USER_EMAIL} + +litellm_settings: + drop_params: true + set_verbose: false + use_chat_completions_url_for_anthropic_messages: true + # MadEye rejects assistant prefills; strip them before the upstream call. + callbacks: strip_prefill.proxy_handler_instance + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/deploy/litellm/docker-compose.yaml b/deploy/litellm/docker-compose.yaml new file mode 100644 index 0000000..6a61589 --- /dev/null +++ b/deploy/litellm/docker-compose.yaml @@ -0,0 +1,23 @@ +services: + litellm: + image: ghcr.io/berriai/litellm:main-latest + ports: + - "4000:4000" + environment: + MADEYE_API_KEY: ${MADEYE_API_KEY:?set MADEYE_API_KEY} + MADEYE_USER_EMAIL: ${MADEYE_USER_EMAIL:?set MADEYE_USER_EMAIL} + MADEYE_API_BASE: ${MADEYE_API_BASE:-https://madeye.internal.pocketfm.org/v1} + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-local-litellm} + PYTHONPATH: /app + LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES: "true" + volumes: + - ./config.yaml:/app/config.template.yaml:ro + - ./entrypoint.sh:/app/entrypoint.sh:ro + - ./strip_prefill.py:/app/strip_prefill.py:ro + - ./prefill_proxy.py:/app/prefill_proxy.py:ro + entrypoint: ["/bin/sh", "/app/entrypoint.sh"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4000/health/liveliness"] + interval: 10s + timeout: 5s + retries: 6 diff --git a/deploy/litellm/entrypoint.sh b/deploy/litellm/entrypoint.sh new file mode 100755 index 0000000..e11a088 --- /dev/null +++ b/deploy/litellm/entrypoint.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -e + +MADEYE_API_BASE="${MADEYE_API_BASE:-https://madeye.internal.pocketfm.org/v1}" +MADEYE_USER_EMAIL="${MADEYE_USER_EMAIL:?MADEYE_USER_EMAIL is required}" + +export LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true + +# Substitute only the placeholders LiteLLM cannot resolve via os.environ/ +sed \ + -e "s|\${MADEYE_API_BASE}|${MADEYE_API_BASE}|g" \ + -e "s|\${MADEYE_USER_EMAIL}|${MADEYE_USER_EMAIL}|g" \ + /app/config.template.yaml > /tmp/config.yaml + +echo "Starting LiteLLM → ${MADEYE_API_BASE} (user=${MADEYE_USER_EMAIL})" +echo "Chat completions mode for Anthropic messages: ON" + +# LiteLLM on 4001; edge proxy on 4000 strips Claude Code assistant prefills +litellm --config /tmp/config.yaml --port 4001 --host 127.0.0.1 & +LITELLM_PID=$! + +cleanup() { + kill "${LITELLM_PID}" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +python - <<'PY' +import time, urllib.request +for _ in range(90): + try: + urllib.request.urlopen("http://127.0.0.1:4001/health/liveliness", timeout=2) + break + except Exception: + time.sleep(1) +else: + raise SystemExit("LiteLLM failed to become ready on :4001") +PY + +export LITELLM_UPSTREAM=http://127.0.0.1:4001 +export PREFILL_PROXY_PORT=4000 +exec python /app/prefill_proxy.py diff --git a/deploy/litellm/k8s-docker-desktop.yaml b/deploy/litellm/k8s-docker-desktop.yaml new file mode 100644 index 0000000..1d11bad --- /dev/null +++ b/deploy/litellm/k8s-docker-desktop.yaml @@ -0,0 +1,186 @@ +# LiteLLM on Docker Desktop Kubernetes +# +# Prerequisites: +# 1. Docker Desktop → Settings → Kubernetes → Enable Kubernetes +# 2. Apply & wait until green +# 3. kubectl config use-context docker-desktop +# +# Usage: +# 1. Put MadEye token in a local secret file (not committed): +# cp secret.env.example secret.env +# # edit MADEYE_API_KEY +# 2. ./apply-docker-desktop.sh +# 3. curl http://localhost:4000/v1/messages ... + +apiVersion: v1 +kind: Namespace +metadata: + name: gastown-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: litellm-config + namespace: gastown-system +data: + config.yaml: | + model_list: + - model_name: claude-opus-4-8 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: "MADEYE_API_KEY_PLACEHOLDER" + extra_body: + metadata: + user_email: "MADEYE_USER_EMAIL_PLACEHOLDER" + # Claude Code often requests Haiku for small side calls — map to MadEye model + - model_name: claude-haiku-4-5-20251001 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: "MADEYE_API_KEY_PLACEHOLDER" + extra_body: + metadata: + user_email: "MADEYE_USER_EMAIL_PLACEHOLDER" + - model_name: claude-haiku-4-5 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: "MADEYE_API_KEY_PLACEHOLDER" + extra_body: + metadata: + user_email: "MADEYE_USER_EMAIL_PLACEHOLDER" + - model_name: claude-sonnet-4 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: "MADEYE_API_KEY_PLACEHOLDER" + extra_body: + metadata: + user_email: "MADEYE_USER_EMAIL_PLACEHOLDER" + - model_name: claude-sonnet-4-5-20250929 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: "MADEYE_API_KEY_PLACEHOLDER" + extra_body: + metadata: + user_email: "MADEYE_USER_EMAIL_PLACEHOLDER" + litellm_settings: + drop_params: true + # MadEye only supports OpenAI /v1/chat/completions (NOT Responses API). + # Without this, Claude Code /v1/messages is translated via Responses and + # MadEye rejects with "assistant message prefill". + use_chat_completions_url_for_anthropic_messages: true + # MadEye rejects assistant prefills (Claude Code sends them). + callbacks: strip_prefill.proxy_handler_instance + general_settings: + master_key: "LITELLM_MASTER_KEY_PLACEHOLDER" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm + namespace: gastown-system + labels: + app: litellm +spec: + replicas: 1 + selector: + matchLabels: + app: litellm + template: + metadata: + labels: + app: litellm + spec: + containers: + - name: litellm + image: ghcr.io/berriai/litellm:main-latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/app/k8s-entrypoint.sh"] + ports: + - containerPort: 4000 + name: http + env: + - name: MADEYE_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: MADEYE_API_KEY + - name: MADEYE_USER_EMAIL + valueFrom: + secretKeyRef: + name: litellm-secrets + key: MADEYE_USER_EMAIL + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: LITELLM_MASTER_KEY + - name: PYTHONPATH + value: /app + # Force Anthropic /v1/messages → OpenAI /chat/completions (not /responses). + # YAML key is unreliable in some LiteLLM builds; env var is authoritative. + - name: LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES + value: "true" + volumeMounts: + - name: config + mountPath: /app/config.template.yaml + subPath: config.yaml + readOnly: true + - name: callbacks + mountPath: /app/strip_prefill.py + subPath: strip_prefill.py + readOnly: true + - name: callbacks + mountPath: /app/prefill_proxy.py + subPath: prefill_proxy.py + readOnly: true + - name: callbacks + mountPath: /app/k8s-entrypoint.sh + subPath: k8s-entrypoint.sh + readOnly: true + readinessProbe: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + volumes: + - name: config + configMap: + name: litellm-config + - name: callbacks + configMap: + name: litellm-callbacks +--- +# NodePort so Docker Desktop exposes LiteLLM on localhost without port-forward +apiVersion: v1 +kind: Service +metadata: + name: litellm + namespace: gastown-system +spec: + type: NodePort + selector: + app: litellm + ports: + - name: http + port: 4000 + targetPort: http + nodePort: 30040 diff --git a/deploy/litellm/k8s-entrypoint.sh b/deploy/litellm/k8s-entrypoint.sh new file mode 100644 index 0000000..ce31f49 --- /dev/null +++ b/deploy/litellm/k8s-entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -e + +# MadEye speaks OpenAI chat/completions only. Force LiteLLM off the Responses API +# path when Claude Code calls Anthropic /v1/messages (critical for prefill errors). +export LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true + +# Inject secrets into config (more reliable than LiteLLM os.environ/ syntax) +sed \ + -e "s|MADEYE_API_KEY_PLACEHOLDER|${MADEYE_API_KEY}|g" \ + -e "s|MADEYE_USER_EMAIL_PLACEHOLDER|${MADEYE_USER_EMAIL}|g" \ + -e "s|LITELLM_MASTER_KEY_PLACEHOLDER|${LITELLM_MASTER_KEY}|g" \ + /app/config.template.yaml > /tmp/config.yaml + +# LiteLLM internally; edge proxy on :4000 strips Claude Code prefills +litellm --config /tmp/config.yaml --port 4001 --host 127.0.0.1 & +LITELLM_PID=$! +trap 'kill ${LITELLM_PID} 2>/dev/null || true' EXIT INT TERM + +python -c 'import time,urllib.request +for _ in range(90): + try: + urllib.request.urlopen("http://127.0.0.1:4001/health/liveliness", timeout=2); break + except Exception: + time.sleep(1) +else: + raise SystemExit("LiteLLM not ready on :4001")' + +export LITELLM_UPSTREAM=http://127.0.0.1:4001 +export PREFILL_PROXY_PORT=4000 +exec python /app/prefill_proxy.py diff --git a/deploy/litellm/k8s.yaml b/deploy/litellm/k8s.yaml new file mode 100644 index 0000000..0ad15ed --- /dev/null +++ b/deploy/litellm/k8s.yaml @@ -0,0 +1,138 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gastown-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: litellm-secrets + namespace: gastown-system +type: Opaque +stringData: + # MadEye Bearer token (Authorization: Bearer …) + MADEYE_API_KEY: "REPLACE_WITH_MADEYE_TOKEN" + # Required by MadEye metadata + MADEYE_USER_EMAIL: "priyanshu.rajput@pocketfm.com" + # Key Claude Code / Polecats use as ANTHROPIC_AUTH_TOKEN + LITELLM_MASTER_KEY: "sk-local-litellm" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: litellm-config + namespace: gastown-system +data: + config.yaml: | + model_list: + - model_name: claude-opus-4-8 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: os.environ/MADEYE_API_KEY + extra_body: + metadata: + user_email: priyanshu.rajput@pocketfm.com + - model_name: claude-sonnet-4 + litellm_params: + model: openai/claude-opus-4-8 + api_base: https://madeye.internal.pocketfm.org/v1 + api_key: os.environ/MADEYE_API_KEY + extra_body: + metadata: + user_email: priyanshu.rajput@pocketfm.com + litellm_settings: + drop_params: true + use_chat_completions_url_for_anthropic_messages: true + callbacks: strip_prefill.proxy_handler_instance + general_settings: + master_key: os.environ/LITELLM_MASTER_KEY +--- +# Apply strip_prefill.py / prefill_proxy separately via ConfigMap litellm-callbacks. +# Also set env LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm + namespace: gastown-system + labels: + app: litellm +spec: + replicas: 1 + selector: + matchLabels: + app: litellm + template: + metadata: + labels: + app: litellm + spec: + containers: + - name: litellm + image: ghcr.io/berriai/litellm:main-latest + args: ["--config", "/app/config.yaml", "--port", "4000", "--host", "0.0.0.0"] + ports: + - containerPort: 4000 + name: http + env: + - name: MADEYE_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: MADEYE_API_KEY + - name: MADEYE_USER_EMAIL + valueFrom: + secretKeyRef: + name: litellm-secrets + key: MADEYE_USER_EMAIL + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: LITELLM_MASTER_KEY + - name: PYTHONPATH + value: /app + - name: LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES + value: "true" + volumeMounts: + - name: config + mountPath: /app/config.yaml + subPath: config.yaml + readOnly: true + - name: callbacks + mountPath: /app/strip_prefill.py + subPath: strip_prefill.py + readOnly: true + readinessProbe: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + volumes: + - name: config + configMap: + name: litellm-config + - name: callbacks + configMap: + name: litellm-callbacks +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm + namespace: gastown-system +spec: + selector: + app: litellm + ports: + - name: http + port: 4000 + targetPort: http diff --git a/deploy/litellm/prefill_proxy.py b/deploy/litellm/prefill_proxy.py new file mode 100644 index 0000000..493f2d9 --- /dev/null +++ b/deploy/litellm/prefill_proxy.py @@ -0,0 +1,171 @@ +"""HTTP front-proxy: strip trailing assistant prefills before LiteLLM. + +Also logs message roles for every /v1/messages call so we can debug Claude Code. +""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import urlsplit + + +UPSTREAM = os.environ.get("LITELLM_UPSTREAM", "http://127.0.0.1:4001").rstrip("/") +LISTEN_HOST = os.environ.get("PREFILL_PROXY_HOST", "0.0.0.0") +LISTEN_PORT = int(os.environ.get("PREFILL_PROXY_PORT", "4000")) +HOP_BY_HOP = { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "content-length", + "host", +} + + +def _role(msg: Any) -> str | None: + if isinstance(msg, dict): + role = msg.get("role") + else: + role = getattr(msg, "role", None) + return str(role).lower() if role is not None else None + + +def strip_trailing_assistant(messages: list[Any]) -> tuple[list[Any], int]: + if not isinstance(messages, list) or not messages: + return messages, 0 + out = list(messages) + dropped = 0 + while out and _role(out[-1]) == "assistant": + out.pop() + dropped += 1 + return out, dropped + + +def maybe_rewrite_body(path: str, body: bytes) -> bytes: + if not body or "/messages" not in path: + return body + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return body + if not isinstance(payload, dict) or "messages" not in payload: + return body + messages = payload.get("messages") or [] + roles = [_role(m) for m in messages[-6:]] + cleaned, dropped = strip_trailing_assistant(messages) + print( + f"[prefill_proxy] {path}: roles_tail={roles} dropped={dropped} " + f"model={payload.get('model')}", + flush=True, + ) + if not dropped: + return body + payload["messages"] = cleaned + return json.dumps(payload, separators=(",", ":")).encode("utf-8") + + +class PrefillProxy(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt: str, *args: Any) -> None: + print("[prefill_proxy] " + (fmt % args), flush=True) + + def _proxy(self) -> None: + parsed = urlsplit(self.path) + path_q = parsed.path + if parsed.query: + path_q = f"{path_q}?{parsed.query}" + + length = int(self.headers.get("content-length", "0") or "0") + body = self.rfile.read(length) if length > 0 else b"" + if self.command in ("POST", "PUT", "PATCH"): + body = maybe_rewrite_body(parsed.path, body) + + headers = { + k: v + for k, v in self.headers.items() + if k.lower() not in HOP_BY_HOP + } + req = urllib.request.Request( + url=f"{UPSTREAM}{path_q}", + data=body if self.command not in ("GET", "HEAD", "DELETE") else None, + headers=headers, + method=self.command, + ) + try: + with urllib.request.urlopen(req, timeout=600) as resp: + resp_body = resp.read() + self.send_response(resp.status) + for k, v in resp.headers.items(): + if k.lower() in HOP_BY_HOP: + continue + self.send_header(k, v) + self.send_header("Content-Length", str(len(resp_body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(resp_body) + except urllib.error.HTTPError as exc: + resp_body = exc.read() + print( + f"[prefill_proxy] upstream HTTP {exc.code} for {path_q}: " + f"{resp_body[:300]!r}", + flush=True, + ) + self.send_response(exc.code) + for k, v in exc.headers.items(): + if k.lower() in HOP_BY_HOP: + continue + self.send_header(k, v) + self.send_header("Content-Length", str(len(resp_body))) + self.end_headers() + self.wfile.write(resp_body) + except Exception as exc: # noqa: BLE001 - edge proxy + msg = json.dumps({"error": f"prefill_proxy upstream failure: {exc}"}).encode() + self.send_response(502) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(msg))) + self.end_headers() + self.wfile.write(msg) + + def do_GET(self) -> None: # noqa: N802 + self._proxy() + + def do_HEAD(self) -> None: # noqa: N802 + self._proxy() + + def do_POST(self) -> None: # noqa: N802 + self._proxy() + + def do_PUT(self) -> None: # noqa: N802 + self._proxy() + + def do_PATCH(self) -> None: # noqa: N802 + self._proxy() + + def do_DELETE(self) -> None: # noqa: N802 + self._proxy() + + def do_OPTIONS(self) -> None: # noqa: N802 + self._proxy() + + +def main() -> None: + server = ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), PrefillProxy) + print( + f"[prefill_proxy] listening on {LISTEN_HOST}:{LISTEN_PORT} → {UPSTREAM}", + flush=True, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/deploy/litellm/secret.env.example b/deploy/litellm/secret.env.example new file mode 100644 index 0000000..a349e5a --- /dev/null +++ b/deploy/litellm/secret.env.example @@ -0,0 +1,4 @@ +# Copy to secret.env (gitignored) before running apply-docker-desktop.sh +MADEYE_API_KEY=replace-with-madeye-bearer-token +MADEYE_USER_EMAIL=priyanshu.rajput@pocketfm.com +LITELLM_MASTER_KEY=sk-local-litellm diff --git a/deploy/litellm/strip_prefill.py b/deploy/litellm/strip_prefill.py new file mode 100644 index 0000000..ef5d7e5 --- /dev/null +++ b/deploy/litellm/strip_prefill.py @@ -0,0 +1,76 @@ +"""Strip trailing assistant prefills before MadEye rejects them. + +MadEye (Vertex-backed claude-opus-4-8) requires conversations to end with a +user message. Claude Code often sends a trailing assistant turn (prefill). +This LiteLLM proxy hook drops those trailing assistant messages. +""" + +from __future__ import annotations + +from typing import Any, Literal, Optional + +from litellm.integrations.custom_logger import CustomLogger + +try: + from litellm.proxy.proxy_server import DualCache, UserAPIKeyAuth +except ImportError: # pragma: no cover - version drift + from litellm.caching.caching import DualCache + from litellm.proxy._types import UserAPIKeyAuth + + +def _role(msg: Any) -> str | None: + if isinstance(msg, dict): + role = msg.get("role") + else: + role = getattr(msg, "role", None) + return str(role).lower() if role is not None else None + + +def _strip_trailing_assistant(messages: list[Any]) -> list[Any]: + if not isinstance(messages, list) or not messages: + return messages + out = list(messages) + while out and _role(out[-1]) == "assistant": + out.pop() + return out + + +class StripPrefillHandler(CustomLogger): + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: Literal[ + "completion", + "text_completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + ], + ) -> Optional[dict]: + messages = data.get("messages") + if not messages: + # Always log trailing roles so we can debug Claude Code shapes. + print("[strip_prefill] no messages in request data", flush=True) + return data + roles = [_role(m) for m in messages[-4:]] + cleaned = _strip_trailing_assistant(messages) + if len(cleaned) != len(messages): + print( + f"[strip_prefill] dropped {len(messages) - len(cleaned)} " + f"trailing assistant message(s); last_roles={roles}", + flush=True, + ) + data["messages"] = cleaned + elif roles and roles[-1] == "assistant": + print( + f"[strip_prefill] WARNING still ends with assistant after strip? " + f"last_roles={roles}", + flush=True, + ) + return data + + +proxy_handler_instance = StripPrefillHandler() diff --git a/docs/CONFIG.md b/docs/CONFIG.md index da5b421..ca34b2f 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -164,9 +164,14 @@ data: | `executionMode` | `kubernetes` | Run as Pod | | `agent` | `claude-code` | Coding agent type | | `agentConfig.provider` | `litellm` | LLM provider | +| `agentConfig.model` | - | Sets `ANTHROPIC_MODEL` on the Polecat pod | +| `agentConfig.modelProvider.endpoint` | - | Sets `ANTHROPIC_BASE_URL` (LiteLLM / gateway) | +| `agentConfig.modelProvider.apiKeySecretRef` | - | Sets `ANTHROPIC_AUTH_TOKEN` (Bearer) | | `kubernetes.gitBranch` | `main` | Base branch | | `kubernetes.activeDeadlineSeconds` | `3600` | 1 hour max runtime | +For MadEye (OpenAI-only) via LiteLLM, see [LITE_LLM.md](LITE_LLM.md). + ### Rig Defaults | Field | Default | Notes | diff --git a/docs/LITE_LLM.md b/docs/LITE_LLM.md new file mode 100644 index 0000000..457f1fa --- /dev/null +++ b/docs/LITE_LLM.md @@ -0,0 +1,181 @@ +# LiteLLM + MadEye + +Route Polecat Claude Code traffic through LiteLLM to PocketFM MadEye +(`OpenAI /v1/chat/completions` only). + +``` +Polecat (claude) + Anthropic /v1/messages + ANTHROPIC_BASE_URL + ↓ +LiteLLM (shared Deployment / docker-compose) + translates tools/messages + ↓ +MadEye https://madeye.internal.pocketfm.org/v1/chat/completions + Bearer token + metadata.user_email +``` + +## Prerequisites + +- MadEye reachable from your machine (VPN / internal DNS) +- MadEye Bearer token +- Your PocketFM email for MadEye `metadata.user_email` + +--- + +## 1. Taste LiteLLM locally (no Kubernetes) + +```bash +cd deploy/litellm + +export MADEYE_API_KEY='your-madeye-bearer-token' +export MADEYE_USER_EMAIL='priyanshu.rajput@pocketfm.com' +export LITELLM_MASTER_KEY='sk-local-litellm' + +chmod +x entrypoint.sh +docker compose up -d +``` + +Smoke-test Anthropic-format call through LiteLLM: + +```bash +curl -s http://localhost:4000/v1/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role":"user","content":"Hello! which model are you using?"}] + }' +``` + +If that returns a model reply, the adapter works. Claude Code uses the same path. + +Optional — point local Claude Code at it: + +```bash +export ANTHROPIC_BASE_URL=http://localhost:4000 +export ANTHROPIC_AUTH_TOKEN="${LITELLM_MASTER_KEY}" +export ANTHROPIC_MODEL=claude-opus-4-8 +claude --print "Hello! which model are you using?" +``` + +--- + +## 2. Deploy LiteLLM on Docker Desktop Kubernetes + +**Enable the cluster first** (one-time): + +1. Open **Docker Desktop → Settings → Kubernetes** +2. Check **Enable Kubernetes** +3. **Apply & Restart** and wait until Kubernetes shows green +4. Verify: + +```bash +kubectl config use-context docker-desktop +kubectl get nodes +``` + +Then deploy LiteLLM: + +```bash +cd deploy/litellm +cp secret.env.example secret.env +# edit MADEYE_API_KEY (and email if needed) + +chmod +x apply-docker-desktop.sh +./apply-docker-desktop.sh +``` + +Smoke test (NodePort **30040** on Docker Desktop): + +```bash +curl -s http://localhost:30040/v1/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-local-litellm" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role":"user","content":"Hello! which model are you using?"}] + }' +``` + +MadEye must be reachable from Docker Desktop (VPN on the host usually works for outbound pod traffic). + +--- + +## 3. Deploy LiteLLM in any Kubernetes + +```bash +# Edit MadEye token + email in the Secret +vim deploy/litellm/k8s.yaml + +kubectl apply -f deploy/litellm/k8s.yaml +kubectl -n gastown-system rollout status deploy/litellm +kubectl -n gastown-system port-forward svc/litellm 4000:4000 +``` + +Re-run the same `curl` against `http://localhost:4000`. + +--- + +## 4. Polecat agentConfig (operator wiring) + +The pod builder injects: + +| `agentConfig` field | Container env | +|---------------------|---------------| +| `modelProvider.endpoint` | `ANTHROPIC_BASE_URL` | +| `modelProvider.apiKeySecretRef` | `ANTHROPIC_AUTH_TOKEN` | +| `model` | `ANTHROPIC_MODEL` | +| `env` | passthrough | + +Sample: `config/samples/litellm/polecat-madeye.yaml` + +```yaml +agentConfig: + provider: litellm + model: claude-opus-4-8 + modelProvider: + endpoint: http://litellm.gastown-system.svc:4000 + apiKeySecretRef: + name: litellm-auth + key: master-key +``` + +--- + +## Local Kind path (operator + LiteLLM) + +Prefer Docker Desktop full stack: see [deploy/docker-desktop/README.md](../deploy/docker-desktop/README.md) or `make demo-docker-desktop`. + +```bash +# 1. Build/load operator into Kind (or make demo) +make demo + +# 2. Deploy LiteLLM +kubectl apply -f deploy/litellm/k8s.yaml + +# 3. Apply sample (after editing git repo/secrets as needed) +kubectl apply -f config/samples/litellm/polecat-madeye.yaml + +# 4. Watch +kubectl -n gastown-system logs -f deploy/litellm +kubectl gt polecat logs smoke/madeye-smoke -f -n gastown-system +``` + +Start with step 1 (docker compose + curl) before involving the operator. + +--- + +## Troubleshooting + +| Symptom | Check | +|---------|--------| +| curl to LiteLLM fails DNS to MadEye | VPN / can you curl MadEye directly? | +| 401 from MadEye | `MADEYE_API_KEY` Bearer token | +| MadEye rejects request | `user_email` in config / Secret | +| `does not support assistant message prefill` | LiteLLM was routing Claude Code through OpenAI **Responses API**. Set `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true` and redeploy. Also ensure `prefill_proxy` strips trailing assistant turns. | +| Polecat still hits api.anthropic.com | `ANTHROPIC_BASE_URL` missing — confirm `agentConfig` on Polecat | +| Tool-call errors | LiteLLM version; check LiteLLM logs for translation failures | diff --git a/images/polecat-agent/Dockerfile b/images/polecat-agent/Dockerfile index fd2a5f2..b82a049 100644 --- a/images/polecat-agent/Dockerfile +++ b/images/polecat-agent/Dockerfile @@ -25,7 +25,7 @@ ARG CLAUDE_CODE_VERSION=2.0.22 # Gas Town CLI: https://github.com/steveyegge/gastown ARG GT_VERSION=main -ARG GOLANG_VERSION=1.24 +ARG GOLANG_VERSION=1.26 # Base image - Debian slim for glibc compatibility ARG DEBIAN_VERSION=bookworm-slim @@ -38,6 +38,7 @@ RUN apk add --no-cache git WORKDIR /build ARG GT_VERSION +ENV GOTOOLCHAIN=auto RUN git clone --depth 1 --branch ${GT_VERSION} https://github.com/steveyegge/gastown.git . && \ CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gt ./cmd/gt diff --git a/pkg/pod/builder.go b/pkg/pod/builder.go index 2d7eb57..c3990f7 100644 --- a/pkg/pod/builder.go +++ b/pkg/pod/builder.go @@ -335,9 +335,13 @@ func (b *Builder) buildGitInitVolumeMounts() []corev1.VolumeMount { // buildClaudeContainer creates the Claude agent container spec func (b *Builder) buildClaudeContainer() corev1.Container { k8sSpec := b.polecat.Spec.Kubernetes + agentConfig := b.polecat.Spec.AgentConfig - // Use custom image if specified, otherwise use configured default + // Image precedence: kubernetes.image > agentConfig.image > env/default image := GetClaudeImage() + if agentConfig != nil && agentConfig.Image != "" { + image = agentConfig.Image + } if k8sSpec.Image != "" { image = k8sSpec.Image } @@ -435,8 +439,11 @@ exec claude --print --dangerously-skip-permissions "$PROMPT" }, } - // Add API key from secret if configured - if k8sSpec.ApiKeySecretRef != nil { + envVars = append(envVars, b.buildAgentConfigEnv()...) + + // Direct Anthropic API key (used when not routing through a gateway). + // Gateway auth from agentConfig.modelProvider takes precedence via ANTHROPIC_AUTH_TOKEN. + if k8sSpec.ApiKeySecretRef != nil && !b.hasGatewayAuth() { envVars = append(envVars, corev1.EnvVar{ Name: "ANTHROPIC_API_KEY", ValueFrom: &corev1.EnvVarSource{ @@ -495,6 +502,70 @@ exec claude --print --dangerously-skip-permissions "$PROMPT" return container } +// hasGatewayAuth reports whether agentConfig provides a gateway API key secret. +// When true, Claude Code should authenticate with ANTHROPIC_AUTH_TOKEN (Bearer) +// instead of ANTHROPIC_API_KEY (x-api-key). +func (b *Builder) hasGatewayAuth() bool { + ac := b.polecat.Spec.AgentConfig + return ac != nil && + ac.ModelProvider != nil && + ac.ModelProvider.APIKeySecretRef != nil +} + +// buildAgentConfigEnv injects LLM gateway settings from Polecat agentConfig. +// This wires LiteLLM / custom OpenAI-compatible backends for Claude Code: +// - ANTHROPIC_BASE_URL from modelProvider.endpoint +// - ANTHROPIC_AUTH_TOKEN from modelProvider.apiKeySecretRef (Bearer auth) +// - ANTHROPIC_MODEL from agentConfig.model +// - any extra agentConfig.env entries +func (b *Builder) buildAgentConfigEnv() []corev1.EnvVar { + ac := b.polecat.Spec.AgentConfig + if ac == nil { + return nil + } + + var envVars []corev1.EnvVar + + if ac.Model != "" { + envVars = append(envVars, + corev1.EnvVar{Name: "ANTHROPIC_MODEL", Value: ac.Model}, + // Keep Claude Code's haiku/sonnet side-calls on the same allowed model. + corev1.EnvVar{Name: "ANTHROPIC_DEFAULT_OPUS_MODEL", Value: ac.Model}, + corev1.EnvVar{Name: "ANTHROPIC_DEFAULT_SONNET_MODEL", Value: ac.Model}, + corev1.EnvVar{Name: "ANTHROPIC_DEFAULT_HAIKU_MODEL", Value: ac.Model}, + ) + } + + if ac.ModelProvider != nil { + if ac.ModelProvider.Endpoint != "" { + envVars = append(envVars, corev1.EnvVar{ + Name: "ANTHROPIC_BASE_URL", + Value: ac.ModelProvider.Endpoint, + }) + } + if ac.ModelProvider.APIKeySecretRef != nil { + ref := ac.ModelProvider.APIKeySecretRef + envVars = append(envVars, corev1.EnvVar{ + Name: "ANTHROPIC_AUTH_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ref.Name, + }, + Key: ref.Key, + }, + }, + }) + } + } + + if len(ac.Env) > 0 { + envVars = append(envVars, ac.Env...) + } + + return envVars +} + // buildTelemetrySidecar creates the telemetry sidecar container spec // //nolint:lll // Prometheus metric lines in embedded shell script cannot be broken diff --git a/pkg/pod/builder_test.go b/pkg/pod/builder_test.go index baf4f37..77ea0fa 100644 --- a/pkg/pod/builder_test.go +++ b/pkg/pod/builder_test.go @@ -609,6 +609,133 @@ func TestContainerEnvironment(t *testing.T) { }) } +func TestAgentConfigGatewayEnv(t *testing.T) { + polecat := &gastownv1alpha1.Polecat{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-polecat", + Namespace: "default", + }, + Spec: gastownv1alpha1.PolecatSpec{ + Rig: "test-rig", + BeadID: "test-bead", + AgentConfig: &gastownv1alpha1.AgentConfig{ + Provider: gastownv1alpha1.LLMProviderLiteLLM, + Model: "claude-opus-4-8", + ModelProvider: &gastownv1alpha1.ModelProviderConfig{ + Endpoint: "http://litellm.gastown-system.svc:4000", + APIKeySecretRef: &gastownv1alpha1.SecretKeyRef{ + Name: "litellm-auth", + Key: "master-key", + }, + }, + Env: []corev1.EnvVar{ + {Name: "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY", Value: "1"}, + }, + }, + Kubernetes: &gastownv1alpha1.KubernetesSpec{ + GitRepository: "git@github.com:org/repo.git", + GitBranch: "main", + GitSecretRef: gastownv1alpha1.SecretReference{Name: "git-secret"}, + ApiKeySecretRef: &gastownv1alpha1.SecretKeyRef{ + Name: "anthropic-api-key", + Key: "api-key", + }, + }, + }, + } + + builder := NewBuilder(polecat) + pod, err := builder.Build() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + envVars := pod.Spec.Containers[0].Env + envMap := make(map[string]corev1.EnvVar, len(envVars)) + for _, env := range envVars { + envMap[env.Name] = env + } + + t.Run("sets ANTHROPIC_BASE_URL from modelProvider.endpoint", func(t *testing.T) { + got := envMap["ANTHROPIC_BASE_URL"].Value + want := "http://litellm.gastown-system.svc:4000" + if got != want { + t.Errorf("expected ANTHROPIC_BASE_URL=%s, got %s", want, got) + } + }) + + t.Run("sets ANTHROPIC_MODEL from agentConfig.model", func(t *testing.T) { + if envMap["ANTHROPIC_MODEL"].Value != "claude-opus-4-8" { + t.Errorf("expected ANTHROPIC_MODEL=claude-opus-4-8, got %s", envMap["ANTHROPIC_MODEL"].Value) + } + }) + + t.Run("sets ANTHROPIC_AUTH_TOKEN from modelProvider.apiKeySecretRef", func(t *testing.T) { + auth := envMap["ANTHROPIC_AUTH_TOKEN"] + if auth.ValueFrom == nil || auth.ValueFrom.SecretKeyRef == nil { + t.Fatal("expected ANTHROPIC_AUTH_TOKEN from secret") + } + if auth.ValueFrom.SecretKeyRef.Name != "litellm-auth" { + t.Errorf("expected secret litellm-auth, got %s", auth.ValueFrom.SecretKeyRef.Name) + } + if auth.ValueFrom.SecretKeyRef.Key != "master-key" { + t.Errorf("expected key master-key, got %s", auth.ValueFrom.SecretKeyRef.Key) + } + }) + + t.Run("skips ANTHROPIC_API_KEY when gateway auth is configured", func(t *testing.T) { + if _, exists := envMap["ANTHROPIC_API_KEY"]; exists { + t.Error("ANTHROPIC_API_KEY should not be set when gateway auth is present") + } + }) + + t.Run("appends agentConfig.env", func(t *testing.T) { + if envMap["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"].Value != "1" { + t.Error("expected agentConfig.env to be appended") + } + }) +} + +func TestAgentConfigImageOverride(t *testing.T) { + polecat := &gastownv1alpha1.Polecat{ + ObjectMeta: metav1.ObjectMeta{ + Name: "image-polecat", + Namespace: "default", + }, + Spec: gastownv1alpha1.PolecatSpec{ + Rig: "test-rig", + AgentConfig: &gastownv1alpha1.AgentConfig{ + Image: "ghcr.io/example/custom-agent:v1", + }, + Kubernetes: &gastownv1alpha1.KubernetesSpec{ + GitRepository: "git@github.com:org/repo.git", + GitSecretRef: gastownv1alpha1.SecretReference{Name: "git-secret"}, + ClaudeCredsSecretRef: &gastownv1alpha1.SecretReference{Name: "claude-secret"}, + }, + }, + } + + builder := NewBuilder(polecat) + pod, err := builder.Build() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if pod.Spec.Containers[0].Image != "ghcr.io/example/custom-agent:v1" { + t.Errorf("expected agentConfig.image override, got %s", pod.Spec.Containers[0].Image) + } + + // kubernetes.image should win over agentConfig.image + polecat.Spec.Kubernetes.Image = "ghcr.io/example/k8s-override:v2" + pod, err = NewBuilder(polecat).Build() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pod.Spec.Containers[0].Image != "ghcr.io/example/k8s-override:v2" { + t.Errorf("expected kubernetes.image to win, got %s", pod.Spec.Containers[0].Image) + } +} + func TestClaudeCredsVolumeMount(t *testing.T) { polecat := &gastownv1alpha1.Polecat{ ObjectMeta: metav1.ObjectMeta{ diff --git a/templates/polecat-kubernetes.yaml b/templates/polecat-kubernetes.yaml index 2ca33d4..670bed8 100644 --- a/templates/polecat-kubernetes.yaml +++ b/templates/polecat-kubernetes.yaml @@ -79,26 +79,26 @@ spec: agent: claude-code # Agent customization (optional) - # Use for custom LLM providers or gateway endpoints + # Use for LiteLLM / custom LLM gateways (e.g. MadEye via LiteLLM). + # See docs/LITE_LLM.md # agentConfig: - # # LLM provider # provider: litellm # litellm, anthropic, openai, ollama - # model: claude-sonnet-4 + # model: claude-opus-4-8 # - # # Custom API endpoint (for LiteLLM gateway) + # # Claude Code talks Anthropic format to this URL (ANTHROPIC_BASE_URL) # modelProvider: - # endpoint: https://ai-gateway.example.com/v1 - # apiKeySecretRef: - # name: litellm-api-key - # key: api-key + # endpoint: http://litellm.gastown-system.svc:4000 + # apiKeySecretRef: # → ANTHROPIC_AUTH_TOKEN (Bearer) + # name: litellm-auth + # key: master-key # # # Custom container image # image: ghcr.io/boshu2/polecat-agent:custom # # # Extra environment variables # env: - # - name: LOG_LEVEL - # value: "debug" + # - name: CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY + # value: "1" # ============================================ # KUBERNETES EXECUTION CONFIG