diff --git a/caipe/base/ai-platform-engineering.yaml b/caipe/base/ai-platform-engineering.yaml index 83a33f51..5ab1110b 100644 --- a/caipe/base/ai-platform-engineering.yaml +++ b/caipe/base/ai-platform-engineering.yaml @@ -11,7 +11,7 @@ spec: # Main chart from GHCR - chart: ai-platform-engineering repoURL: ghcr.io/cnoe-io/helm-charts - targetRevision: 0.3.3 + targetRevision: 0.4.14 helm: parameters: - name: tags.basic diff --git a/caipe/base/ai-platform-engineering/values.yaml b/caipe/base/ai-platform-engineering/values.yaml index 05e299ed..d7c82343 100644 --- a/caipe/base/ai-platform-engineering/values.yaml +++ b/caipe/base/ai-platform-engineering/values.yaml @@ -1,6 +1,10 @@ # Override values for idpbuilder local deployment - Base configuration # Global configuration shared across all subcharts global: + # Disable graph RAG for now + rag: + enableGraphRag: false + slim: enabled: false @@ -108,24 +112,52 @@ global: key: secret/ai-platform-engineering/global property: AWS_BEDROCK_PROVIDER +promptConfigType: "deep_agent" + supervisor-agent: nameOverride: "supervisor-agent" image: repository: "ghcr.io/cnoe-io/ai-platform-engineering" - tag: "0.1.19" + tag: "0.2.4" env: EXTERNAL_URL: "https://cnoe.localtest.me:8443/ai-platform-engineering" RAG_AGENT_PORT: 8099 + + # Agent Connectivity Configuration + AGENT_CONNECTIVITY_ENABLE_BACKGROUND: "true" # Routinely checks each subagent connectivity to add or remove any from existing tools list + SKIP_AGENT_CONNECTIVITY_CHECK: "false" # Do not skip the connectivity check; supervisor agent will check each subagent is reachable + + # Orchestration and Streaming Configuration + ENABLE_ENHANCED_STREAMING: "false" # Enable enhanced streaming with intelligent routing (DIRECT/PARALLEL/COMPLEX modes) + FORCE_DEEP_AGENT_ORCHESTRATION: "true" # Force all queries through Deep Agent with parallel orchestration hints (DEFAULT - best performance) + ENABLE_ENHANCED_ORCHESTRATION: "false" # EXPERIMENTAL: Smart routing + orchestration hints (4th mode for comparison) + STREAM_SUB_AGENT_TOOL_OUTPUT: "true" # Stream intermediate tool outputs (📄) from sub-agents to end user + + # Response Format Configuration + ENABLE_STRUCTURED_OUTPUT: "false" # Set to "true" to enforce execution plans with Pydantic validation + ENABLE_ARTIFACT_STREAMING: "true" + + # # OAuth2 Configuration + A2A_AUTH_OAUTH2: true + # Backstage authentication + # JWKS_URI and TOKEN_ENDPOINT use internal cluster service to avoid SSL issues + # ISSUER must match the external URL because that's what's in the JWT tokens + JWKS_URI: "http://backstage.backstage.svc.cluster.local:7007/api/auth/.well-known/jwks.json" + AUDIENCE: "backstage" + ISSUER: "https://cnoe.localtest.me:8443/api/auth" + OAUTH2_CLIENT_ID: "backstage" + TOKEN_ENDPOINT: "http://backstage.backstage.svc.cluster.local:7007/api/auth/oauth2/token" + agent-argocd: nameOverride: "agent-argocd" image: repository: "ghcr.io/cnoe-io/agent-argocd" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mcp: image: repository: "ghcr.io/cnoe-io/mcp-argocd" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -151,7 +183,7 @@ agent-aws: image: repository: "ghcr.io/cnoe-io/agent-aws" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: mode: "stdio" # AWS currently does not support HTTP mode agentSecrets: @@ -176,11 +208,11 @@ agent-backstage: image: repository: "ghcr.io/cnoe-io/agent-backstage" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-backstage" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -202,11 +234,11 @@ agent-confluence: image: repository: "ghcr.io/cnoe-io/agent-confluence" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-confluence" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -232,7 +264,7 @@ agent-github: image: repository: "ghcr.io/cnoe-io/agent-github" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: useRemoteMcpServer: true agentSecrets: @@ -249,11 +281,11 @@ agent-jira: image: repository: "ghcr.io/cnoe-io/agent-jira" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-jira" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -280,11 +312,11 @@ agent-komodor: image: repository: "ghcr.io/cnoe-io/agent-komodor" pullPolicy: "Always" - tag: "0.1.19" + tag: "41c4392" # Use the latest fix that includes https://github.com/cnoe-io/ai-platform-engineering/pull/507. This is a temporary fix until the next release of the agent-komodor chart. mcp: image: repository: "ghcr.io/cnoe-io/mcp-komodor" - tag: "0.1.19" + tag: "41c4392" # Use the latest fix that includes https://github.com/cnoe-io/ai-platform-engineering/pull/507. This is a temporary fix until the next release of the mcp-komodor chart. pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -306,11 +338,11 @@ agent-pagerduty: image: repository: "ghcr.io/cnoe-io/agent-pagerduty" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-pagerduty" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -332,11 +364,11 @@ agent-slack: image: repository: "ghcr.io/cnoe-io/agent-slack" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-slack" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -371,11 +403,11 @@ agent-splunk: image: repository: "ghcr.io/cnoe-io/agent-splunk" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-splunk" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -397,11 +429,11 @@ agent-webex: image: repository: "ghcr.io/cnoe-io/agent-webex" pullPolicy: "Always" - tag: "0.1.19" + tag: "0.2.4" mcp: image: repository: "ghcr.io/cnoe-io/mcp-webex" - tag: "0.1.19" + tag: "0.2.4" pullPolicy: "Always" mode: "http" # Options: stdio, http port: 8000 @@ -415,7 +447,23 @@ agent-webex: property: WEBEX_TOKEN rag-stack: + rag-server: + image: + repository: "ghcr.io/cnoe-io/caipe-rag-server" + tag: "0.2.4" + pullPolicy: "Always" + + agent-rag: + image: + repository: "ghcr.io/cnoe-io/caipe-rag-agent-rag" + tag: "0.2.4" + pullPolicy: "Always" + rag-webui: + image: + repository: "ghcr.io/cnoe-io/caipe-rag-webui" + tag: "0.2.4" + pullPolicy: "Always" ingress: enabled: true className: "nginx" @@ -451,7 +499,3 @@ rag-stack: enabled: false # Default is true, we need false woodpecker: enabled: true # Default is false, we need true - - # Jarvis node scheduling - - # Performance overrides only for compute nodes \ No newline at end of file diff --git a/caipe/base/backstage/manifests/install.yaml b/caipe/base/backstage/manifests/install.yaml index b6cf75cf..1cec3124 100644 --- a/caipe/base/backstage/manifests/install.yaml +++ b/caipe/base/backstage/manifests/install.yaml @@ -178,8 +178,13 @@ data: entityFilename: catalog-info.yaml pullRequestBranchName: backstage-integration rules: - - allow: [Component, System, API, Resource, Location, Template] + - allow: [Component, System, API, Resource, Location, Template, User, Group, Domain] locations: + # Mock catalog data for testing MCP tools + - type: file + target: /app/mock-catalog/all.yaml + rules: + - allow: [Component, System, API, Resource, Location, Template, User, Group, Domain] # Examples from a public GitHub repository. - type: url target: https://cnoe.localtest.me/gitea/giteaAdmin/idpbuilder-localdev-backstage-templates-entities/raw/branch/main/catalog-info.yaml @@ -207,19 +212,21 @@ data: botName: CAIPE infoPage: https://cnoe-io.github.io/ai-platform-engineering/ showOptions: true - useOpenIDToken: true # Set to true to use OpenIdConnectApi.getIdToken(), false to use IdentityApi.getCredentials() - enableStreaming: false # Set to true to enable streaming responses, false for non-streaming + useOpenIDToken: false # use backstage auth instead of custom auth e.g. keycloak + enableStreaming: true # Set to true to enable streaming responses, false for non-streaming requestTimeout: 300 - headerTitle: CAIPE + autoReloadOnTokenExpiry: false + userEmailMode: "none" # Options: "none", "message" (prepends user email to every user message), "metadata" (adds user email to metadata) + headerTitle: CAIPE Distributed Multi-Agentic System headerSubtitle: AI Platform Engineer Assistant inputPlaceholder: Ask CAIPE anything... initialSuggestions: - - Create GitHub Repository - - Deploy ArgoCD Application - - Create AWS Resources - - Get LLM keys - - Add MyID Groups - - Invite users to Github Organization + - What can you do? + - What ArgoCD Applications are deployed? + - List all users and groups in Backstage + - Who is on the platform-team? + - Show me all teams in the catalog + - Get my Github profile details thinkingMessagesInterval: 7000 # Interval in milliseconds (default: 7000) thinkingMessages: - "⚙️ Processing query — even great thoughts start with a single thread." @@ -320,7 +327,7 @@ spec: name: argocd-credentials - secretRef: name: backstage-api-token - image: ghcr.io/sriaradhyula/backstage-app:35e30d165584e583b205c365d44cb938864896c7 + image: ghcr.io/suwhang-cisco/backstage-app:608f93ad3fc75cd3161d7163ba677e9ccc9caeff name: backstage ports: - containerPort: 7007 @@ -329,6 +336,9 @@ spec: - mountPath: /app/config name: backstage-config readOnly: true + - mountPath: /app/mock-catalog + name: mock-catalog + readOnly: true serviceAccountName: backstage volumes: - name: backstage-config @@ -344,6 +354,9 @@ spec: - key: k8s-config.yaml path: k8s-config.yaml name: k8s-config + - name: mock-catalog + configMap: + name: backstage-mock-catalog --- apiVersion: apps/v1 kind: StatefulSet diff --git a/caipe/base/backstage/manifests/mock-catalog.yaml b/caipe/base/backstage/manifests/mock-catalog.yaml new file mode 100644 index 00000000..a1653c13 --- /dev/null +++ b/caipe/base/backstage/manifests/mock-catalog.yaml @@ -0,0 +1,455 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backstage-mock-catalog + namespace: backstage + annotations: + argocd.argoproj.io/sync-wave: "-5" +data: + all.yaml: | + # ============================================================================= + # MOCK CATALOG DATA FOR BACKSTAGE + # Optimized for get_entities_by_query which filters for users and groups + # ============================================================================= + + # ----------------------------------------------------------------------------- + # GROUP: Platform Team + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: Group + metadata: + name: platform-team + namespace: default + description: Platform Engineering Team - maintains CAIPE, infrastructure, and developer tools + annotations: + backstage.io/techdocs-ref: dir:. + tags: + - platform + - infrastructure + - devops + links: + - url: https://github.com/cnoe-io + title: GitHub Organization + icon: github + spec: + type: team + profile: + displayName: Platform Team + email: platform-team@example.com + picture: https://api.dicebear.com/7.x/identicon/svg?seed=platform + children: [] + members: + - alice + - bob + - david + + --- + # ----------------------------------------------------------------------------- + # GROUP: Backend Team + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: Group + metadata: + name: backend-team + namespace: default + description: Backend Development Team - builds and maintains backend services and APIs + tags: + - backend + - api + - microservices + spec: + type: team + profile: + displayName: Backend Team + email: backend-team@example.com + picture: https://api.dicebear.com/7.x/identicon/svg?seed=backend + children: [] + members: + - charlie + - eve + + --- + # ----------------------------------------------------------------------------- + # GROUP: Frontend Team + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: Group + metadata: + name: frontend-team + namespace: default + description: Frontend Development Team - builds user interfaces and web applications + tags: + - frontend + - react + - ui + spec: + type: team + profile: + displayName: Frontend Team + email: frontend-team@example.com + picture: https://api.dicebear.com/7.x/identicon/svg?seed=frontend + children: [] + members: + - frank + - grace + + --- + # ----------------------------------------------------------------------------- + # GROUP: SRE Team + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: Group + metadata: + name: sre-team + namespace: default + description: Site Reliability Engineering Team - ensures system reliability and performance + tags: + - sre + - reliability + - oncall + spec: + type: team + profile: + displayName: SRE Team + email: sre-team@example.com + picture: https://api.dicebear.com/7.x/identicon/svg?seed=sre + children: [] + members: + - henry + - ivy + + --- + # ----------------------------------------------------------------------------- + # GROUP: Data Team + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: Group + metadata: + name: data-team + namespace: default + description: Data Engineering Team - manages data pipelines, analytics, and ML infrastructure + tags: + - data + - analytics + - ml + spec: + type: team + profile: + displayName: Data Team + email: data-team@example.com + picture: https://api.dicebear.com/7.x/identicon/svg?seed=data + children: [] + members: + - jack + - kate + + --- + # ----------------------------------------------------------------------------- + # GROUP: Security Team + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: Group + metadata: + name: security-team + namespace: default + description: Security Team - handles security reviews, compliance, and incident response + tags: + - security + - compliance + - infosec + spec: + type: team + profile: + displayName: Security Team + email: security-team@example.com + picture: https://api.dicebear.com/7.x/identicon/svg?seed=security + children: [] + members: + - leo + - mary + + --- + # ----------------------------------------------------------------------------- + # USER: Alice (Platform Engineer - Team Lead) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: alice + namespace: default + description: Senior Platform Engineer and Team Lead + tags: + - lead + - kubernetes + - terraform + spec: + profile: + displayName: Alice Anderson + email: alice.anderson@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=alice + memberOf: + - platform-team + + --- + # ----------------------------------------------------------------------------- + # USER: Bob (Platform Engineer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: bob + namespace: default + description: Platform Engineer specializing in CI/CD and GitOps + tags: + - argocd + - github-actions + - gitops + spec: + profile: + displayName: Bob Builder + email: bob.builder@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=bob + memberOf: + - platform-team + + --- + # ----------------------------------------------------------------------------- + # USER: Charlie (Backend Developer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: charlie + namespace: default + description: Senior Backend Developer - Java and Go specialist + tags: + - java + - golang + - microservices + spec: + profile: + displayName: Charlie Chen + email: charlie.chen@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=charlie + memberOf: + - backend-team + + --- + # ----------------------------------------------------------------------------- + # USER: David (Platform Engineer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: david + namespace: default + description: Platform Engineer focused on observability and monitoring + tags: + - prometheus + - grafana + - observability + spec: + profile: + displayName: David Davis + email: david.davis@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=david + memberOf: + - platform-team + + --- + # ----------------------------------------------------------------------------- + # USER: Eve (Backend Developer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: eve + namespace: default + description: Backend Developer specializing in Python and FastAPI + tags: + - python + - fastapi + - databases + spec: + profile: + displayName: Eve Evans + email: eve.evans@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=eve + memberOf: + - backend-team + + --- + # ----------------------------------------------------------------------------- + # USER: Frank (Frontend Developer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: frank + namespace: default + description: Senior Frontend Developer - React and TypeScript expert + tags: + - react + - typescript + - nextjs + spec: + profile: + displayName: Frank Foster + email: frank.foster@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=frank + memberOf: + - frontend-team + + --- + # ----------------------------------------------------------------------------- + # USER: Grace (Frontend Developer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: grace + namespace: default + description: Frontend Developer focused on accessibility and UX + tags: + - accessibility + - ux + - css + spec: + profile: + displayName: Grace Garcia + email: grace.garcia@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=grace + memberOf: + - frontend-team + + --- + # ----------------------------------------------------------------------------- + # USER: Henry (SRE) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: henry + namespace: default + description: Site Reliability Engineer - incident response lead + tags: + - oncall + - incident-response + - linux + spec: + profile: + displayName: Henry Hall + email: henry.hall@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=henry + memberOf: + - sre-team + + --- + # ----------------------------------------------------------------------------- + # USER: Ivy (SRE) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: ivy + namespace: default + description: Site Reliability Engineer - automation specialist + tags: + - automation + - ansible + - chaos-engineering + spec: + profile: + displayName: Ivy Irving + email: ivy.irving@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=ivy + memberOf: + - sre-team + + --- + # ----------------------------------------------------------------------------- + # USER: Jack (Data Engineer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: jack + namespace: default + description: Senior Data Engineer - Spark and Kafka expert + tags: + - spark + - kafka + - airflow + spec: + profile: + displayName: Jack Johnson + email: jack.johnson@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=jack + memberOf: + - data-team + + --- + # ----------------------------------------------------------------------------- + # USER: Kate (ML Engineer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: kate + namespace: default + description: ML Engineer - MLOps and model deployment + tags: + - mlops + - pytorch + - kubeflow + spec: + profile: + displayName: Kate Kim + email: kate.kim@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=kate + memberOf: + - data-team + + --- + # ----------------------------------------------------------------------------- + # USER: Leo (Security Engineer) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: leo + namespace: default + description: Security Engineer - AppSec and penetration testing + tags: + - appsec + - pentesting + - devsecops + spec: + profile: + displayName: Leo Lee + email: leo.lee@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=leo + memberOf: + - security-team + + --- + # ----------------------------------------------------------------------------- + # USER: Mary (Security Analyst) + # ----------------------------------------------------------------------------- + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: mary + namespace: default + description: Security Analyst - compliance and risk management + tags: + - compliance + - soc2 + - risk-management + spec: + profile: + displayName: Mary Miller + email: mary.miller@example.com + picture: https://api.dicebear.com/7.x/avataaars/svg?seed=mary + memberOf: + - security-team diff --git a/caipe/base/vault/manifests/argocd-token-cronjob.yaml b/caipe/base/vault/manifests/argocd-token-cronjob.yaml index b7329874..5230c285 100644 --- a/caipe/base/vault/manifests/argocd-token-cronjob.yaml +++ b/caipe/base/vault/manifests/argocd-token-cronjob.yaml @@ -105,6 +105,11 @@ spec: ARGOCD_API_URL="http://argocd-server.argocd.svc.cluster.local" \ ARGOCD_VERIFY_SSL="false" echo "ArgoCD API token created and stored in Vault" + + # Force ExternalSecret to refresh immediately + echo "Triggering ExternalSecret refresh..." + kubectl annotate externalsecret agent-argocd-secret -n ai-platform-engineering \ + force-sync=$(date +%s) --overwrite 2>/dev/null || echo "ExternalSecret annotation skipped (may not exist yet)" else echo "Failed to create ArgoCD token" exit 1