diff --git a/.github/workflows/kubectl_ai_build.yaml b/.github/workflows/kubectl_ai_build.yaml new file mode 100644 index 000000000..0a695e4e0 --- /dev/null +++ b/.github/workflows/kubectl_ai_build.yaml @@ -0,0 +1,23 @@ +name: Kubeclt-ai Image Build on Push + +on: + workflow_dispatch: + inputs: + tag: + description: "The tag to apply to this build. Defaults to 'latest'." + default: "latest" + required: false + push: + paths: + - gen3-integration-tests/gen3_ci/kubectl_ai/** + +jobs: + AmazonLinuxBase2023: + name: Build Helm Cleanup Image + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + DOCKERFILE_LOCATION: "./gen3-integration-tests/gen3_ci/kubectl_ai/Dockerfile" + OVERRIDE_REPO_NAME: "kubectl_ai" + OVERRIDE_TAG_NAME: "$(echo ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.tag || format('{0}', github.head_ref || github.ref_name) }} | tr / _)" + USE_QUAY_ONLY: true + secrets: inherit diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/Chart.yaml b/gen3-integration-tests/gen3_ci/kubectl_ai/Chart.yaml new file mode 100644 index 000000000..b9b0459e5 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: kubectl-ai +description: A Helm chart for kubeclt-ai + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "master" diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/Dockerfile b/gen3-integration-tests/gen3_ci/kubectl_ai/Dockerfile new file mode 100644 index 000000000..ba208f4b8 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/Dockerfile @@ -0,0 +1,25 @@ +ARG AZLINUX_BASE_VERSION=master + +FROM quay.io/cdis/amazonlinux-base:${AZLINUX_BASE_VERSION} AS base + +# Install curl +RUN dnf install -y --allowerasing \ + curl \ + findutils \ + zstd \ + tar \ + gzip + +# Create kubectl directory +WORKDIR /kubectl + +# Install kubectl-ai +RUN curl -fsSL https://raw.githubusercontent.com/GoogleCloudPlatform/kubectl-ai/main/install.sh -o install.sh \ + && sed -i 's/sudo //g' install.sh \ + && bash install.sh + +# Install Kubectl +RUN curl -LO https://dl.k8s.io/release/v1.33.0/bin/linux/amd64/kubectl && \ + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + +CMD ["sleep", "infinity"] diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/templates/_helpers.tpl b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/_helpers.tpl new file mode 100644 index 000000000..a152c3c20 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{/* +Return the fully qualified app name +*/}} +{{- define "kubectl-ai.fullname" -}} +{{- printf "%s" .Release.Name -}} +{{- end }} diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/templates/deployment.yaml b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/deployment.yaml new file mode 100644 index 000000000..ba6ac3e99 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubectl-ai +spec: + replicas: 1 + selector: + matchLabels: + app: kubectl-ai + template: + metadata: + labels: + app: kubectl-ai + spec: + serviceAccountName: kubectl-access-kubectl-ai + containers: + - name: kubectl-ai + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + ports: + - containerPort: 80 + env: + - name: OLLAMA_KEEP_ALIVE + value: "-1" + - name: KUBECTL_AI_TIMEOUT + value: "300" diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/templates/kubectl-ai-pdb.yaml b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/kubectl-ai-pdb.yaml new file mode 100644 index 000000000..5f66810e8 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/kubectl-ai-pdb.yaml @@ -0,0 +1,11 @@ +{{- if .Values.pdb }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kubectl-ai-pdb +spec: + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: "kubectl-ai" +{{- end }} diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/templates/service-account.yaml b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/service-account.yaml new file mode 100644 index 000000000..6cee55888 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/service-account.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubectl-access-kubectl-ai + namespace: {{ .Release.Namespace }} + annotations: + eks.amazonaws.com/role-arn: {{ .Values.serviceAccount.iamRoleArn }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubectl-access-role-kubectl-ai + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["pods", "pods/exec", "pods/log", "configmaps", "namespaces", "labels"] + verbs: ["get", "list", "create", "patch"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get", "list", "create", "delete", "watch", "patch"] + - apiGroups: ["apps"] + resources: [deployments, deployments/scale] + verbs: ["get", "list", "create", "delete", "watch", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubectl-access-binding-kubectl-ai + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubectl-access-role-kubectl-ai +subjects: + - kind: ServiceAccount + name: kubectl-access-kubectl-ai + namespace: {{ .Release.Namespace }} diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/templates/service.yaml b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/service.yaml new file mode 100644 index 000000000..19420ece8 --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/templates/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubectl-ai +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + selector: + app: kubectl-ai diff --git a/gen3-integration-tests/gen3_ci/kubectl_ai/values.yaml b/gen3-integration-tests/gen3_ci/kubectl_ai/values.yaml new file mode 100644 index 000000000..6879da69e --- /dev/null +++ b/gen3-integration-tests/gen3_ci/kubectl_ai/values.yaml @@ -0,0 +1,23 @@ +image: + repository: quay.io/cdis/kubectl_ai + tag: chore_add_kubectl_ai + pullPolicy: Always + +pdb: true + +resources: + requests: + cpu: 2 + memory: "2Gi" + limits: + cpu: 4 + memory: "4Gi" + +service: + type: ClusterIP + port: 80 + +# Service account for accessing S3 bedrock access +serviceAccount: + name: s3-bedrock-access + iamRoleArn: arn:aws:iam::707767160287:role/kubectl-ai-irsa diff --git a/gen3-integration-tests/gen3_ci/ollama/Dockerfile b/gen3-integration-tests/gen3_ci/ollama/Dockerfile index c595b47e5..107d86d38 100644 --- a/gen3-integration-tests/gen3_ci/ollama/Dockerfile +++ b/gen3-integration-tests/gen3_ci/ollama/Dockerfile @@ -7,7 +7,8 @@ RUN dnf install -y --allowerasing \ curl \ findutils \ zstd \ - tar + tar \ + gzip # Create model directory WORKDIR /models @@ -22,6 +23,7 @@ RUN bash -c "\ ollama pull gemma4:e4b \ " + # List ollama model RUN bash -c "\ ollama serve & \ diff --git a/gen3-integration-tests/gen3_ci/ollama/templates/deployment.yaml b/gen3-integration-tests/gen3_ci/ollama/templates/deployment.yaml index 2a5b472fe..d043d34e0 100644 --- a/gen3-integration-tests/gen3_ci/ollama/templates/deployment.yaml +++ b/gen3-integration-tests/gen3_ci/ollama/templates/deployment.yaml @@ -12,6 +12,7 @@ spec: labels: app: ollama spec: + serviceAccountName: kubectl-access-ollama containers: - name: ollama image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -25,3 +26,7 @@ spec: value: "debug" - name: OLLAMA_HOST value: "0.0.0.0:11434" + - name: OLLAMA_KEEP_ALIVE + value: "-1" + - name: KUBECTL_AI_TIMEOUT + value: "300" diff --git a/gen3-integration-tests/gen3_ci/ollama/values.yaml b/gen3-integration-tests/gen3_ci/ollama/values.yaml index 6e316036d..68c18a370 100644 --- a/gen3-integration-tests/gen3_ci/ollama/values.yaml +++ b/gen3-integration-tests/gen3_ci/ollama/values.yaml @@ -1,17 +1,17 @@ image: repository: quay.io/cdis/ollama tag: master - pullPolicy: IfNotPresent + pullPolicy: Always pdb: true resources: requests: cpu: 4 - memory: "10Gi" + memory: "8Gi" limits: cpu: 6 - memory: "12Gi" + memory: "10Gi" service: type: ClusterIP diff --git a/gen3-integration-tests/gen3_ci/scripts/analyze_failed_tests_using_ai_agent.py b/gen3-integration-tests/gen3_ci/scripts/analyze_failed_tests_using_ai_agent.py index a806abe67..d98ee32ac 100644 --- a/gen3-integration-tests/gen3_ci/scripts/analyze_failed_tests_using_ai_agent.py +++ b/gen3-integration-tests/gen3_ci/scripts/analyze_failed_tests_using_ai_agent.py @@ -13,12 +13,12 @@ load_dotenv() -def setup_ollama_helm_chart(): +def setup_helm_chart(service): cmd = [ "helm", "install", - "ollama", - "gen3_ci/ollama", + service, + f"gen3_ci/{service}", "-n", os.getenv("NAMESPACE"), ] @@ -31,7 +31,7 @@ def setup_ollama_helm_chart(): ) if not helm_install_result.returncode == 0: raise Exception( - f"Unable to install ollama. Error: {helm_install_result.stderr.strip()}" + f"Unable to install {service}. Error: {helm_install_result.stderr.strip()}" ) cmd = [ @@ -40,21 +40,21 @@ def setup_ollama_helm_chart(): "--for=condition=ready", "pod", "-l", - "app=ollama", + f"app={service}", "--timeout=10m", "-n", os.getenv("NAMESPACE"), ] - ollama_pod_ready_result = subprocess.run( + service_pod_ready_result = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) - if not ollama_pod_ready_result.returncode == 0: + if not service_pod_ready_result.returncode == 0: raise Exception( - f"Ollama pod hasn't started yet. Error: {ollama_pod_ready_result.stderr.strip()}" + f"{service} pod hasn't started yet. Error: {service_pod_ready_result.stderr.strip()}" ) @@ -69,11 +69,11 @@ def wait_for_port(host="localhost", port=11434, timeout=60): raise TimeoutError("Port-forward did not become ready") -def setup_port_forwarding(): +def setup_port_forwarding(service): cmd = [ "kubectl", "port-forward", - "svc/ollama", + f"svc/{service}", "11434:11434", "-n", os.getenv("NAMESPACE"), @@ -88,11 +88,11 @@ def setup_port_forwarding(): return process -def uninstall_ollama_helm_chart(): +def uninstall_helm_chart(service): cmd = [ "helm", "uninstall", - "ollama", + service, "-n", os.getenv("NAMESPACE"), ] @@ -105,7 +105,7 @@ def uninstall_ollama_helm_chart(): ) if not helm_install_result.returncode == 0: raise Exception( - f"Unable to uninstall ollama. Error: {helm_install_result.stderr.strip()}" + f"Unable to uninstall {service}. Error: {helm_install_result.stderr.strip()}" ) @@ -149,7 +149,7 @@ def analyze_env_setup_failure() -> str: {"role": "system", "content": debug_prompt}, {"role": "user", "content": "analyse the errors from this logfile"}, ] - payload = {"model": "gemma4:e4b", "messages": messages, "temperature": 0} + payload = {"model": "qwen3.5:2b", "messages": messages, "temperature": 0} headers = {"Content-Type": "application/json"} url = "http://localhost:11434/v1/chat/completions" response = requests.post(url, json=payload, headers=headers) @@ -158,6 +158,61 @@ def analyze_env_setup_failure() -> str: return response.content +def analyze_env_setup_failure_using_kubectl_ai() -> str: + cmd = [ + "kubectl", + "-n", + os.getenv("NAMESPACE"), + "get", + "pods", + "-l", + "app=kubectl-ai", + "-o", + "jsonpath='{.items[0].metadata.name}'", + ] + kubeclt_ai_pod_result = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=600, + ) + if not kubeclt_ai_pod_result.returncode == 0: + raise Exception( + f"Failed to get kubectl-ai pod. Error: {kubeclt_ai_pod_result.stderr.strip()}" + ) + kubeclt_ai_pod_name = kubeclt_ai_pod_result.stdout.strip().replace("'", "") + kubectl_ai_cmd = [ + "kubectl", + "-n", + os.getenv("NAMESPACE"), + "exec", + kubeclt_ai_pod_name, + "--", + "kubectl-ai", + "--llm-provider", + "ollama", + "--model", + "qwen3.5:2b", + "--skip-permissions", + f'Check if any pods are not healthy on {os.getenv("NAMESPACE")} namespace and anaylyze the logs', + ] + logger.info(kubectl_ai_cmd) + kubectl_ai_result = subprocess.run( + kubectl_ai_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=600, + ) + if not kubectl_ai_result.returncode == 0: + raise Exception( + f"kubectl-ai command output. Error: {kubectl_ai_result.stdout.strip()}" + f"kubectl-ai command failed. Error: {kubectl_ai_result.stderr.strip()}" + ) + return kubectl_ai_result.stdout.strip() + + def analyze_failed_tests() -> str: """Analyze the failed tests and provide fixes""" if Path("rerun-allure-report").exists(): @@ -205,7 +260,7 @@ def analyze_failed_tests() -> str: {"role": "system", "content": debug_prompt}, {"role": "user", "content": "analyse the failed tests"}, ] - payload = {"model": "gemma4:e4b", "messages": messages, "temperature": 0} + payload = {"model": "qwen3.5:2b", "messages": messages, "temperature": 0} headers = {"Content-Type": "application/json"} url = "http://localhost:11434/v1/chat/completions" response = requests.post(url, json=payload, headers=headers) @@ -218,23 +273,39 @@ def analyze_failed_tests() -> str: def run_test_failure_analysis(): if os.getenv("PR_ERROR_MSG") == "Failed to Prepare CI environment": - response = analyze_env_setup_failure() + try: + setup_helm_chart(service="kubectl-ai") + process = setup_port_forwarding(service="kubectl-ai") + assert "qwen3.5:2b" in str(validate_ollama_model()) + # response = analyze_env_setup_failure() + response = analyze_env_setup_failure_using_kubectl_ai() + except Exception as e: + logger.info( + f"Failed to run analyze_env_setup_failure_using_kubectl_ai: {e}" + ) + finally: + uninstall_helm_chart(service="kubectl-ai") else: - response = analyze_failed_tests() + try: + setup_helm_chart(service="ollama") + process = setup_port_forwarding(service="ollama") + assert "gemma4:e4b" in str(validate_ollama_model()) + response = analyze_failed_tests() + except Exception as e: + logger.info(f"Failed to run analyze_failed_tests: {e}") + finally: + uninstall_helm_chart(service="ollama") if response is None: return "No logs found to analyze" data = json.loads(response.decode("utf-8")) reasoning = data["choices"][0]["message"].get("content") - return reasoning + return reasoning, process if __name__ == "__main__": process = None try: - setup_ollama_helm_chart() - process = setup_port_forwarding() - assert "gemma4:e4b" in str(validate_ollama_model()) - response = run_test_failure_analysis() + response, process = run_test_failure_analysis() with open("logs/failure_analysis.txt", "w") as f: f.write(response) except Exception as e: @@ -243,4 +314,3 @@ def run_test_failure_analysis(): if process and process.poll() is None: process.terminate() process.wait() - uninstall_ollama_helm_chart() diff --git a/gen3-integration-tests/gen3_ci/scripts/setup_ci_env.sh b/gen3-integration-tests/gen3_ci/scripts/setup_ci_env.sh index 21a4bba72..6aa169ce2 100755 --- a/gen3-integration-tests/gen3_ci/scripts/setup_ci_env.sh +++ b/gen3-integration-tests/gen3_ci/scripts/setup_ci_env.sh @@ -557,11 +557,14 @@ ci_es_indices_setup() { } wait_for_pods_ready() { - export timeout=1800 + export timeout=600 export interval=20 - end=$((SECONDS + timeout)) - while [ $SECONDS -lt $end ]; do + SECONDS = 0 + + end=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -lt "$end" ]; do + echo "Running at $(date)" # Get JSON for not-ready, non-terminating pods not_ready_json=$(kubectl get pods -l app!=gen3job -n "${namespace}" -o json | \ jq '[.items[]