[Kafka] Change Kafka installation - multi namespace support

.gitignore

@@ -1,3 +1,5 @@
 .idea/*
 charts/mlrun-ce/charts/*
 .DS_Store
+**/.DS_Store
+*.DS_Store

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.10.0-rc5
+version: 0.10.0-rc6
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/requirements.lock

@@ -17,8 +17,8 @@ dependencies:
 - name: kube-prometheus-stack
   repository: https://prometheus-community.github.io/helm-charts
   version: 72.1.1
-- name: kafka
-  repository: https://charts.bitnami.com/bitnami
-  version: 31.3.1
-digest: sha256:d92e2702f26b3fbbe527fd4439cec8ce50bc79ad54fc69e10c28301e04e0114a
-generated: "2025-11-04T09:39:37.92185Z"
+- name: strimzi-kafka-operator
+  repository: https://strimzi.io/charts/
+  version: 0.48.0
+digest: sha256:f45be2a1208958d753b2e8a95f33eee17718ad1e691317ec0b50e3c088a7cae8
+generated: "2025-11-04T15:56:02.250773+02:00"

charts/mlrun-ce/requirements.yaml

@@ -21,7 +21,7 @@ dependencies:
     repository: "https://prometheus-community.github.io/helm-charts"
     version: "72.1.1"
     condition: kube-prometheus-stack.enabled
-  - name: kafka
-    repository: "https://charts.bitnami.com/bitnami"
-    version: "31.3.1"
-    condition: kafka.enabled
+  - name: strimzi-kafka-operator
+    repository: "https://strimzi.io/charts/"
+    version: "0.48.0"
+    condition: strimzi-kafka-operator.enabled

charts/mlrun-ce/templates/kafka/kafka-bootstrap-alias.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.kafka.bootstrapAlias.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.kafka.bootstrapAlias.name | default "kafka-stream" }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kafka
+    app.kubernetes.io/component: bootstrap-alias
+    {{- include "mlrun-ce.common.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+  - name: client
+    port: 9092
+    targetPort: 9092
+    protocol: TCP
+  selector:
+    strimzi.io/cluster: {{ .Values.kafka.name | default "kafka-stream" }}
+    strimzi.io/kind: Kafka
+    strimzi.io/name: {{ .Values.kafka.name | default "kafka-stream" }}-kafka
+{{- end }}
+

charts/mlrun-ce/templates/kafka/kafka-cluster.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.kafka.enabled }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ .Values.kafka.name }}
+  namespace: {{ .Values.kafka.namespace | default .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kafka
+    app.kubernetes.io/component: cluster
+    {{- include "mlrun-ce.common.labels" . | nindent 4 }}
+spec:
+  kafka:
+    listeners:
+    {{- range .Values.kafka.listeners }}
+      - name: {{ .name }}
+        port: {{ .port }}
+        type: {{ .type }}
+        tls: {{ .tls }}
+    {{- end }}
+    config:
+    {{- range $key, $value := .Values.kafka.config }}
+      {{ $key }}: {{ $value | quote }}
+    {{- end }}
+  {{- if gt (.Values.kafka.zookeeper.replicas | int) 0 }}
+  zookeeper:
+    replicas: {{ .Values.kafka.zookeeper.replicas }}
+    storage:
+      type: persistent-claim
+      size: 8Gi
+  {{- end }}
+{{- end }}

charts/mlrun-ce/templates/kafka/kafka-network-policy.yaml

@@ -0,0 +1,64 @@
+{{- if .Values.kafka.rbac.enabled -}}
+{{- $operatorNamespace := .Values.kafka.rbac.operatorNamespace | default "controller" -}}
+{{- $kafkaName := .Values.kafka.name | default "kafka-stream" -}}
+{{- $currentNamespace := .Release.Namespace -}}
+---
+# NetworkPolicy: Allow egress from this namespace to Kafka namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-kafka-access
+  namespace: {{ $currentNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+spec:
+  # Apply to all pods in this namespace
+  podSelector: {}
+
+  policyTypes:
+  - Egress
+
+  egress:
+  # Allow egress to Kafka namespace
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: {{ $operatorNamespace }}
+      podSelector:
+        matchLabels:
+          strimzi.io/cluster: {{ $kafkaName }}
+    ports:
+    - protocol: TCP
+      port: 9092  # client listener
+    - protocol: TCP
+      port: 9093  # controller listener
+    - protocol: TCP
+      port: 9094  # internal listener
+
+  # Allow DNS resolution (required for service discovery)
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: kube-system
+      podSelector:
+        matchLabels:
+          k8s-app: kube-dns
+    ports:
+    - protocol: UDP
+      port: 53
+    - protocol: TCP
+      port: 53
+
+  # Allow egress to the internet/other services (optional)
+  # Comment out the next section if you want to restrict to Kafka only
+  - to:
+    - namespaceSelector: {}
+    - podSelector: {}
+
+  # Allow egress within same namespace
+  - to:
+    - podSelector: {}
+{{- end }}
+

charts/mlrun-ce/templates/kafka/kafka-nodepool.yaml

@@ -0,0 +1,30 @@
+{{- if .Values.kafka.enabled }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: {{ .Values.kafka.name }}-pool
+  namespace: {{ .Values.kafka.namespace | default .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kafka
+    app.kubernetes.io/component: nodepool
+    strimzi.io/cluster: {{ .Values.kafka.name }}
+    {{- include "mlrun-ce.common.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.kafka.replicas }}
+  roles:
+    - controller
+    - broker
+  storage:
+    type: {{ .Values.kafka.storage.type }}
+    size: {{ .Values.kafka.storage.size }}
+    {{- if .Values.kafka.storage.class }}
+    class: {{ .Values.kafka.storage.class }}
+    {{- end }}
+  resources:
+    requests:
+      memory: {{ .Values.kafka.resources.requests.memory }}
+      cpu: {{ .Values.kafka.resources.requests.cpu }}
+    limits:
+      memory: {{ .Values.kafka.resources.limits.memory }}
+      cpu: {{ .Values.kafka.resources.limits.cpu }}
+{{- end }}

charts/mlrun-ce/templates/kafka/kafka-rbac.yaml

@@ -0,0 +1,90 @@
+{{- if .Values.kafka.rbac.enabled -}}
+{{- $operatorNamespace := .Values.kafka.rbac.operatorNamespace | default "controller" -}}
+{{- $kafkaName := .Values.kafka.name | default "kafka-stream" -}}
+{{- $currentNamespace := .Release.Namespace -}}
+---
+# ServiceAccount for Kafka client applications
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kafka-client
+  namespace: {{ $currentNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+---
+# Role: Allow managing Kafka resources via CRDs in the operator namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $currentNamespace }}-kafka-resource-manager
+  namespace: {{ $operatorNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+    user-namespace: {{ $currentNamespace }}
+rules:
+  # Allow creating and managing KafkaTopic CRDs
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkatopics
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  # Allow checking KafkaTopic status
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkatopics/status
+    verbs:
+      - get
+      - list
+      - watch
+  # Allow reading KafkaUser CRDs (if using SCRAM auth)
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkausers
+    verbs:
+      - get
+      - list
+      - watch
+  # Allow reading the Kafka cluster info
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkas
+    verbs:
+      - get
+      - list
+      - watch
+---
+# RoleBinding: Grant Kafka resource management permissions to ServiceAccount
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $currentNamespace }}-kafka-resource-manager
+  namespace: {{ $operatorNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+    user-namespace: {{ $currentNamespace }}
+subjects:
+  - kind: ServiceAccount
+    name: kafka-client
+    namespace: {{ $currentNamespace }}
+roleRef:
+  kind: Role
+  name: {{ $currentNamespace }}-kafka-resource-manager
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
+