From 2f7bf092d36ac60b36d82959604c2ad65250547a Mon Sep 17 00:00:00 2001 From: jdaln <150942337+jdaln@users.noreply.github.com> Date: Thu, 11 Dec 2025 15:06:37 +0100 Subject: [PATCH 1/6] feat: submariner deployment --- .gitignore | 2 +- README.md | 34 ++ argo-setup/README.md | 7 +- argo-setup/patches/kustomization.yaml | 2 +- ...cd-application-controller-clusterrole.yaml | 34 +- argo-setup/patches/patch-argocd-cm.yaml | 1 + base/argo-projects.yaml | 2 + .../mip-infrastructure.yaml | 1 + .../rbac/submariner-rbac.yaml | 383 ++++++++++++++++++ common/submariner/README.md | 97 +++++ .../submariner/broker/copy-secret-hook.yaml | 102 +++++ common/submariner/broker/kustomization.yaml | 32 ++ common/submariner/broker/values.yaml | 10 + common/submariner/kustomization.yaml | 18 + .../operator/gateway-loadbalancer-ip.yaml | 18 + common/submariner/operator/kustomization.yaml | 206 ++++++++++ common/submariner/operator/values.yaml | 66 +++ .../patches/gateway-loadbalancer-ip.yaml | 12 + common/submariner/submariner.yaml | 70 ++++ common/submariner/values-public-cluster.yaml | 3 + .../customizations/exareme2-kustomize.yaml | 27 ++ .../customizations/exareme2-values.yaml | 5 + .../customizations/mip-stack-kustomize.yaml | 39 ++ .../customizations/mip-stack-values.yaml | 5 + .../mip-infrastructure/federation-z.yaml | 23 ++ .../mip-infrastructure/kustomization.yaml | 24 ++ .../federation-Z/remote-node/README.md | 185 +++++++++ .../remote-node/setup-microk8s.sh | 115 ++++++ .../federation-Z/remote-node/setup-tools.sh | 49 +++ .../remote-node/submariner-values.yaml | 72 ++++ projects/mip-infrastructure.yaml | 7 +- projects/static/mip-common/mip-common.yaml | 15 +- .../mip-federations/mip-federations.yaml | 7 +- .../static/mip-security/mip-security.yaml | 7 +- .../mip-shared-apps/mip-shared-apps.yaml | 13 +- projects/static/submariner/kustomization.yaml | 5 + .../static/submariner/submariner-project.yaml | 64 +++ .../submariner/submariner-rbac-project.yaml | 30 ++ .../federation/templates/appproject.yaml | 6 +- 39 files changed, 1765 insertions(+), 33 deletions(-) create mode 100644 base/mip-infrastructure/rbac/submariner-rbac.yaml create mode 100644 common/submariner/README.md create mode 100644 common/submariner/broker/copy-secret-hook.yaml create mode 100644 common/submariner/broker/kustomization.yaml create mode 100644 common/submariner/broker/values.yaml create mode 100644 common/submariner/kustomization.yaml create mode 100644 common/submariner/operator/gateway-loadbalancer-ip.yaml create mode 100644 common/submariner/operator/kustomization.yaml create mode 100644 common/submariner/operator/values.yaml create mode 100644 common/submariner/patches/gateway-loadbalancer-ip.yaml create mode 100644 common/submariner/submariner.yaml create mode 100644 common/submariner/values-public-cluster.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/customizations/exareme2-kustomize.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/customizations/exareme2-values.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/customizations/mip-stack-kustomize.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/customizations/mip-stack-values.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/federation-z.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml create mode 100644 deployments/hybrid/federations/federation-Z/remote-node/README.md create mode 100644 deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh create mode 100644 deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh create mode 100644 deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml create mode 100644 projects/static/submariner/kustomization.yaml create mode 100644 projects/static/submariner/submariner-project.yaml create mode 100644 projects/static/submariner/submariner-rbac-project.yaml diff --git a/.gitignore b/.gitignore index 073a168..1bbe27b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ _*/ ### Helm ### # Chart dependencies -**/charts/*.tgz +charts/ ### JetBrains+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider diff --git a/README.md b/README.md index e5aa001..c9a197b 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,39 @@ graph TD - `argocd` CLI installed (optional but recommended) - SSH access to the repository configured in Argo CD +### 🔐 Remote Access (VPN/SSH) + +If you are not on the same network as the cluster, you need to tunnel your traffic. + +#### Option A: SOCKS Proxy (Recommended for `kubectl`) +This routes all your traffic through the jump host. + +1. **Open Tunnel**: + ```bash + ssh -D 1080 -C -q -N @ + ``` +2. **Configure Environment**: + ```bash + export HTTPS_PROXY=socks5://127.0.0.1:1080 + ``` + +#### Option B: Port Forward + Hosts (Recommended for `argocd` CLI / UI) +This tricks your local machine into thinking `localhost` is the remote server, while preserving the hostname for Ingress routing. + +1. **Update `/etc/hosts`**: + ```bash + # Add this line + 127.0.0.1 argocd.mip-tds.chuv.cscs.ch + ``` +2. **Open Tunnel (Sudo required for port 443)**: + ```bash + sudo ssh -L 443:argocd.mip-tds.chuv.cscs.ch:443 @ + ``` +3. **Login**: + ```bash + argocd login argocd.mip-tds.chuv.cscs.ch:443 --insecure --grpc-web + ``` + ### Initial secrets: The following secrets must exist in the cluster before or after running this repository's setup scripts. If you run it after, creation will hang until these are present. @@ -288,6 +321,7 @@ argocd repo add git@github.com:NeuroTech-Platform/mip-deployments.git \ --name mip-infra kubectl apply -f base/mip-infrastructure/rbac/nginx-public-rbac.yaml +kubectl apply -f base/mip-infrastructure/rbac/submariner-rbac.yaml ``` diff --git a/argo-setup/README.md b/argo-setup/README.md index 7f3b4d4..7813b5c 100644 --- a/argo-setup/README.md +++ b/argo-setup/README.md @@ -44,9 +44,9 @@ ARGOCD_HOST=argocd.example.com #YOUR SUBDOMAIN HERE # Replace placeholder hostname in all patch files cd argo-setup # BSD-Style -find . -type f -exec sed -i '' "s/argocd.example.com/$ARGOCD_HOST/g" {} + +LC_ALL=C find . -type f -not -path '*/.git/*' -exec sed -i '' "s/argocd.example.com/$ARGOCD_HOST/g" {} + # GNU-Style -find . -type f -exec sed -i "s/argocd.example.com/$ARGOCD_HOST/g" {} + +LC_ALL=C find . -type f -not -path '*/.git/*' -exec sed -i "s/argocd.example.com/$ARGOCD_HOST/g" {} + # Resolve latest 3.0 version (or pin to specific version) export ARGOCD_SERIES=v3.0 @@ -58,6 +58,9 @@ export ARGOCD_VER=$(curl -s https://api.github.com/repos/argoproj/argo-cd/releas echo "Resolved latest 3.0 tag: $ARGOCD_VER" # Update kustomization.yaml with resolved version +# BSD-Style +sed -i '' "s|/v[0-9.]*/manifests/ha/install.yaml|/${ARGOCD_VER}/manifests/ha/install.yaml|g" patches/kustomization.yaml +# GNU-Style sed -i "s|/v[0-9.]*/manifests/ha/install.yaml|/${ARGOCD_VER}/manifests/ha/install.yaml|g" patches/kustomization.yaml # 1. Namespace diff --git a/argo-setup/patches/kustomization.yaml b/argo-setup/patches/kustomization.yaml index ffba647..46d0a21 100644 --- a/argo-setup/patches/kustomization.yaml +++ b/argo-setup/patches/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization namespace: argocd-mip-team # Base: upstream HA manifest (resolved tag) resources: - - https://raw.githubusercontent.com/argoproj/argo-cd/v3.0.11/manifests/ha/install.yaml + - https://raw.githubusercontent.com/argoproj/argo-cd/v3.0.20/manifests/ha/install.yaml - patch-argocd-ingress.yaml patchesStrategicMerge: - patch-argocd-application-controller-clusterrole.yaml diff --git a/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml b/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml index b8f188b..bbe9c3d 100644 --- a/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml +++ b/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml @@ -36,15 +36,9 @@ rules: - apiGroups: [monitoring.coreos.com] resources: [prometheusrules, servicemonitors] verbs: [create, delete, patch, update] - - apiGroups: [cert-manager.io] - resources: [clusterissuers] - verbs: [create, delete, patch, update] - apiGroups: [networking.k8s.io] resources: [ingresses, networkpolicies, ingressclasses] verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: [rbac.authorization.k8s.io] - resources: [clusterrolebindings, clusterroles, rolebindings, roles] - verbs: [create, delete, patch, update] - apiGroups: [''] resources: - configmaps @@ -61,3 +55,31 @@ rules: - apiGroups: [''] resources: [pods] verbs: [create, delete, patch, update] # in the future, only delete but at the moment we still have standalone pods + + # Rule 4: Submariner requirements + # The controller needs these permissions to grant them to Submariner components + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, list, watch, update] + - apiGroups: [config.openshift.io] + resources: [networks] + verbs: [get, list] + - apiGroups: [projectcalico.org] + resources: [ippools] + verbs: [create, delete, update, deletecollection] + - apiGroups: [submariner.io] + resources: [clusters, endpoints, gateways, clusterglobalegressips, globalegressips, globalingressips, submariners] + verbs: [create, delete, deletecollection, get, list, patch, update, watch] + - apiGroups: [submariner.io] + resources: [clusterglobalegressips/status, globalegressips/status, globalingressips/status] + verbs: [create, delete, deletecollection, update] + - apiGroups: [network.openshift.io] + resources: [service/externalips] + verbs: [create, delete] + - apiGroups: [''] + resources: [nodes] + verbs: [get, list, watch, update] +# Add if we ever use GlobalNet +# - apiGroups: [''] +# resources: [endpoints] +# verbs: [get, list, watch, create, update, delete, patch] diff --git a/argo-setup/patches/patch-argocd-cm.yaml b/argo-setup/patches/patch-argocd-cm.yaml index 242b041..302b323 100644 --- a/argo-setup/patches/patch-argocd-cm.yaml +++ b/argo-setup/patches/patch-argocd-cm.yaml @@ -12,3 +12,4 @@ data: application.instanceLabelKey: argocd.argoproj.io/instanceTracking installationID: mip-team-argo-cd resource.respectRBAC: normal + kustomize.buildOptions: --enable-helm diff --git a/base/argo-projects.yaml b/base/argo-projects.yaml index 6245b1f..c8d41e3 100644 --- a/base/argo-projects.yaml +++ b/base/argo-projects.yaml @@ -23,6 +23,8 @@ spec: fileName: mip-common - projectName: mip-argo-project-security fileName: mip-security + - projectName: mip-argo-project-submariner + fileName: submariner goTemplate: true template: metadata: diff --git a/base/mip-infrastructure/mip-infrastructure.yaml b/base/mip-infrastructure/mip-infrastructure.yaml index 1a1f212..bbcff1b 100644 --- a/base/mip-infrastructure/mip-infrastructure.yaml +++ b/base/mip-infrastructure/mip-infrastructure.yaml @@ -36,6 +36,7 @@ spec: - path: common/datacatalog - path: common/nginx-ingress - path: common/security + - path: common/submariner - list: elements: - cluster: https://kubernetes.default.svc diff --git a/base/mip-infrastructure/rbac/submariner-rbac.yaml b/base/mip-infrastructure/rbac/submariner-rbac.yaml new file mode 100644 index 0000000..f73f1b7 --- /dev/null +++ b/base/mip-infrastructure/rbac/submariner-rbac.yaml @@ -0,0 +1,383 @@ +--- +# Namespaces required by Submariner components and RBACs below +apiVersion: v1 +kind: Namespace +metadata: + name: submariner-operator +--- +apiVersion: v1 +kind: Namespace +metadata: + name: submariner-k8s-broker +--- +# Extracted from submariner-operator Helm chart +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-operator +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "create", "update", "delete", "watch"] + - apiGroups: [""] + resources: ["pods", "services", "nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["operator.openshift.io"] + resources: ["dnses"] + verbs: ["get", "update"] + - apiGroups: ["config.openshift.io"] + resources: ["networks"] + resourceNames: ["cluster"] + verbs: ["get"] + - apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["get", "create"] + - apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["list"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "get", "list", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-operator +subjects: + - kind: ServiceAccount + name: submariner-operator + namespace: submariner-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-operator +--- +# Extracted from submariner-k8s-broker (submariner) Helm chart +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-gateway +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "create", "update", "delete"] + - apiGroups: [""] + resources: ["pods", "services", "nodes"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["operator.openshift.io"] + resources: ["dnses"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["config.openshift.io"] + resources: ["networks"] + verbs: ["get", "list"] + - apiGroups: ["submariner.io"] + resources: ["endpoints", "gateways", "clusters"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] + - apiGroups: ["submariner.io"] + resources: ["gateways/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "get", "list", "update"] +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-routeagent +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "create", "update", "delete"] + - apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list", "watch"] + - apiGroups: ["operator.openshift.io"] + resources: ["dnses"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["config.openshift.io"] + resources: ["networks"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-routeagent +subjects: + - kind: ServiceAccount + name: submariner-routeagent + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-routeagent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-globalnet +subjects: + - kind: ServiceAccount + name: submariner-globalnet + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-globalnet +--- +# Namespaced Roles/Bindings (Extracted to prevent ArgoCD sync errors) +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: submariner-broker-submariner-k8s-broker-cluster + namespace: submariner-k8s-broker +rules: + - apiGroups: ["submariner.io"] + resources: ["clusters", "endpoints"] + verbs: ["create", "get", "list", "watch", "patch", "update", "delete"] + - apiGroups: ["submariner.io"] + resources: ["brokers"] + verbs: ["get", "list"] + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceimports", "serviceimports/status"] + verbs: ["create", "get", "list", "watch", "patch", "update", "delete"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices", "endpointslices/restricted"] + verbs: ["create", "get", "list", "watch", "patch", "update", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: submariner-broker-submariner-k8s-broker-cluster + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: submariner-broker-submariner-k8s-broker-cluster +subjects: + - kind: ServiceAccount + name: submariner-broker-submariner-k8s-broker-client + namespace: submariner-k8s-broker +--- +# Operator ClusterRoles and ClusterRoleBindings + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-gateway +subjects: + - kind: ServiceAccount + name: submariner-gateway + namespace: submariner-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-lighthouse-agent +rules: + - apiGroups: [""] + resources: ["services", "namespaces", "endpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices", "endpointslices/restricted"] + verbs: ["create", "get", "list", "watch", "update", "delete", "deletecollection"] + - apiGroups: ["submariner.io"] + resources: ["gateways", "globalingressips"] + verbs: ["get", "list", "watch"] + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceimports", "serviceimports/status"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceexports"] + verbs: ["get", "list", "watch"] + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceexports/status"] + verbs: ["update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-lighthouse-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-lighthouse-agent +subjects: + - kind: ServiceAccount + name: submariner-lighthouse-agent + namespace: submariner-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-lighthouse-coredns +rules: + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["submariner.io"] + resources: ["gateways", "submariners"] + verbs: ["get", "list", "watch"] + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceimports"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-lighthouse-coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-lighthouse-coredns +subjects: + - kind: ServiceAccount + name: submariner-lighthouse-coredns + namespace: submariner-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-routeagent +rules: + - apiGroups: [""] + resources: ["pods", "services", "configmaps", "endpoints"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["ovn-cert"] + verbs: ["get"] + - apiGroups: ["config.openshift.io"] + resources: ["networks"] + resourceNames: ["cluster"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["projectcalico.org"] + resources: ["ippools"] + verbs: ["get", "create", "delete", "update", "deletecollection"] + - apiGroups: ["submariner.io"] + resources: ["endpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: ["submariner.io"] + resources: ["routeagents", "gatewayroutes", "nongatewayroutes"] + verbs: ["get", "list", "watch", "create", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-routeagent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-routeagent +subjects: + - kind: ServiceAccount + name: submariner-routeagent + namespace: submariner-operator +--- +# RBAC for broker secret copier job (used by PostSync hook) +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: broker-secret-handler + namespace: submariner-k8s-broker +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: broker-secret-copier-reader + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: broker-secret-handler +subjects: + - kind: ServiceAccount + name: broker-secret-copier + namespace: submariner-k8s-broker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator-secret-writer + namespace: submariner-operator +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: broker-secret-copier-writer + namespace: submariner-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operator-secret-writer +subjects: + - kind: ServiceAccount + name: broker-secret-copier + namespace: submariner-k8s-broker +--- +# Namespaced permissions for submariner-operator to watch resources in its own namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: submariner-operator-namespace + namespace: submariner-operator +rules: + - apiGroups: ["submariner.io"] + resources: ["gateways", "servicediscoveries", "submariners", "brokers"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["submariner.io"] + resources: ["gateways/status", "servicediscoveries/status", "submariners/status", "brokers/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["apps"] + resources: ["daemonsets", "deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["services", "configmaps", "secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: submariner-operator-namespace + namespace: submariner-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: submariner-operator-namespace +subjects: + - kind: ServiceAccount + name: submariner-operator + namespace: submariner-operator diff --git a/common/submariner/README.md b/common/submariner/README.md new file mode 100644 index 0000000..6e42808 --- /dev/null +++ b/common/submariner/README.md @@ -0,0 +1,97 @@ +# MIP Submariner Deployment + +Deploys Submariner for multi-cluster connectivity in MIP infrastructure using official Submariner Helm charts. + +## Overview + +This deployment uses: +- Official `submariner-k8s-broker` Helm chart for broker components +- Official `submariner-operator` Helm chart for cluster connectivity +- Custom values files for environment-specific configuration +- Kustomize for additional customization when needed + +## Prerequisites + +- Kubernetes 1.23+ (both clusters) +- Helm 3.8+ (on nodes where deploying manually) +- Calico CNI with VXLAN encapsulation +- Non-overlapping cluster CIDRs +- LoadBalancer service support (or MetalLB) for broker cluster +- Argo CD installed on broker cluster + +## Manual Installation guide for testing (we use argocd normally) + +### 1. Deploy Broker and Operator (Public Cluster via Argo CD) + +```bash +# Apply the Argo CD Applications +kubectl apply -f submariner.yaml + +# Sync the applications +argocd app sync submariner-broker +argocd app sync submariner-operator +``` + +### 2. Get Broker Info + +After broker deployment, extract connection details for remote cluster: + +```bash +# Get broker API server +kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' + +# Create service account token +kubectl create token submariner-broker -n submariner-k8s-broker --duration=87600h + +# Get CA certificate +kubectl get secret -n submariner-k8s-broker -o jsonpath='{.data.ca\.crt}' | base64 -d +``` + +Update these values in `deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml`. + +### 3. Deploy to Remote Cluster + +See `../../deployments/hybrid/federations/federation-Z/remote-node/README.md` + +## Configuration + +### Key Values + +All values are set in `values-public-cluster.yaml`: + +- `submariner-operator.submariner.clusterID`: Unique identifier for cluster +- `submariner-operator.submariner.clusterCIDR`: Pod network CIDR +- `submariner-operator.submariner.serviceCIDR`: Service network CIDR +- `submariner-k8s-broker.globalnet.enabled`: Enable for overlapping CIDRs (false by default) + +See official chart documentation for all available options: +- [submariner-k8s-broker chart](https://github.com/submariner-io/submariner-charts/tree/main/submariner-k8s-broker) +- [submariner-operator chart](https://github.com/submariner-io/submariner-charts/tree/main/submariner-operator) + +## Verification + +```bash +# Check broker pods +kubectl get pods -n submariner-k8s-broker + +# Check operator pods +kubectl get pods -n submariner-operator + +# Check connections (after remote cluster joins) +subctl show connections + +# Test service discovery +kubectl run test --rm -it --image=busybox -- nslookup test.federation-z.svc.clusterset.local +``` + +## Customization with Kustomize + +For settings not exposed by Helm charts, use Kustomize patches in `kustomization.yaml`. + +Example: Setting a specific LoadBalancer IP (see `patches/` directory). + +## Troubleshooting + +See project documentation for detailed troubleshooting guide. + + diff --git a/common/submariner/broker/copy-secret-hook.yaml b/common/submariner/broker/copy-secret-hook.yaml new file mode 100644 index 0000000..6623f12 --- /dev/null +++ b/common/submariner/broker/copy-secret-hook.yaml @@ -0,0 +1,102 @@ +--- +# ServiceAccount for the secret copier job +apiVersion: v1 +kind: ServiceAccount +metadata: + name: broker-secret-copier + namespace: submariner-k8s-broker + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +--- +# Job to copy the secret from broker to operator namespace +# Note: RBAC for this job is in base/mip-infrastructure/rbac/submariner-rbac.yaml +# and must be applied manually before ArgoCD deployment +apiVersion: batch/v1 +kind: Job +metadata: + name: copy-broker-secret + namespace: submariner-k8s-broker + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + name: broker-secret-copier + spec: + serviceAccountName: broker-secret-copier + restartPolicy: OnFailure + containers: + - name: copier + image: bitnami/kubectl:latest + command: + - /bin/sh + - -c + - | + set -e + echo "Waiting for broker secret to be created..." + for i in $(seq 1 30); do + if kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker >/dev/null 2>&1; then + echo "Broker secret found!" + break + fi + echo "Attempt $i/30: Secret not found yet, waiting..." + sleep 2 + done + + echo "Extracting CA and token from broker secret..." + CA=$(kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker -o jsonpath='{.data.ca\.crt}') + TOKEN=$(kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker -o jsonpath='{.data.token}') + + # Handle IPSec PSK + echo "Checking for existing IPSec PSK in broker namespace..." + if kubectl get secret submariner-ipsec-psk -n submariner-k8s-broker >/dev/null 2>&1; then + echo "Existing PSK found." + PSK_BASE64=$(kubectl get secret submariner-ipsec-psk -n submariner-k8s-broker -o jsonpath='{.data.psk}') + else + echo "No PSK found. Generating new IPSec PSK..." + # Generate a random alphanumeric string for PSK + PSK=$(head -c 64 /dev/urandom | base64 | tr -d '\n') + PSK_BASE64=$(echo -n "$PSK" | base64 | tr -d '\n') + + echo "Saving PSK to submariner-ipsec-psk in broker namespace..." + cat < broker-token.txt + +# Extract CA certificate (keep as base64 - Helm chart expects base64-encoded CA) +kubectl -n submariner-k8s-broker get secret submariner-broker-submariner-k8s-broker-client-token -o jsonpath='{.data.ca\.crt}' > broker-ca-base64.txt + +# Extract IPSec PSK from the main cluster's submariner-operator namespace +kubectl -n submariner-operator get secret submariner-ipsec-psk -o jsonpath='{.data.psk}' > broker-psk.txt + +# Copy these 3 files to the REMOTE node via scp or other secure means (broker-token.txt, broker-ca-base64.txt, broker-psk.txt) +``` + +**Security note**: These files contain sensitive credentials. Transfer them securely (scp, rsync over SSH) and delete them after use. + +## 2) Bootstrap tools on the fresh Ubuntu VM + +Run this once on a clean machine to install required tooling (curl, jq, helm, subctl). Note: kubectl will be provided by MicroK8s in the next step. + +```bash +sudo ./setup-tools.sh +``` + +What it does: +- Installs curl, jq, ca-certificates +- Installs Helm (snap classic) +- Installs subctl (latest) to /usr/local/bin +- Ensures /snap/bin is in PATH + +**Important**: kubectl is NOT installed here - it will be aliased from microk8s.kubectl after the next step. + +## 3) Install MicroK8s with custom IPv4 CIDRs + +Use the helper script to set `IPv4_CLUSTER_CIDR` and `IPv4_SERVICE_CIDR`, then install MicroK8s and wait for readiness. + +```bash +# Example (adjust the CIDRs if needed) +sudo IPv4_CLUSTER_CIDR=10.3.0.0/16 IPv4_SERVICE_CIDR=10.152.185.0/24 ./setup-microk8s.sh +``` + +What the script does: +- Writes `/var/snap/microk8s/common/.microk8s.yaml` with your CIDRs +- Installs MicroK8s: `snap install microk8s --classic --channel=1.31/stable` +- Waits for MicroK8s to be ready and exports kubeconfig +- Creates kubectl alias (microk8s.kubectl → kubectl) +- Adds your user to the microk8s group +- Verifies Pod/Service CIDRs +- Installs Calico API server (required for Submariner to detect network settings) +- Generates TLS certificates and patches the APIService + +If you prefer manual steps, see `subctl-procedure.md` (MicroK8s section). + +**After the script completes**: +1. Run `newgrp microk8s` to activate group membership (or log out/in) +2. Verify: `kubectl get nodes` + +The script takes several minutes (MicroK8s initialization + Calico API server). + +## 4) Helm Installation (Recommended for Production) + +### Prerequisites +- MicroK8s installed with your custom CIDRs (via the script above) +- Broker credentials files (broker-token.txt, broker-ca-base64.txt, broker-psk.txt) transferred to remote node +- Helm CLI installed + +### Installation Steps + +1. Add Submariner Helm repository: + +```bash +helm repo add submariner-latest https://submariner-io.github.io/submariner-charts/charts +helm repo update +``` + +2. Install Submariner operator with credentials: + +**Note**: The chart expects the token and PSK as plain text, and the CA certificate as **base64-encoded** (not decoded PEM). + +```bash +# Label node for gateway (replace NODE_NAME with your actual node name) +microk8s.kubectl label node NODE_NAME submariner.io/gateway=true + +# Install with credentials from files +helm install submariner-operator submariner-latest/submariner-operator \ + --namespace submariner-operator \ + --version 0.21.0 \ + --create-namespace \ + --set-string broker.token="$(cat broker-token.txt)" \ + --set-string broker.ca="$(cat broker-ca-base64.txt)" \ + --set-string ipsec.psk="$(cat broker-psk.txt)" \ + --values submariner-values.yaml +``` + +Alternative (if you prefer to store credentials in a Kubernetes secret first): +```bash +kubectl create namespace submariner-operator +kubectl -n submariner-operator create secret generic broker-secret \ + --from-literal=token="$(cat broker-token.txt)" \ + --from-literal=ca="$(cat broker-ca-base64.txt)" \ + --from-literal=psk="$(cat broker-psk.txt)" + +# Then extract and use in helm install +helm install submariner-operator submariner-latest/submariner-operator \ + --version 0.21.0 \ + + --namespace submariner-operator \ + --create-namespace \ + --set-string broker.token="$(kubectl -n submariner-operator get secret broker-secret -o jsonpath='{.data.token}' | base64 -d)" \ + --set-string broker.ca="$(kubectl -n submariner-operator get secret broker-secret -o jsonpath='{.data.ca}' | base64 -d)" \ + --set-string ipsec.psk="$(kubectl -n submariner-operator get secret broker-secret -o jsonpath='{.data.psk}' )" \ + --values submariner-values.yaml +``` + +3. Verify installation: +```bash +kubectl get pods -n submariner-operator +subctl show connections +``` + +4. **Cleanup credentials** (important for security): +```bash +# Securely wipe sensitive files after use +shred -u broker-token.txt broker-ca-base64.txt broker-psk.txt + +# If credentials were included inline, securely delete submariner-values.yaml as well +[ -f submariner-values.yaml ] && shred -u submariner-values.yaml +``` + +## Option 2: subctl Installation (Fallback) + +If Helm installation encounters issues, fall back to tested subctl method: + +```bash +# Follow steps from subctl-procedure.md "Setup on Private Remote Cluster (MicroK8s)" +subctl join broker-info.subm --clusterid federation-z-remote --check-broker-certificate=false +``` + +## Verification + +Test connectivity from remote cluster: +```bash +kubectl -n test run tmp-shell --rm -it --image quay.io/submariner/nettest -- /bin/bash +# Inside pod: +curl nginx.test.svc.clusterset.local:8080 +``` + +## Troubleshooting + +### Certificate Errors ("x509: certificate signed by unknown authority") + +If you see certificate errors in the `submariner-operator` logs or `ServiceExport` status, it means the broker CA was not correctly configured during installation. + +1. **Verify the CA** using the `openssl` command in the "Verify the CA" section above. + +```bash +# On the REMOTE node (after copying the files) +# 1. Decode the CA to a temporary file +base64 -d broker-ca-base64.txt > broker-ca.crt + +# 2. Verify connection to the broker API server +# Replace mip-tds.chuv.cscs.ch:6443 with your broker address if different +openssl s_client -connect mip-tds.chuv.cscs.ch:6443 -CAfile broker-ca.crt -showcerts < /dev/null + +# You should see "Verify return code: 0 (ok)" at the end. +# If you see "Verify return code: 19 (self-signed certificate...)", the CA is incorrect or missing. +``` + +2. **Update the installation** with the correct CA: + +```bash +helm upgrade submariner-operator submariner-latest/submariner-operator \ + --namespace submariner-operator \ + --reuse-values \ + --version 0.21.0 \ + --set-string broker.ca="$(cat broker-ca-base64.txt)" +``` diff --git a/deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh b/deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh new file mode 100644 index 0000000..ed10592 --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo "Please run as root (or sudo) to install MicroK8s and write config." >&2 + exit 1 +fi + +# Require that the user explicitly set both CIDRs +: "${IPv4_CLUSTER_CIDR:?Error: IPv4_CLUSTER_CIDR must be set. Example: sudo IPv4_CLUSTER_CIDR=10.3.0.0/16 IPv4_SERVICE_CIDR=10.152.185.0/24 $0}" +: "${IPv4_SERVICE_CIDR:?Error: IPv4_SERVICE_CIDR must be set. Example: sudo IPv4_CLUSTER_CIDR=10.3.0.0/16 IPv4_SERVICE_CIDR=10.152.185.0/24 $0}" + +echo "==> Using IPv4_CLUSTER_CIDR=${IPv4_CLUSTER_CIDR}" +echo "==> Using IPv4_SERVICE_CIDR=${IPv4_SERVICE_CIDR}" + +echo "==> Writing MicroK8s launch configuration with custom IPv4 CIDRs" +mkdir -p /var/snap/microk8s/common +service_gw_ip="$(echo "${IPv4_SERVICE_CIDR}" | awk -F'[./]' '{print $1"."$2"."$3"."$4+1}')" + +echo "==> Using extraSANs ${service_gw_ip}" + + +cat >/var/snap/microk8s/common/.microk8s.yaml < Installing MicroK8s (channel 1.31/stable)" +snap install microk8s --classic --channel=1.31/stable + +echo "==> Waiting for MicroK8s to be ready" +microk8s status --wait-ready + +# Create kubectl alias for convenience +echo "==> Creating kubectl alias" +snap alias microk8s.kubectl kubectl 2>/dev/null || true + +# Ensure current user can run microk8s commands without sudo +if [[ -n "${SUDO_USER:-}" ]]; then + echo "==> Adding $SUDO_USER to microk8s group" + usermod -a -G microk8s "$SUDO_USER" || true +fi + +# Export kubeconfig for current user if HOME is set and writable +if [[ -n "${SUDO_USER:-}" ]]; then + user_home=$(eval echo ~"${SUDO_USER}") +else + user_home="$HOME" +fi + +if [[ -n "$user_home" && -d "$user_home" ]]; then + echo "==> Exporting kubeconfig to $user_home/.kube/config" + mkdir -p "$user_home/.kube" + microk8s.kubectl config view --raw > "$user_home/.kube/config" + chown -R "${SUDO_USER:-root}": "$user_home/.kube" +fi + +echo "==> Verifying CIDRs" +echo -n "Kubernetes service IP: " +microk8s.kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}' && echo || true + +echo -n "Calico Pod CIDR (from DS env): " +microk8s.kubectl -n kube-system get daemonset calico-node \ + -o jsonpath="{.spec.template.spec.containers[?(@.name=='calico-node')].env[?(@.name=='CALICO_IPV4POOL_CIDR')].value}" && echo || true + +echo "==> Detecting Calico version" +CALICO_VERSION=$(microk8s.kubectl -n kube-system get daemonset calico-node \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="calico-node")].image}' | \ + grep -oP 'v\d+\.\d+\.\d+' ) +echo "==> Found Calico version: ${CALICO_VERSION}" + +echo "==> Installing Calico API server (required for Submariner)" +# Create the namespace declaratively to avoid apply warning +microk8s.kubectl apply -f - < Generating TLS certificate for Calico API server" +openssl req -x509 -nodes -newkey rsa:4096 \ + -keyout /tmp/apiserver.key -out /tmp/apiserver.crt -days 365 -subj "/" \ + -addext "subjectAltName = DNS:calico-api.calico-apiserver.svc" 2>/dev/null + +microk8s.kubectl -n calico-apiserver create secret generic calico-apiserver-certs \ + --from-file=apiserver.key=/tmp/apiserver.key \ + --from-file=apiserver.crt=/tmp/apiserver.crt \ + --dry-run=client -o yaml | microk8s.kubectl apply -f - + +echo "==> Patching APIService to trust Calico certificate" +microk8s.kubectl patch apiservice v3.projectcalico.org --type=merge \ + -p "{\"spec\":{\"caBundle\":\"$(microk8s.kubectl get secret -n calico-apiserver calico-apiserver-certs -o go-template='{{ index .data "apiserver.crt" }}')\"}}" + +rm -f /tmp/apiserver.key /tmp/apiserver.crt + +echo "==> Waiting for Calico API server to be ready" +sleep 60 +microk8s.kubectl -n calico-apiserver wait --for=condition=Ready pod -l apiserver=true --timeout=120s + +echo "" +echo "==> Done! MicroK8s is installed and configured." +echo "==> IMPORTANT: Run this command to activate microk8s group:" +echo " newgrp microk8s" +echo "" +echo "Then verify with:" +echo " kubectl get nodes" \ No newline at end of file diff --git a/deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh b/deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh new file mode 100644 index 0000000..abc8885 --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo "Please run as root: sudo $0" >&2 + exit 1 +fi + +echo "==> Updating APT and installing base tools" +apt-get update -y +apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + jq \ + gnupg \ + apt-transport-https + +# Ensure snap is available (normally on Ubuntu it is) +if ! command -v snap >/dev/null 2>&1; then + apt-get install -y snapd +fi + +# Ensure /snap/bin is in PATH for all shells +if ! grep -q "/snap/bin" /etc/environment; then + echo "==> Adding /snap/bin to PATH" + sed -i 's#^PATH=\(.*\)$#PATH=\1:/snap/bin#' /etc/environment || true +fi + +# Install Helm via snap (classic) +if ! command -v helm >/dev/null 2>&1; then + echo "==> Installing Helm" + snap install helm --classic +fi + +# Install subctl (latest) +if ! command -v subctl >/dev/null 2>&1; then + echo "==> Installing subctl (latest)" + curl -Ls https://get.submariner.io | bash + # Move to /usr/local/bin if installed in ~/.local/bin + if [[ -f "/root/.local/bin/subctl" ]]; then + mv /root/.local/bin/subctl /usr/local/bin/subctl + chmod +x /usr/local/bin/subctl + fi +fi + +echo "==> Tools installed" +helm version || true +subctl version || true +echo "Note: kubectl will be available after MicroK8s installation (next step)" \ No newline at end of file diff --git a/deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml b/deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml new file mode 100644 index 0000000..644abaa --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml @@ -0,0 +1,72 @@ +# Submariner values for direct Helm install +# Based on official chart: https://submariner.io/operations/deployment/helm/ +# +# Use with: +# helm install submariner-operator submariner-latest/submariner-operator \ +# --namespace submariner-operator --create-namespace \ +# --values submariner-values.yaml + +# Broker configuration - populated via --set during helm install +broker: + server: "mip-tds.chuv.cscs.ch:6443" + token: "" # Populated via --set-string broker.token="$(cat broker-token.txt)" (plain text) + ca: "" # Populated via --set-string broker.ca="$(cat broker-ca-base64.txt)" (base64-encoded) + namespace: "submariner-k8s-broker" + insecure: false # Set to true only if broker uses self-signed cert (not recommended) + +# Submariner configuration +submariner: + clusterId: "node3" + clusterCidr: "10.3.0.0/16" + serviceCidr: "10.152.185.0/24" + globalCidr: "" # Empty unless using Globalnet + natEnabled: true + cableDriver: "libreswan" # or "wireguard" or "vxlan" + serviceDiscovery: true + healthcheckEnabled: true + loadBalancerEnabled: false # false for remote cluster + debug: false + +# IPSec configuration - PSK must match the main cluster's PSK +ipsec: + psk: "" # Populated via --set-string ipsec.psk="$(cat broker-psk.txt)" (extracted from main cluster) + natPort: 4500 + ikePort: 500 + forceUDPEncaps: false + debug: false + +# Service discovery (Lighthouse) +serviceAccounts: + lighthouseAgent: + create: true + lighthouseCoreDns: + create: true + +# Gateway configuration +gateway: + nattPort: 4500 + metricsPort: 8080 + +# Resource constraints (MicroK8s-optimized) +resources: + gateway: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" + routeAgent: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "250m" + memory: "256Mi" + lighthouse: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "250m" + memory: "256Mi" \ No newline at end of file diff --git a/projects/mip-infrastructure.yaml b/projects/mip-infrastructure.yaml index f308333..7025870 100644 --- a/projects/mip-infrastructure.yaml +++ b/projects/mip-infrastructure.yaml @@ -49,6 +49,9 @@ spec: - name: infrastructure-admin description: Full access to infrastructure resources policies: - - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, - *, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, get, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, create, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, update, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, delete, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, sync, mip-argo-project-infrastructure/*, allow groups: [argocd-admins] diff --git a/projects/static/mip-common/mip-common.yaml b/projects/static/mip-common/mip-common.yaml index be3a23e..113033e 100644 --- a/projects/static/mip-common/mip-common.yaml +++ b/projects/static/mip-common/mip-common.yaml @@ -31,16 +31,8 @@ spec: kind: Namespace - group: '' kind: PersistentVolume - - group: rbac.authorization.k8s.io - kind: ClusterRole - - group: rbac.authorization.k8s.io - kind: ClusterRoleBinding - group: apiextensions.k8s.io kind: CustomResourceDefinition - - group: admissionregistration.k8s.io - kind: ValidatingAdmissionWebhook - - group: admissionregistration.k8s.io - kind: MutatingAdmissionWebhook - group: networking.k8s.io kind: IngressClass - group: cert-manager.io @@ -111,8 +103,11 @@ spec: - name: common-admin description: Full access to common resources policies: - - p, proj:mip-argo-project-common:common-admin, applications, *, mip-argo-project-common/*, - allow + - p, proj:mip-argo-project-common:common-admin, applications, get, mip-argo-project-common/*, allow + - p, proj:mip-argo-project-common:common-admin, applications, create, mip-argo-project-common/*, allow + - p, proj:mip-argo-project-common:common-admin, applications, update, mip-argo-project-common/*, allow + - p, proj:mip-argo-project-common:common-admin, applications, delete, mip-argo-project-common/*, allow + - p, proj:mip-argo-project-common:common-admin, applications, sync, mip-argo-project-common/*, allow groups: [argocd-admins] - name: common-operator description: Limited access to common resources diff --git a/projects/static/mip-federations/mip-federations.yaml b/projects/static/mip-federations/mip-federations.yaml index 2b5dca1..2bf6515 100644 --- a/projects/static/mip-federations/mip-federations.yaml +++ b/projects/static/mip-federations/mip-federations.yaml @@ -75,8 +75,11 @@ spec: - name: federation-admin description: Full access to federation resources policies: - - p, proj:mip-argo-project-federations:federation-admin, applications, *, - mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, get, mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, create, mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, update, mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, delete, mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, sync, mip-argo-project-federations/*, allow groups: [argocd-admins] - name: federation-developer description: Limited access to federation resources diff --git a/projects/static/mip-security/mip-security.yaml b/projects/static/mip-security/mip-security.yaml index e0f1247..3e67598 100644 --- a/projects/static/mip-security/mip-security.yaml +++ b/projects/static/mip-security/mip-security.yaml @@ -48,8 +48,11 @@ spec: - name: security-admin description: Full access to network security policies policies: - - p, proj:mip-argo-project-security:security-admin, applications, *, mip-argo-project-security/*, - allow + - p, proj:mip-argo-project-security:security-admin, applications, get, mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, create, mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, update, mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, delete, mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, sync, mip-argo-project-security/*, allow groups: [argocd-admins] - name: security-operator description: Limited access to network security policies diff --git a/projects/static/mip-shared-apps/mip-shared-apps.yaml b/projects/static/mip-shared-apps/mip-shared-apps.yaml index 45678d1..b2e27b1 100644 --- a/projects/static/mip-shared-apps/mip-shared-apps.yaml +++ b/projects/static/mip-shared-apps/mip-shared-apps.yaml @@ -74,14 +74,15 @@ spec: - name: shared-apps-admin description: Full access to shared applications policies: - - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, *, - mip-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, get, mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, create, mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, update, mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, delete, mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, sync, mip-argo-project-shared-apps/*, allow groups: [argocd-admins] - name: shared-apps-developer description: Limited access to shared applications policies: - - p, proj:mip-argo-project-shared-apps:shared-apps-developer, applications, - sync, mip-shared-apps/*, allow - - p, proj:mip-argo-project-shared-apps:shared-apps-developer, applications, - get, mip-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-developer, applications, sync, mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-developer, applications, get, mip-argo-project-shared-apps/*, allow groups: [argocd-developers] diff --git a/projects/static/submariner/kustomization.yaml b/projects/static/submariner/kustomization.yaml new file mode 100644 index 0000000..8181f69 --- /dev/null +++ b/projects/static/submariner/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - submariner-project.yaml diff --git a/projects/static/submariner/submariner-project.yaml b/projects/static/submariner/submariner-project.yaml new file mode 100644 index 0000000..aeeac77 --- /dev/null +++ b/projects/static/submariner/submariner-project.yaml @@ -0,0 +1,64 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: mip-argo-project-submariner + namespace: argocd-mip-team + # RBAC rules derived from: https://github.com/submariner-io/submariner-charts/tree/main/charts +spec: + description: Project for Submariner deployment + sourceRepos: + - git@github.com:NeuroTech-Platform/mip-infra-staging.git + - https://submariner-io.github.io/submariner-charts/charts + destinations: + - namespace: submariner-k8s-broker + server: https://kubernetes.default.svc + - namespace: submariner-operator + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + - group: submariner.io + kind: Cluster + - group: submariner.io + kind: Endpoint + - group: submariner.io + kind: Gateway + - group: submariner.io + kind: ClusterGlobalEgressIP + - group: submariner.io + kind: GlobalEgressIP + namespaceResourceWhitelist: + - group: apps + kind: Deployment + - group: apps + kind: DaemonSet + - group: batch + kind: Job + - group: '' + kind: ServiceAccount + - group: '' + kind: Service + - group: '' + kind: ConfigMap + - group: '' + kind: Secret + - group: rbac.authorization.k8s.io + kind: Role + - group: rbac.authorization.k8s.io + kind: RoleBinding + - group: submariner.io + kind: Submariner + - group: submariner.io + kind: ServiceDiscovery + - group: submariner.io + kind: Broker + roles: + - name: submariner-admin + description: Admin access to Submariner resources + policies: + - p, proj:mip-argo-project-submariner:submariner-admin, applications, get, mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, create, mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, update, mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, delete, mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, sync, mip-argo-project-submariner/*, allow + groups: [argocd-admins] diff --git a/projects/static/submariner/submariner-rbac-project.yaml b/projects/static/submariner/submariner-rbac-project.yaml new file mode 100644 index 0000000..432da11 --- /dev/null +++ b/projects/static/submariner/submariner-rbac-project.yaml @@ -0,0 +1,30 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: mip-argo-project-submariner-rbac + namespace: argocd-mip-team +spec: + description: Privileged project for Submariner RBAC resources + sourceRepos: + - git@github.com:NeuroTech-Platform/mip-infra-staging.git + destinations: + - namespace: submariner-k8s-broker + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + namespaceResourceWhitelist: + - group: argoproj.io + kind: Application + roles: + - name: submariner-rbac-admin + description: Admin access to Submariner RBAC + policies: + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, get, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, create, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, update, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, delete, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, sync, mip-argo-project-submariner-rbac/*, allow + groups: [argocd-admins] diff --git a/projects/templates/federation/templates/appproject.yaml b/projects/templates/federation/templates/appproject.yaml index 782f816..9f6c1a2 100644 --- a/projects/templates/federation/templates/appproject.yaml +++ b/projects/templates/federation/templates/appproject.yaml @@ -35,7 +35,11 @@ spec: - name: federation-admin description: "Admin access to {{ .Values.federationName }} resources" policies: - - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, *, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, get, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, create, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, update, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, delete, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, sync, mip-argo-project-{{ .Values.federationName }}/*, allow groups: {{- range .Values.rbac.adminGroups }} - {{ . }} From 8693373c3f78e0a3cddea5e2b3ebb108320123a8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 11 Dec 2025 14:18:58 +0000 Subject: [PATCH 2/6] =?UTF-8?q?style:=20auto-fix=20YAML=20linting=20issues?= =?UTF-8?q?=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...cd-application-controller-clusterrole.yaml | 24 +- .../rbac/submariner-rbac.yaml | 319 +++++++++--------- .../submariner/broker/copy-secret-hook.yaml | 8 +- common/submariner/broker/kustomization.yaml | 4 +- common/submariner/broker/values.yaml | 6 +- common/submariner/kustomization.yaml | 3 - .../operator/gateway-loadbalancer-ip.yaml | 3 +- common/submariner/operator/kustomization.yaml | 4 +- common/submariner/operator/values.yaml | 49 ++- .../patches/gateway-loadbalancer-ip.yaml | 4 +- common/submariner/submariner.yaml | 31 +- common/submariner/values-public-cluster.yaml | 1 - .../customizations/exareme2-values.yaml | 2 +- .../customizations/mip-stack-values.yaml | 2 +- .../remote-node/submariner-values.yaml | 51 ++- projects/mip-infrastructure.yaml | 15 +- projects/static/mip-common/mip-common.yaml | 15 +- .../mip-federations/mip-federations.yaml | 15 +- .../static/mip-security/mip-security.yaml | 15 +- .../mip-shared-apps/mip-shared-apps.yaml | 21 +- projects/static/submariner/kustomization.yaml | 5 +- .../static/submariner/submariner-project.yaml | 16 +- .../submariner/submariner-rbac-project.yaml | 19 +- 23 files changed, 334 insertions(+), 298 deletions(-) diff --git a/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml b/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml index bbe9c3d..094e99b 100644 --- a/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml +++ b/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml @@ -68,10 +68,28 @@ rules: resources: [ippools] verbs: [create, delete, update, deletecollection] - apiGroups: [submariner.io] - resources: [clusters, endpoints, gateways, clusterglobalegressips, globalegressips, globalingressips, submariners] - verbs: [create, delete, deletecollection, get, list, patch, update, watch] + resources: + - clusters + - endpoints + - gateways + - clusterglobalegressips + - globalegressips + - globalingressips + - submariners + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch - apiGroups: [submariner.io] - resources: [clusterglobalegressips/status, globalegressips/status, globalingressips/status] + resources: + - clusterglobalegressips/status + - globalegressips/status + - globalingressips/status verbs: [create, delete, deletecollection, update] - apiGroups: [network.openshift.io] resources: [service/externalips] diff --git a/base/mip-infrastructure/rbac/submariner-rbac.yaml b/base/mip-infrastructure/rbac/submariner-rbac.yaml index f73f1b7..58f144a 100644 --- a/base/mip-infrastructure/rbac/submariner-rbac.yaml +++ b/base/mip-infrastructure/rbac/submariner-rbac.yaml @@ -16,31 +16,31 @@ kind: ClusterRole metadata: name: submariner-operator rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "get", "list", "watch", "update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "create", "update", "delete", "watch"] - - apiGroups: [""] - resources: ["pods", "services", "nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["operator.openshift.io"] - resources: ["dnses"] - verbs: ["get", "update"] - - apiGroups: ["config.openshift.io"] - resources: ["networks"] - resourceNames: ["cluster"] - verbs: ["get"] - - apiGroups: ["monitoring.coreos.com"] - resources: ["servicemonitors"] - verbs: ["get", "create"] - - apiGroups: ["apps"] - resources: ["daemonsets"] - verbs: ["list"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "get", "list", "update"] + - apiGroups: [''] + resources: [configmaps] + verbs: [create, get, list, watch, update] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [get, list, create, update, delete, watch] + - apiGroups: [''] + resources: [pods, services, nodes] + verbs: [get, list, watch] + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, update] + - apiGroups: [config.openshift.io] + resources: [networks] + resourceNames: [cluster] + verbs: [get] + - apiGroups: [monitoring.coreos.com] + resources: [servicemonitors] + verbs: [get, create] + - apiGroups: [apps] + resources: [daemonsets] + verbs: [list] + - apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [create, get, list, update] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -61,55 +61,54 @@ kind: ClusterRole metadata: name: submariner-gateway rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods", "services", "nodes"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["operator.openshift.io"] - resources: ["dnses"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["config.openshift.io"] - resources: ["networks"] - verbs: ["get", "list"] - - apiGroups: ["submariner.io"] - resources: ["endpoints", "gateways", "clusters"] - verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - - apiGroups: ["submariner.io"] - resources: ["gateways/status"] - verbs: ["update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "get", "list", "update"] + - apiGroups: [''] + resources: [configmaps] + verbs: [get, list, watch, create, update] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [get, list, create, update, delete] + - apiGroups: [''] + resources: [pods, services, nodes] + verbs: [get, list, watch, patch, update] + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, list, watch, update] + - apiGroups: [config.openshift.io] + resources: [networks] + verbs: [get, list] + - apiGroups: [submariner.io] + resources: [endpoints, gateways, clusters] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [submariner.io] + resources: [gateways/status] + verbs: [update, patch] + - apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [create, get, list, update] --- - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: submariner-routeagent rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods", "services"] - verbs: ["get", "list", "watch"] - - apiGroups: ["operator.openshift.io"] - resources: ["dnses"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["config.openshift.io"] - resources: ["networks"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch", "update"] + - apiGroups: [''] + resources: [configmaps] + verbs: [get, list, watch, update] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [get, list, create, update, delete] + - apiGroups: [''] + resources: [pods, services] + verbs: [get, list, watch] + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, list, watch, update] + - apiGroups: [config.openshift.io] + resources: [networks] + verbs: [get, list] + - apiGroups: [''] + resources: [nodes] + verbs: [get, list, watch, update] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -144,24 +143,24 @@ metadata: name: submariner-broker-submariner-k8s-broker-cluster namespace: submariner-k8s-broker rules: - - apiGroups: ["submariner.io"] - resources: ["clusters", "endpoints"] - verbs: ["create", "get", "list", "watch", "patch", "update", "delete"] - - apiGroups: ["submariner.io"] - resources: ["brokers"] - verbs: ["get", "list"] - - apiGroups: ["multicluster.x-k8s.io"] - resources: ["serviceimports", "serviceimports/status"] - verbs: ["create", "get", "list", "watch", "patch", "update", "delete"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices", "endpointslices/restricted"] - verbs: ["create", "get", "list", "watch", "patch", "update", "delete"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get", "list"] + - apiGroups: [submariner.io] + resources: [clusters, endpoints] + verbs: [create, get, list, watch, patch, update, delete] + - apiGroups: [submariner.io] + resources: [brokers] + verbs: [get, list] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceimports, serviceimports/status] + verbs: [create, get, list, watch, patch, update, delete] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices, endpointslices/restricted] + verbs: [create, get, list, watch, patch, update, delete] + - apiGroups: [''] + resources: [secrets] + verbs: [get, list, watch] + - apiGroups: [''] + resources: [serviceaccounts] + verbs: [get, list] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -178,7 +177,6 @@ subjects: namespace: submariner-k8s-broker --- # Operator ClusterRoles and ClusterRoleBindings - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -197,24 +195,31 @@ kind: ClusterRole metadata: name: submariner-lighthouse-agent rules: - - apiGroups: [""] - resources: ["services", "namespaces", "endpoints"] - verbs: ["get", "list", "watch"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices", "endpointslices/restricted"] - verbs: ["create", "get", "list", "watch", "update", "delete", "deletecollection"] - - apiGroups: ["submariner.io"] - resources: ["gateways", "globalingressips"] - verbs: ["get", "list", "watch"] - - apiGroups: ["multicluster.x-k8s.io"] - resources: ["serviceimports", "serviceimports/status"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["multicluster.x-k8s.io"] - resources: ["serviceexports"] - verbs: ["get", "list", "watch"] - - apiGroups: ["multicluster.x-k8s.io"] - resources: ["serviceexports/status"] - verbs: ["update"] + - apiGroups: [''] + resources: [services, namespaces, endpoints] + verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices, endpointslices/restricted] + verbs: + - create + - get + - list + - watch + - update + - delete + - deletecollection + - apiGroups: [submariner.io] + resources: [gateways, globalingressips] + verbs: [get, list, watch] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceimports, serviceimports/status] + verbs: [create, get, list, watch, update, delete] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceexports] + verbs: [get, list, watch] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceexports/status] + verbs: [update] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -234,15 +239,15 @@ kind: ClusterRole metadata: name: submariner-lighthouse-coredns rules: - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["get", "list", "watch"] - - apiGroups: ["submariner.io"] - resources: ["gateways", "submariners"] - verbs: ["get", "list", "watch"] - - apiGroups: ["multicluster.x-k8s.io"] - resources: ["serviceimports"] - verbs: ["get", "list", "watch"] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] + - apiGroups: [submariner.io] + resources: [gateways, submariners] + verbs: [get, list, watch] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceimports] + verbs: [get, list, watch] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -262,29 +267,29 @@ kind: ClusterRole metadata: name: submariner-routeagent rules: - - apiGroups: [""] - resources: ["pods", "services", "configmaps", "endpoints"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["ovn-cert"] - verbs: ["get"] - - apiGroups: ["config.openshift.io"] - resources: ["networks"] - resourceNames: ["cluster"] - verbs: ["get"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["projectcalico.org"] - resources: ["ippools"] - verbs: ["get", "create", "delete", "update", "deletecollection"] - - apiGroups: ["submariner.io"] - resources: ["endpoints"] - verbs: ["get", "list", "watch"] - - apiGroups: ["submariner.io"] - resources: ["routeagents", "gatewayroutes", "nongatewayroutes"] - verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [''] + resources: [pods, services, configmaps, endpoints] + verbs: [get, list] + - apiGroups: [''] + resources: [secrets] + resourceNames: [ovn-cert] + verbs: [get] + - apiGroups: [config.openshift.io] + resources: [networks] + resourceNames: [cluster] + verbs: [get] + - apiGroups: [''] + resources: [nodes] + verbs: [get, list, watch, update] + - apiGroups: [projectcalico.org] + resources: [ippools] + verbs: [get, create, delete, update, deletecollection] + - apiGroups: [submariner.io] + resources: [endpoints] + verbs: [get, list, watch] + - apiGroups: [submariner.io] + resources: [routeagents, gatewayroutes, nongatewayroutes] + verbs: [get, list, watch, create, update, delete] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -306,9 +311,9 @@ metadata: name: broker-secret-handler namespace: submariner-k8s-broker rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "update", "patch"] + - apiGroups: [''] + resources: [secrets] + verbs: [get, create, update, patch] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -330,9 +335,9 @@ metadata: name: operator-secret-writer namespace: submariner-operator rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "update", "patch"] + - apiGroups: [''] + resources: [secrets] + verbs: [get, create, update, patch] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -355,18 +360,22 @@ metadata: name: submariner-operator-namespace namespace: submariner-operator rules: - - apiGroups: ["submariner.io"] - resources: ["gateways", "servicediscoveries", "submariners", "brokers"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["submariner.io"] - resources: ["gateways/status", "servicediscoveries/status", "submariners/status", "brokers/status"] - verbs: ["get", "update", "patch"] - - apiGroups: ["apps"] - resources: ["daemonsets", "deployments"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: [""] - resources: ["services", "configmaps", "secrets"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [submariner.io] + resources: [gateways, servicediscoveries, submariners, brokers] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [submariner.io] + resources: + - gateways/status + - servicediscoveries/status + - submariners/status + - brokers/status + verbs: [get, update, patch] + - apiGroups: [apps] + resources: [daemonsets, deployments] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [''] + resources: [services, configmaps, secrets] + verbs: [get, list, watch, create, update, patch, delete] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/common/submariner/broker/copy-secret-hook.yaml b/common/submariner/broker/copy-secret-hook.yaml index 6623f12..56249b8 100644 --- a/common/submariner/broker/copy-secret-hook.yaml +++ b/common/submariner/broker/copy-secret-hook.yaml @@ -34,7 +34,7 @@ spec: command: - /bin/sh - -c - - | + - |- set -e echo "Waiting for broker secret to be created..." for i in $(seq 1 30); do @@ -45,11 +45,10 @@ spec: echo "Attempt $i/30: Secret not found yet, waiting..." sleep 2 done - echo "Extracting CA and token from broker secret..." CA=$(kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker -o jsonpath='{.data.ca\.crt}') TOKEN=$(kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker -o jsonpath='{.data.token}') - + # Handle IPSec PSK echo "Checking for existing IPSec PSK in broker namespace..." if kubectl get secret submariner-ipsec-psk -n submariner-k8s-broker >/dev/null 2>&1; then @@ -73,7 +72,6 @@ spec: psk: ${PSK_BASE64} EOF fi - echo "Creating submariner-ipsec-psk in operator namespace..." cat < Date: Mon, 2 Feb 2026 17:12:24 +0000 Subject: [PATCH 3/6] New LB IP --- common/submariner/operator/gateway-loadbalancer-ip.yaml | 2 +- common/submariner/operator/values.yaml | 2 +- common/submariner/patches/gateway-loadbalancer-ip.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/submariner/operator/gateway-loadbalancer-ip.yaml b/common/submariner/operator/gateway-loadbalancer-ip.yaml index a6f8182..fb4114e 100644 --- a/common/submariner/operator/gateway-loadbalancer-ip.yaml +++ b/common/submariner/operator/gateway-loadbalancer-ip.yaml @@ -6,7 +6,7 @@ metadata: namespace: submariner-operator spec: type: LoadBalancer - loadBalancerIP: 148.187.27.151 + loadBalancerIP: 148.187.143.43 externalTrafficPolicy: Local ports: - name: ipsec-ike diff --git a/common/submariner/operator/values.yaml b/common/submariner/operator/values.yaml index 25db93e..1ae4252 100644 --- a/common/submariner/operator/values.yaml +++ b/common/submariner/operator/values.yaml @@ -46,7 +46,7 @@ service: annotations: metallb.universe.tf/address-pool: pool-no-auto metallb.io/address-pool: pool-no-auto - loadBalancerIP: 148.187.27.151 + loadBalancerIP: 148.187.143.43 externalTrafficPolicy: Local # Resource requirements operator: diff --git a/common/submariner/patches/gateway-loadbalancer-ip.yaml b/common/submariner/patches/gateway-loadbalancer-ip.yaml index 43ec524..a1afa14 100644 --- a/common/submariner/patches/gateway-loadbalancer-ip.yaml +++ b/common/submariner/patches/gateway-loadbalancer-ip.yaml @@ -8,5 +8,5 @@ metadata: metallb.universe.tf/address-pool: pool-no-auto metallb.io/address-pool: pool-no-auto spec: - loadBalancerIP: 148.187.27.151 + loadBalancerIP: 148.187.143.43 externalTrafficPolicy: Local From a6a0452b1deb120e412b5dade08b98cbfcc656d5 Mon Sep 17 00:00:00 2001 From: jdaln <150942337+jdaln@users.noreply.github.com> Date: Tue, 3 Feb 2026 18:15:53 +0100 Subject: [PATCH 4/6] network policies for federation --- .../templates/exareme-network-policy.yaml | 63 +++++++++++++++++++ .../templates/federation-network-policy.yaml | 8 ++- common/security/netpol.yaml | 6 ++ .../mip-infrastructure/kustomization.yaml | 1 + .../policies/submariner/kustomization.yaml | 5 ++ .../policies/submariner/network-policy.yaml | 25 ++++++++ .../submariner-network-policy.yaml | 21 +++++++ 7 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 common/security/federation/templates/exareme-network-policy.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml create mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml diff --git a/common/security/federation/templates/exareme-network-policy.yaml b/common/security/federation/templates/exareme-network-policy.yaml new file mode 100644 index 0000000..249237f --- /dev/null +++ b/common/security/federation/templates/exareme-network-policy.yaml @@ -0,0 +1,63 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exareme-worker-isolation + namespace: {{.Values.targetNamespace}} + labels: + app.kubernetes.io/name: federation-network-policy + app.kubernetes.io/instance: {{.Release.Name}} + app.kubernetes.io/managed-by: {{.Release.Service}} + mip.namespace-type: federation +spec: + # Select all workers + podSelector: + matchLabels: + app: exareme2-worker + policyTypes: + - Ingress + - Egress + + # Allow Ingress ONLY from Controller + ingress: + - from: + - podSelector: + matchLabels: + app: exareme2-controller + + # Worker only query its local dataset + egress: [] + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exareme-controller-access + namespace: {{.Values.targetNamespace}} + labels: + app.kubernetes.io/name: federation-network-policy + app.kubernetes.io/instance: {{.Release.Name}} + app.kubernetes.io/managed-by: {{.Release.Service}} + mip.namespace-type: federation +spec: + # Select the controller + podSelector: + matchLabels: + app: exareme2-controller + policyTypes: + - Egress + + # Allow Controller to talk to Workers and DNS + egress: + - to: + - podSelector: + matchLabels: + app: exareme2-worker + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{.Values.systemNamespace}} + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 diff --git a/common/security/federation/templates/federation-network-policy.yaml b/common/security/federation/templates/federation-network-policy.yaml index 5ce802f..b3fa2d6 100644 --- a/common/security/federation/templates/federation-network-policy.yaml +++ b/common/security/federation/templates/federation-network-policy.yaml @@ -12,7 +12,13 @@ metadata: mip.namespace-type: federation mip.federation-type: {{.Values.federationType}} spec: - podSelector: {} + podSelector: + {{- if .Values.networkPolicy.excludedSelector }} + matchExpressions: + {{- toYaml .Values.networkPolicy.excludedSelector | nindent 6 }} + {{- else }} + {} + {{- end }} policyTypes: - Ingress - Egress diff --git a/common/security/netpol.yaml b/common/security/netpol.yaml index 70cf144..93810c6 100644 --- a/common/security/netpol.yaml +++ b/common/security/netpol.yaml @@ -51,6 +51,12 @@ spec: - name: federationType value: '{{ if contains "local" .path.path }}local{{ else }}hybrid{{ end }}' + - name: networkPolicy.excludedSelector[0].key + value: app + - name: networkPolicy.excludedSelector[0].operator + value: In + - name: networkPolicy.excludedSelector[0].values[0] + value: exareme2-worker destination: server: '{{ .cluster }}' namespace: '{{ if contains "local" .path.path }}{{ .path.basename | lower diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml index 7875911..86541fd 100644 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml @@ -12,6 +12,7 @@ commonLabels: resources: - ../../../../../deployments/shared-apps/exareme2 - ../../../../../deployments/shared-apps/mip-stack + - submariner-network-policy.yaml patches: # JSON patches to customize project, namespace, and add federation-specific values - path: customizations/exareme2-kustomize.yaml diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml new file mode 100644 index 0000000..a07c251 --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - network-policy.yaml diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml new file mode 100644 index 0000000..15e16fc --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-submariner-cidrs + namespace: federation-z +spec: + # Apply ONLY to exareme2-controller + podSelector: + matchLabels: + app: exareme2-controller + policyTypes: + - Ingress + - Egress + + # Allow inbound traffic from the remote clusters' Pod CIDRs + ingress: + - from: + - ipBlock: { cidr: 10.3.0.0/16 } + # - one per remote node + + # Allow outbound traffic to the remote clusters' Pod CIDRs + egress: + - to: + - ipBlock: { cidr: 10.3.0.0/16 } + # - one per remote node diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml new file mode 100644 index 0000000..39ef2de --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: submariner-network-policy + namespace: argocd-mip-team + finalizers: [resources-finalizer.argocd.argoproj.io] +spec: + project: mip-argo-project-federation-z + source: + repoURL: https://github.com/NeuroTech-Platform/mip-infra.git + targetRevision: main + path: deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner + destination: + server: https://kubernetes.default.svc + namespace: federation-z + syncPolicy: + automated: + selfHeal: true + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true From 542734e39b3d0c9c85a5d0bda007f2d57bb336ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Feb 2026 17:16:28 +0000 Subject: [PATCH 5/6] =?UTF-8?q?style:=20auto-fix=20YAML=20linting=20issues?= =?UTF-8?q?=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../policies/submariner/kustomization.yaml | 5 ++--- .../policies/submariner/network-policy.yaml | 5 +++-- .../mip-infrastructure/submariner-network-policy.yaml | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml index a07c251..e17dcc6 100644 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml @@ -1,5 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - -resources: - - network-policy.yaml +resources: [network-policy.yaml] diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml index 15e16fc..857c9d8 100644 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml @@ -1,3 +1,4 @@ +--- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -15,11 +16,11 @@ spec: # Allow inbound traffic from the remote clusters' Pod CIDRs ingress: - from: - - ipBlock: { cidr: 10.3.0.0/16 } + - ipBlock: {cidr: 10.3.0.0/16} # - one per remote node # Allow outbound traffic to the remote clusters' Pod CIDRs egress: - to: - - ipBlock: { cidr: 10.3.0.0/16 } + - ipBlock: {cidr: 10.3.0.0/16} # - one per remote node diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml index 39ef2de..ceaa6a4 100644 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -16,6 +17,4 @@ spec: syncPolicy: automated: selfHeal: true - syncOptions: - - CreateNamespace=true - - ApplyOutOfSyncOnly=true + syncOptions: [CreateNamespace=true, ApplyOutOfSyncOnly=true] From dce265778e4044ba385b96cf3d6e8d2b990cf35c Mon Sep 17 00:00:00 2001 From: jdaln <150942337+jdaln@users.noreply.github.com> Date: Tue, 3 Feb 2026 18:26:56 +0100 Subject: [PATCH 6/6] fix for the submariner netpol --- .../mip-infrastructure/kustomization.yaml | 2 +- ...work-policy.yaml => network-policies.yaml} | 2 +- .../policies/submariner/kustomization.yaml | 4 --- .../policies/submariner/network-policy.yaml | 26 ------------------- 4 files changed, 2 insertions(+), 32 deletions(-) rename deployments/hybrid/federations/federation-Z/mip-infrastructure/{submariner-network-policy.yaml => network-policies.yaml} (95%) delete mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml delete mode 100644 deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml index 86541fd..c83b13d 100644 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/kustomization.yaml @@ -12,7 +12,7 @@ commonLabels: resources: - ../../../../../deployments/shared-apps/exareme2 - ../../../../../deployments/shared-apps/mip-stack - - submariner-network-policy.yaml + - network-policies.yaml patches: # JSON patches to customize project, namespace, and add federation-specific values - path: customizations/exareme2-kustomize.yaml diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/network-policies.yaml similarity index 95% rename from deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml rename to deployments/hybrid/federations/federation-Z/mip-infrastructure/network-policies.yaml index ceaa6a4..48f28a1 100644 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-network-policy.yaml +++ b/deployments/hybrid/federations/federation-Z/mip-infrastructure/network-policies.yaml @@ -10,7 +10,7 @@ spec: source: repoURL: https://github.com/NeuroTech-Platform/mip-infra.git targetRevision: main - path: deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner + path: deployments/hybrid/federations/federation-Z/mip-infrastructure/submariner-policies/ destination: server: https://kubernetes.default.svc namespace: federation-z diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml deleted file mode 100644 index e17dcc6..0000000 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: [network-policy.yaml] diff --git a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml b/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml deleted file mode 100644 index 857c9d8..0000000 --- a/deployments/hybrid/federations/federation-Z/mip-infrastructure/policies/submariner/network-policy.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-submariner-cidrs - namespace: federation-z -spec: - # Apply ONLY to exareme2-controller - podSelector: - matchLabels: - app: exareme2-controller - policyTypes: - - Ingress - - Egress - - # Allow inbound traffic from the remote clusters' Pod CIDRs - ingress: - - from: - - ipBlock: {cidr: 10.3.0.0/16} - # - one per remote node - - # Allow outbound traffic to the remote clusters' Pod CIDRs - egress: - - to: - - ipBlock: {cidr: 10.3.0.0/16} - # - one per remote node