diff --git a/.gitignore b/.gitignore index 073a168..1bbe27b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ _*/ ### Helm ### # Chart dependencies -**/charts/*.tgz +charts/ ### JetBrains+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider diff --git a/README.md b/README.md index e5aa001..c9a197b 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,39 @@ graph TD - `argocd` CLI installed (optional but recommended) - SSH access to the repository configured in Argo CD +### 🔐 Remote Access (VPN/SSH) + +If you are not on the same network as the cluster, you need to tunnel your traffic. + +#### Option A: SOCKS Proxy (Recommended for `kubectl`) +This routes all your traffic through the jump host. + +1. **Open Tunnel**: + ```bash + ssh -D 1080 -C -q -N @ + ``` +2. **Configure Environment**: + ```bash + export HTTPS_PROXY=socks5://127.0.0.1:1080 + ``` + +#### Option B: Port Forward + Hosts (Recommended for `argocd` CLI / UI) +This tricks your local machine into thinking `localhost` is the remote server, while preserving the hostname for Ingress routing. + +1. **Update `/etc/hosts`**: + ```bash + # Add this line + 127.0.0.1 argocd.mip-tds.chuv.cscs.ch + ``` +2. **Open Tunnel (Sudo required for port 443)**: + ```bash + sudo ssh -L 443:argocd.mip-tds.chuv.cscs.ch:443 @ + ``` +3. **Login**: + ```bash + argocd login argocd.mip-tds.chuv.cscs.ch:443 --insecure --grpc-web + ``` + ### Initial secrets: The following secrets must exist in the cluster before or after running this repository's setup scripts. If you run it after, creation will hang until these are present. @@ -288,6 +321,7 @@ argocd repo add git@github.com:NeuroTech-Platform/mip-deployments.git \ --name mip-infra kubectl apply -f base/mip-infrastructure/rbac/nginx-public-rbac.yaml +kubectl apply -f base/mip-infrastructure/rbac/submariner-rbac.yaml ``` diff --git a/argo-setup/README.md b/argo-setup/README.md index 7f3b4d4..7813b5c 100644 --- a/argo-setup/README.md +++ b/argo-setup/README.md @@ -44,9 +44,9 @@ ARGOCD_HOST=argocd.example.com #YOUR SUBDOMAIN HERE # Replace placeholder hostname in all patch files cd argo-setup # BSD-Style -find . -type f -exec sed -i '' "s/argocd.example.com/$ARGOCD_HOST/g" {} + +LC_ALL=C find . -type f -not -path '*/.git/*' -exec sed -i '' "s/argocd.example.com/$ARGOCD_HOST/g" {} + # GNU-Style -find . -type f -exec sed -i "s/argocd.example.com/$ARGOCD_HOST/g" {} + +LC_ALL=C find . -type f -not -path '*/.git/*' -exec sed -i "s/argocd.example.com/$ARGOCD_HOST/g" {} + # Resolve latest 3.0 version (or pin to specific version) export ARGOCD_SERIES=v3.0 @@ -58,6 +58,9 @@ export ARGOCD_VER=$(curl -s https://api.github.com/repos/argoproj/argo-cd/releas echo "Resolved latest 3.0 tag: $ARGOCD_VER" # Update kustomization.yaml with resolved version +# BSD-Style +sed -i '' "s|/v[0-9.]*/manifests/ha/install.yaml|/${ARGOCD_VER}/manifests/ha/install.yaml|g" patches/kustomization.yaml +# GNU-Style sed -i "s|/v[0-9.]*/manifests/ha/install.yaml|/${ARGOCD_VER}/manifests/ha/install.yaml|g" patches/kustomization.yaml # 1. Namespace diff --git a/argo-setup/patches/kustomization.yaml b/argo-setup/patches/kustomization.yaml index ffba647..46d0a21 100644 --- a/argo-setup/patches/kustomization.yaml +++ b/argo-setup/patches/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization namespace: argocd-mip-team # Base: upstream HA manifest (resolved tag) resources: - - https://raw.githubusercontent.com/argoproj/argo-cd/v3.0.11/manifests/ha/install.yaml + - https://raw.githubusercontent.com/argoproj/argo-cd/v3.0.20/manifests/ha/install.yaml - patch-argocd-ingress.yaml patchesStrategicMerge: - patch-argocd-application-controller-clusterrole.yaml diff --git a/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml b/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml index b8f188b..094e99b 100644 --- a/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml +++ b/argo-setup/patches/patch-argocd-application-controller-clusterrole.yaml @@ -36,15 +36,9 @@ rules: - apiGroups: [monitoring.coreos.com] resources: [prometheusrules, servicemonitors] verbs: [create, delete, patch, update] - - apiGroups: [cert-manager.io] - resources: [clusterissuers] - verbs: [create, delete, patch, update] - apiGroups: [networking.k8s.io] resources: [ingresses, networkpolicies, ingressclasses] verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: [rbac.authorization.k8s.io] - resources: [clusterrolebindings, clusterroles, rolebindings, roles] - verbs: [create, delete, patch, update] - apiGroups: [''] resources: - configmaps @@ -61,3 +55,49 @@ rules: - apiGroups: [''] resources: [pods] verbs: [create, delete, patch, update] # in the future, only delete but at the moment we still have standalone pods + + # Rule 4: Submariner requirements + # The controller needs these permissions to grant them to Submariner components + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, list, watch, update] + - apiGroups: [config.openshift.io] + resources: [networks] + verbs: [get, list] + - apiGroups: [projectcalico.org] + resources: [ippools] + verbs: [create, delete, update, deletecollection] + - apiGroups: [submariner.io] + resources: + - clusters + - endpoints + - gateways + - clusterglobalegressips + - globalegressips + - globalingressips + - submariners + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: [submariner.io] + resources: + - clusterglobalegressips/status + - globalegressips/status + - globalingressips/status + verbs: [create, delete, deletecollection, update] + - apiGroups: [network.openshift.io] + resources: [service/externalips] + verbs: [create, delete] + - apiGroups: [''] + resources: [nodes] + verbs: [get, list, watch, update] +# Add if we ever use GlobalNet +# - apiGroups: [''] +# resources: [endpoints] +# verbs: [get, list, watch, create, update, delete, patch] diff --git a/argo-setup/patches/patch-argocd-cm.yaml b/argo-setup/patches/patch-argocd-cm.yaml index 242b041..302b323 100644 --- a/argo-setup/patches/patch-argocd-cm.yaml +++ b/argo-setup/patches/patch-argocd-cm.yaml @@ -12,3 +12,4 @@ data: application.instanceLabelKey: argocd.argoproj.io/instanceTracking installationID: mip-team-argo-cd resource.respectRBAC: normal + kustomize.buildOptions: --enable-helm diff --git a/base/argo-projects.yaml b/base/argo-projects.yaml index 6245b1f..c8d41e3 100644 --- a/base/argo-projects.yaml +++ b/base/argo-projects.yaml @@ -23,6 +23,8 @@ spec: fileName: mip-common - projectName: mip-argo-project-security fileName: mip-security + - projectName: mip-argo-project-submariner + fileName: submariner goTemplate: true template: metadata: diff --git a/base/mip-infrastructure/mip-infrastructure.yaml b/base/mip-infrastructure/mip-infrastructure.yaml index 1a1f212..bbcff1b 100644 --- a/base/mip-infrastructure/mip-infrastructure.yaml +++ b/base/mip-infrastructure/mip-infrastructure.yaml @@ -36,6 +36,7 @@ spec: - path: common/datacatalog - path: common/nginx-ingress - path: common/security + - path: common/submariner - list: elements: - cluster: https://kubernetes.default.svc diff --git a/base/mip-infrastructure/rbac/submariner-rbac.yaml b/base/mip-infrastructure/rbac/submariner-rbac.yaml new file mode 100644 index 0000000..58f144a --- /dev/null +++ b/base/mip-infrastructure/rbac/submariner-rbac.yaml @@ -0,0 +1,392 @@ +--- +# Namespaces required by Submariner components and RBACs below +apiVersion: v1 +kind: Namespace +metadata: + name: submariner-operator +--- +apiVersion: v1 +kind: Namespace +metadata: + name: submariner-k8s-broker +--- +# Extracted from submariner-operator Helm chart +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-operator +rules: + - apiGroups: [''] + resources: [configmaps] + verbs: [create, get, list, watch, update] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [get, list, create, update, delete, watch] + - apiGroups: [''] + resources: [pods, services, nodes] + verbs: [get, list, watch] + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, update] + - apiGroups: [config.openshift.io] + resources: [networks] + resourceNames: [cluster] + verbs: [get] + - apiGroups: [monitoring.coreos.com] + resources: [servicemonitors] + verbs: [get, create] + - apiGroups: [apps] + resources: [daemonsets] + verbs: [list] + - apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [create, get, list, update] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-operator +subjects: + - kind: ServiceAccount + name: submariner-operator + namespace: submariner-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-operator +--- +# Extracted from submariner-k8s-broker (submariner) Helm chart +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-gateway +rules: + - apiGroups: [''] + resources: [configmaps] + verbs: [get, list, watch, create, update] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [get, list, create, update, delete] + - apiGroups: [''] + resources: [pods, services, nodes] + verbs: [get, list, watch, patch, update] + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, list, watch, update] + - apiGroups: [config.openshift.io] + resources: [networks] + verbs: [get, list] + - apiGroups: [submariner.io] + resources: [endpoints, gateways, clusters] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [submariner.io] + resources: [gateways/status] + verbs: [update, patch] + - apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [create, get, list, update] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-routeagent +rules: + - apiGroups: [''] + resources: [configmaps] + verbs: [get, list, watch, update] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [get, list, create, update, delete] + - apiGroups: [''] + resources: [pods, services] + verbs: [get, list, watch] + - apiGroups: [operator.openshift.io] + resources: [dnses] + verbs: [get, list, watch, update] + - apiGroups: [config.openshift.io] + resources: [networks] + verbs: [get, list] + - apiGroups: [''] + resources: [nodes] + verbs: [get, list, watch, update] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-routeagent +subjects: + - kind: ServiceAccount + name: submariner-routeagent + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-routeagent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-globalnet +subjects: + - kind: ServiceAccount + name: submariner-globalnet + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-globalnet +--- +# Namespaced Roles/Bindings (Extracted to prevent ArgoCD sync errors) +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: submariner-broker-submariner-k8s-broker-cluster + namespace: submariner-k8s-broker +rules: + - apiGroups: [submariner.io] + resources: [clusters, endpoints] + verbs: [create, get, list, watch, patch, update, delete] + - apiGroups: [submariner.io] + resources: [brokers] + verbs: [get, list] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceimports, serviceimports/status] + verbs: [create, get, list, watch, patch, update, delete] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices, endpointslices/restricted] + verbs: [create, get, list, watch, patch, update, delete] + - apiGroups: [''] + resources: [secrets] + verbs: [get, list, watch] + - apiGroups: [''] + resources: [serviceaccounts] + verbs: [get, list] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: submariner-broker-submariner-k8s-broker-cluster + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: submariner-broker-submariner-k8s-broker-cluster +subjects: + - kind: ServiceAccount + name: submariner-broker-submariner-k8s-broker-client + namespace: submariner-k8s-broker +--- +# Operator ClusterRoles and ClusterRoleBindings +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-gateway +subjects: + - kind: ServiceAccount + name: submariner-gateway + namespace: submariner-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-lighthouse-agent +rules: + - apiGroups: [''] + resources: [services, namespaces, endpoints] + verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices, endpointslices/restricted] + verbs: + - create + - get + - list + - watch + - update + - delete + - deletecollection + - apiGroups: [submariner.io] + resources: [gateways, globalingressips] + verbs: [get, list, watch] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceimports, serviceimports/status] + verbs: [create, get, list, watch, update, delete] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceexports] + verbs: [get, list, watch] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceexports/status] + verbs: [update] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-lighthouse-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-lighthouse-agent +subjects: + - kind: ServiceAccount + name: submariner-lighthouse-agent + namespace: submariner-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-lighthouse-coredns +rules: + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] + - apiGroups: [submariner.io] + resources: [gateways, submariners] + verbs: [get, list, watch] + - apiGroups: [multicluster.x-k8s.io] + resources: [serviceimports] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-lighthouse-coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-lighthouse-coredns +subjects: + - kind: ServiceAccount + name: submariner-lighthouse-coredns + namespace: submariner-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-routeagent +rules: + - apiGroups: [''] + resources: [pods, services, configmaps, endpoints] + verbs: [get, list] + - apiGroups: [''] + resources: [secrets] + resourceNames: [ovn-cert] + verbs: [get] + - apiGroups: [config.openshift.io] + resources: [networks] + resourceNames: [cluster] + verbs: [get] + - apiGroups: [''] + resources: [nodes] + verbs: [get, list, watch, update] + - apiGroups: [projectcalico.org] + resources: [ippools] + verbs: [get, create, delete, update, deletecollection] + - apiGroups: [submariner.io] + resources: [endpoints] + verbs: [get, list, watch] + - apiGroups: [submariner.io] + resources: [routeagents, gatewayroutes, nongatewayroutes] + verbs: [get, list, watch, create, update, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: submariner-routeagent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: submariner-routeagent +subjects: + - kind: ServiceAccount + name: submariner-routeagent + namespace: submariner-operator +--- +# RBAC for broker secret copier job (used by PostSync hook) +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: broker-secret-handler + namespace: submariner-k8s-broker +rules: + - apiGroups: [''] + resources: [secrets] + verbs: [get, create, update, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: broker-secret-copier-reader + namespace: submariner-k8s-broker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: broker-secret-handler +subjects: + - kind: ServiceAccount + name: broker-secret-copier + namespace: submariner-k8s-broker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator-secret-writer + namespace: submariner-operator +rules: + - apiGroups: [''] + resources: [secrets] + verbs: [get, create, update, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: broker-secret-copier-writer + namespace: submariner-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operator-secret-writer +subjects: + - kind: ServiceAccount + name: broker-secret-copier + namespace: submariner-k8s-broker +--- +# Namespaced permissions for submariner-operator to watch resources in its own namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: submariner-operator-namespace + namespace: submariner-operator +rules: + - apiGroups: [submariner.io] + resources: [gateways, servicediscoveries, submariners, brokers] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [submariner.io] + resources: + - gateways/status + - servicediscoveries/status + - submariners/status + - brokers/status + verbs: [get, update, patch] + - apiGroups: [apps] + resources: [daemonsets, deployments] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [''] + resources: [services, configmaps, secrets] + verbs: [get, list, watch, create, update, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: submariner-operator-namespace + namespace: submariner-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: submariner-operator-namespace +subjects: + - kind: ServiceAccount + name: submariner-operator + namespace: submariner-operator diff --git a/common/security/federation/templates/exareme-network-policy.yaml b/common/security/federation/templates/exareme-network-policy.yaml new file mode 100644 index 0000000..249237f --- /dev/null +++ b/common/security/federation/templates/exareme-network-policy.yaml @@ -0,0 +1,63 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exareme-worker-isolation + namespace: {{.Values.targetNamespace}} + labels: + app.kubernetes.io/name: federation-network-policy + app.kubernetes.io/instance: {{.Release.Name}} + app.kubernetes.io/managed-by: {{.Release.Service}} + mip.namespace-type: federation +spec: + # Select all workers + podSelector: + matchLabels: + app: exareme2-worker + policyTypes: + - Ingress + - Egress + + # Allow Ingress ONLY from Controller + ingress: + - from: + - podSelector: + matchLabels: + app: exareme2-controller + + # Worker only query its local dataset + egress: [] + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exareme-controller-access + namespace: {{.Values.targetNamespace}} + labels: + app.kubernetes.io/name: federation-network-policy + app.kubernetes.io/instance: {{.Release.Name}} + app.kubernetes.io/managed-by: {{.Release.Service}} + mip.namespace-type: federation +spec: + # Select the controller + podSelector: + matchLabels: + app: exareme2-controller + policyTypes: + - Egress + + # Allow Controller to talk to Workers and DNS + egress: + - to: + - podSelector: + matchLabels: + app: exareme2-worker + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{.Values.systemNamespace}} + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 diff --git a/common/security/federation/templates/federation-network-policy.yaml b/common/security/federation/templates/federation-network-policy.yaml index 5ce802f..b3fa2d6 100644 --- a/common/security/federation/templates/federation-network-policy.yaml +++ b/common/security/federation/templates/federation-network-policy.yaml @@ -12,7 +12,13 @@ metadata: mip.namespace-type: federation mip.federation-type: {{.Values.federationType}} spec: - podSelector: {} + podSelector: + {{- if .Values.networkPolicy.excludedSelector }} + matchExpressions: + {{- toYaml .Values.networkPolicy.excludedSelector | nindent 6 }} + {{- else }} + {} + {{- end }} policyTypes: - Ingress - Egress diff --git a/common/security/netpol.yaml b/common/security/netpol.yaml index 70cf144..93810c6 100644 --- a/common/security/netpol.yaml +++ b/common/security/netpol.yaml @@ -51,6 +51,12 @@ spec: - name: federationType value: '{{ if contains "local" .path.path }}local{{ else }}hybrid{{ end }}' + - name: networkPolicy.excludedSelector[0].key + value: app + - name: networkPolicy.excludedSelector[0].operator + value: In + - name: networkPolicy.excludedSelector[0].values[0] + value: exareme2-worker destination: server: '{{ .cluster }}' namespace: '{{ if contains "local" .path.path }}{{ .path.basename | lower diff --git a/common/submariner/README.md b/common/submariner/README.md new file mode 100644 index 0000000..6e42808 --- /dev/null +++ b/common/submariner/README.md @@ -0,0 +1,97 @@ +# MIP Submariner Deployment + +Deploys Submariner for multi-cluster connectivity in MIP infrastructure using official Submariner Helm charts. + +## Overview + +This deployment uses: +- Official `submariner-k8s-broker` Helm chart for broker components +- Official `submariner-operator` Helm chart for cluster connectivity +- Custom values files for environment-specific configuration +- Kustomize for additional customization when needed + +## Prerequisites + +- Kubernetes 1.23+ (both clusters) +- Helm 3.8+ (on nodes where deploying manually) +- Calico CNI with VXLAN encapsulation +- Non-overlapping cluster CIDRs +- LoadBalancer service support (or MetalLB) for broker cluster +- Argo CD installed on broker cluster + +## Manual Installation guide for testing (we use argocd normally) + +### 1. Deploy Broker and Operator (Public Cluster via Argo CD) + +```bash +# Apply the Argo CD Applications +kubectl apply -f submariner.yaml + +# Sync the applications +argocd app sync submariner-broker +argocd app sync submariner-operator +``` + +### 2. Get Broker Info + +After broker deployment, extract connection details for remote cluster: + +```bash +# Get broker API server +kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' + +# Create service account token +kubectl create token submariner-broker -n submariner-k8s-broker --duration=87600h + +# Get CA certificate +kubectl get secret -n submariner-k8s-broker -o jsonpath='{.data.ca\.crt}' | base64 -d +``` + +Update these values in `deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml`. + +### 3. Deploy to Remote Cluster + +See `../../deployments/hybrid/federations/federation-Z/remote-node/README.md` + +## Configuration + +### Key Values + +All values are set in `values-public-cluster.yaml`: + +- `submariner-operator.submariner.clusterID`: Unique identifier for cluster +- `submariner-operator.submariner.clusterCIDR`: Pod network CIDR +- `submariner-operator.submariner.serviceCIDR`: Service network CIDR +- `submariner-k8s-broker.globalnet.enabled`: Enable for overlapping CIDRs (false by default) + +See official chart documentation for all available options: +- [submariner-k8s-broker chart](https://github.com/submariner-io/submariner-charts/tree/main/submariner-k8s-broker) +- [submariner-operator chart](https://github.com/submariner-io/submariner-charts/tree/main/submariner-operator) + +## Verification + +```bash +# Check broker pods +kubectl get pods -n submariner-k8s-broker + +# Check operator pods +kubectl get pods -n submariner-operator + +# Check connections (after remote cluster joins) +subctl show connections + +# Test service discovery +kubectl run test --rm -it --image=busybox -- nslookup test.federation-z.svc.clusterset.local +``` + +## Customization with Kustomize + +For settings not exposed by Helm charts, use Kustomize patches in `kustomization.yaml`. + +Example: Setting a specific LoadBalancer IP (see `patches/` directory). + +## Troubleshooting + +See project documentation for detailed troubleshooting guide. + + diff --git a/common/submariner/broker/copy-secret-hook.yaml b/common/submariner/broker/copy-secret-hook.yaml new file mode 100644 index 0000000..56249b8 --- /dev/null +++ b/common/submariner/broker/copy-secret-hook.yaml @@ -0,0 +1,98 @@ +--- +# ServiceAccount for the secret copier job +apiVersion: v1 +kind: ServiceAccount +metadata: + name: broker-secret-copier + namespace: submariner-k8s-broker + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +--- +# Job to copy the secret from broker to operator namespace +# Note: RBAC for this job is in base/mip-infrastructure/rbac/submariner-rbac.yaml +# and must be applied manually before ArgoCD deployment +apiVersion: batch/v1 +kind: Job +metadata: + name: copy-broker-secret + namespace: submariner-k8s-broker + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + name: broker-secret-copier + spec: + serviceAccountName: broker-secret-copier + restartPolicy: OnFailure + containers: + - name: copier + image: bitnami/kubectl:latest + command: + - /bin/sh + - -c + - |- + set -e + echo "Waiting for broker secret to be created..." + for i in $(seq 1 30); do + if kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker >/dev/null 2>&1; then + echo "Broker secret found!" + break + fi + echo "Attempt $i/30: Secret not found yet, waiting..." + sleep 2 + done + echo "Extracting CA and token from broker secret..." + CA=$(kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker -o jsonpath='{.data.ca\.crt}') + TOKEN=$(kubectl get secret submariner-broker-submariner-k8s-broker-client-token -n submariner-k8s-broker -o jsonpath='{.data.token}') + + # Handle IPSec PSK + echo "Checking for existing IPSec PSK in broker namespace..." + if kubectl get secret submariner-ipsec-psk -n submariner-k8s-broker >/dev/null 2>&1; then + echo "Existing PSK found." + PSK_BASE64=$(kubectl get secret submariner-ipsec-psk -n submariner-k8s-broker -o jsonpath='{.data.psk}') + else + echo "No PSK found. Generating new IPSec PSK..." + # Generate a random alphanumeric string for PSK + PSK=$(head -c 64 /dev/urandom | base64 | tr -d '\n') + PSK_BASE64=$(echo -n "$PSK" | base64 | tr -d '\n') + + echo "Saving PSK to submariner-ipsec-psk in broker namespace..." + cat < broker-token.txt + +# Extract CA certificate (keep as base64 - Helm chart expects base64-encoded CA) +kubectl -n submariner-k8s-broker get secret submariner-broker-submariner-k8s-broker-client-token -o jsonpath='{.data.ca\.crt}' > broker-ca-base64.txt + +# Extract IPSec PSK from the main cluster's submariner-operator namespace +kubectl -n submariner-operator get secret submariner-ipsec-psk -o jsonpath='{.data.psk}' > broker-psk.txt + +# Copy these 3 files to the REMOTE node via scp or other secure means (broker-token.txt, broker-ca-base64.txt, broker-psk.txt) +``` + +**Security note**: These files contain sensitive credentials. Transfer them securely (scp, rsync over SSH) and delete them after use. + +## 2) Bootstrap tools on the fresh Ubuntu VM + +Run this once on a clean machine to install required tooling (curl, jq, helm, subctl). Note: kubectl will be provided by MicroK8s in the next step. + +```bash +sudo ./setup-tools.sh +``` + +What it does: +- Installs curl, jq, ca-certificates +- Installs Helm (snap classic) +- Installs subctl (latest) to /usr/local/bin +- Ensures /snap/bin is in PATH + +**Important**: kubectl is NOT installed here - it will be aliased from microk8s.kubectl after the next step. + +## 3) Install MicroK8s with custom IPv4 CIDRs + +Use the helper script to set `IPv4_CLUSTER_CIDR` and `IPv4_SERVICE_CIDR`, then install MicroK8s and wait for readiness. + +```bash +# Example (adjust the CIDRs if needed) +sudo IPv4_CLUSTER_CIDR=10.3.0.0/16 IPv4_SERVICE_CIDR=10.152.185.0/24 ./setup-microk8s.sh +``` + +What the script does: +- Writes `/var/snap/microk8s/common/.microk8s.yaml` with your CIDRs +- Installs MicroK8s: `snap install microk8s --classic --channel=1.31/stable` +- Waits for MicroK8s to be ready and exports kubeconfig +- Creates kubectl alias (microk8s.kubectl → kubectl) +- Adds your user to the microk8s group +- Verifies Pod/Service CIDRs +- Installs Calico API server (required for Submariner to detect network settings) +- Generates TLS certificates and patches the APIService + +If you prefer manual steps, see `subctl-procedure.md` (MicroK8s section). + +**After the script completes**: +1. Run `newgrp microk8s` to activate group membership (or log out/in) +2. Verify: `kubectl get nodes` + +The script takes several minutes (MicroK8s initialization + Calico API server). + +## 4) Helm Installation (Recommended for Production) + +### Prerequisites +- MicroK8s installed with your custom CIDRs (via the script above) +- Broker credentials files (broker-token.txt, broker-ca-base64.txt, broker-psk.txt) transferred to remote node +- Helm CLI installed + +### Installation Steps + +1. Add Submariner Helm repository: + +```bash +helm repo add submariner-latest https://submariner-io.github.io/submariner-charts/charts +helm repo update +``` + +2. Install Submariner operator with credentials: + +**Note**: The chart expects the token and PSK as plain text, and the CA certificate as **base64-encoded** (not decoded PEM). + +```bash +# Label node for gateway (replace NODE_NAME with your actual node name) +microk8s.kubectl label node NODE_NAME submariner.io/gateway=true + +# Install with credentials from files +helm install submariner-operator submariner-latest/submariner-operator \ + --namespace submariner-operator \ + --version 0.21.0 \ + --create-namespace \ + --set-string broker.token="$(cat broker-token.txt)" \ + --set-string broker.ca="$(cat broker-ca-base64.txt)" \ + --set-string ipsec.psk="$(cat broker-psk.txt)" \ + --values submariner-values.yaml +``` + +Alternative (if you prefer to store credentials in a Kubernetes secret first): +```bash +kubectl create namespace submariner-operator +kubectl -n submariner-operator create secret generic broker-secret \ + --from-literal=token="$(cat broker-token.txt)" \ + --from-literal=ca="$(cat broker-ca-base64.txt)" \ + --from-literal=psk="$(cat broker-psk.txt)" + +# Then extract and use in helm install +helm install submariner-operator submariner-latest/submariner-operator \ + --version 0.21.0 \ + + --namespace submariner-operator \ + --create-namespace \ + --set-string broker.token="$(kubectl -n submariner-operator get secret broker-secret -o jsonpath='{.data.token}' | base64 -d)" \ + --set-string broker.ca="$(kubectl -n submariner-operator get secret broker-secret -o jsonpath='{.data.ca}' | base64 -d)" \ + --set-string ipsec.psk="$(kubectl -n submariner-operator get secret broker-secret -o jsonpath='{.data.psk}' )" \ + --values submariner-values.yaml +``` + +3. Verify installation: +```bash +kubectl get pods -n submariner-operator +subctl show connections +``` + +4. **Cleanup credentials** (important for security): +```bash +# Securely wipe sensitive files after use +shred -u broker-token.txt broker-ca-base64.txt broker-psk.txt + +# If credentials were included inline, securely delete submariner-values.yaml as well +[ -f submariner-values.yaml ] && shred -u submariner-values.yaml +``` + +## Option 2: subctl Installation (Fallback) + +If Helm installation encounters issues, fall back to tested subctl method: + +```bash +# Follow steps from subctl-procedure.md "Setup on Private Remote Cluster (MicroK8s)" +subctl join broker-info.subm --clusterid federation-z-remote --check-broker-certificate=false +``` + +## Verification + +Test connectivity from remote cluster: +```bash +kubectl -n test run tmp-shell --rm -it --image quay.io/submariner/nettest -- /bin/bash +# Inside pod: +curl nginx.test.svc.clusterset.local:8080 +``` + +## Troubleshooting + +### Certificate Errors ("x509: certificate signed by unknown authority") + +If you see certificate errors in the `submariner-operator` logs or `ServiceExport` status, it means the broker CA was not correctly configured during installation. + +1. **Verify the CA** using the `openssl` command in the "Verify the CA" section above. + +```bash +# On the REMOTE node (after copying the files) +# 1. Decode the CA to a temporary file +base64 -d broker-ca-base64.txt > broker-ca.crt + +# 2. Verify connection to the broker API server +# Replace mip-tds.chuv.cscs.ch:6443 with your broker address if different +openssl s_client -connect mip-tds.chuv.cscs.ch:6443 -CAfile broker-ca.crt -showcerts < /dev/null + +# You should see "Verify return code: 0 (ok)" at the end. +# If you see "Verify return code: 19 (self-signed certificate...)", the CA is incorrect or missing. +``` + +2. **Update the installation** with the correct CA: + +```bash +helm upgrade submariner-operator submariner-latest/submariner-operator \ + --namespace submariner-operator \ + --reuse-values \ + --version 0.21.0 \ + --set-string broker.ca="$(cat broker-ca-base64.txt)" +``` diff --git a/deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh b/deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh new file mode 100644 index 0000000..ed10592 --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/remote-node/setup-microk8s.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo "Please run as root (or sudo) to install MicroK8s and write config." >&2 + exit 1 +fi + +# Require that the user explicitly set both CIDRs +: "${IPv4_CLUSTER_CIDR:?Error: IPv4_CLUSTER_CIDR must be set. Example: sudo IPv4_CLUSTER_CIDR=10.3.0.0/16 IPv4_SERVICE_CIDR=10.152.185.0/24 $0}" +: "${IPv4_SERVICE_CIDR:?Error: IPv4_SERVICE_CIDR must be set. Example: sudo IPv4_CLUSTER_CIDR=10.3.0.0/16 IPv4_SERVICE_CIDR=10.152.185.0/24 $0}" + +echo "==> Using IPv4_CLUSTER_CIDR=${IPv4_CLUSTER_CIDR}" +echo "==> Using IPv4_SERVICE_CIDR=${IPv4_SERVICE_CIDR}" + +echo "==> Writing MicroK8s launch configuration with custom IPv4 CIDRs" +mkdir -p /var/snap/microk8s/common +service_gw_ip="$(echo "${IPv4_SERVICE_CIDR}" | awk -F'[./]' '{print $1"."$2"."$3"."$4+1}')" + +echo "==> Using extraSANs ${service_gw_ip}" + + +cat >/var/snap/microk8s/common/.microk8s.yaml < Installing MicroK8s (channel 1.31/stable)" +snap install microk8s --classic --channel=1.31/stable + +echo "==> Waiting for MicroK8s to be ready" +microk8s status --wait-ready + +# Create kubectl alias for convenience +echo "==> Creating kubectl alias" +snap alias microk8s.kubectl kubectl 2>/dev/null || true + +# Ensure current user can run microk8s commands without sudo +if [[ -n "${SUDO_USER:-}" ]]; then + echo "==> Adding $SUDO_USER to microk8s group" + usermod -a -G microk8s "$SUDO_USER" || true +fi + +# Export kubeconfig for current user if HOME is set and writable +if [[ -n "${SUDO_USER:-}" ]]; then + user_home=$(eval echo ~"${SUDO_USER}") +else + user_home="$HOME" +fi + +if [[ -n "$user_home" && -d "$user_home" ]]; then + echo "==> Exporting kubeconfig to $user_home/.kube/config" + mkdir -p "$user_home/.kube" + microk8s.kubectl config view --raw > "$user_home/.kube/config" + chown -R "${SUDO_USER:-root}": "$user_home/.kube" +fi + +echo "==> Verifying CIDRs" +echo -n "Kubernetes service IP: " +microk8s.kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}' && echo || true + +echo -n "Calico Pod CIDR (from DS env): " +microk8s.kubectl -n kube-system get daemonset calico-node \ + -o jsonpath="{.spec.template.spec.containers[?(@.name=='calico-node')].env[?(@.name=='CALICO_IPV4POOL_CIDR')].value}" && echo || true + +echo "==> Detecting Calico version" +CALICO_VERSION=$(microk8s.kubectl -n kube-system get daemonset calico-node \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="calico-node")].image}' | \ + grep -oP 'v\d+\.\d+\.\d+' ) +echo "==> Found Calico version: ${CALICO_VERSION}" + +echo "==> Installing Calico API server (required for Submariner)" +# Create the namespace declaratively to avoid apply warning +microk8s.kubectl apply -f - < Generating TLS certificate for Calico API server" +openssl req -x509 -nodes -newkey rsa:4096 \ + -keyout /tmp/apiserver.key -out /tmp/apiserver.crt -days 365 -subj "/" \ + -addext "subjectAltName = DNS:calico-api.calico-apiserver.svc" 2>/dev/null + +microk8s.kubectl -n calico-apiserver create secret generic calico-apiserver-certs \ + --from-file=apiserver.key=/tmp/apiserver.key \ + --from-file=apiserver.crt=/tmp/apiserver.crt \ + --dry-run=client -o yaml | microk8s.kubectl apply -f - + +echo "==> Patching APIService to trust Calico certificate" +microk8s.kubectl patch apiservice v3.projectcalico.org --type=merge \ + -p "{\"spec\":{\"caBundle\":\"$(microk8s.kubectl get secret -n calico-apiserver calico-apiserver-certs -o go-template='{{ index .data "apiserver.crt" }}')\"}}" + +rm -f /tmp/apiserver.key /tmp/apiserver.crt + +echo "==> Waiting for Calico API server to be ready" +sleep 60 +microk8s.kubectl -n calico-apiserver wait --for=condition=Ready pod -l apiserver=true --timeout=120s + +echo "" +echo "==> Done! MicroK8s is installed and configured." +echo "==> IMPORTANT: Run this command to activate microk8s group:" +echo " newgrp microk8s" +echo "" +echo "Then verify with:" +echo " kubectl get nodes" \ No newline at end of file diff --git a/deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh b/deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh new file mode 100644 index 0000000..abc8885 --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/remote-node/setup-tools.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo "Please run as root: sudo $0" >&2 + exit 1 +fi + +echo "==> Updating APT and installing base tools" +apt-get update -y +apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + jq \ + gnupg \ + apt-transport-https + +# Ensure snap is available (normally on Ubuntu it is) +if ! command -v snap >/dev/null 2>&1; then + apt-get install -y snapd +fi + +# Ensure /snap/bin is in PATH for all shells +if ! grep -q "/snap/bin" /etc/environment; then + echo "==> Adding /snap/bin to PATH" + sed -i 's#^PATH=\(.*\)$#PATH=\1:/snap/bin#' /etc/environment || true +fi + +# Install Helm via snap (classic) +if ! command -v helm >/dev/null 2>&1; then + echo "==> Installing Helm" + snap install helm --classic +fi + +# Install subctl (latest) +if ! command -v subctl >/dev/null 2>&1; then + echo "==> Installing subctl (latest)" + curl -Ls https://get.submariner.io | bash + # Move to /usr/local/bin if installed in ~/.local/bin + if [[ -f "/root/.local/bin/subctl" ]]; then + mv /root/.local/bin/subctl /usr/local/bin/subctl + chmod +x /usr/local/bin/subctl + fi +fi + +echo "==> Tools installed" +helm version || true +subctl version || true +echo "Note: kubectl will be available after MicroK8s installation (next step)" \ No newline at end of file diff --git a/deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml b/deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml new file mode 100644 index 0000000..ca70365 --- /dev/null +++ b/deployments/hybrid/federations/federation-Z/remote-node/submariner-values.yaml @@ -0,0 +1,67 @@ +--- +# Submariner values for direct Helm install +# Based on official chart: https://submariner.io/operations/deployment/helm/ +# +# Use with: +# helm install submariner-operator submariner-latest/submariner-operator \ +# --namespace submariner-operator --create-namespace \ +# --values submariner-values.yaml +# Broker configuration - populated via --set during helm install +broker: + server: mip-tds.chuv.cscs.ch:6443 + token: '' # Populated via --set-string broker.token="$(cat broker-token.txt)" (plain text) + ca: '' # Populated via --set-string broker.ca="$(cat broker-ca-base64.txt)" (base64-encoded) + namespace: submariner-k8s-broker + insecure: false # Set to true only if broker uses self-signed cert (not recommended) +# Submariner configuration +submariner: + clusterId: node3 + clusterCidr: 10.3.0.0/16 + serviceCidr: 10.152.185.0/24 + globalCidr: '' # Empty unless using Globalnet + natEnabled: true + cableDriver: libreswan # or "wireguard" or "vxlan" + serviceDiscovery: true + healthcheckEnabled: true + loadBalancerEnabled: false # false for remote cluster + debug: false +# IPSec configuration - PSK must match the main cluster's PSK +ipsec: + psk: '' # Populated via --set-string ipsec.psk="$(cat broker-psk.txt)" (extracted from main cluster) + natPort: 4500 + ikePort: 500 + forceUDPEncaps: false + debug: false +# Service discovery (Lighthouse) +serviceAccounts: + lighthouseAgent: + create: true + lighthouseCoreDns: + create: true +# Gateway configuration +gateway: + nattPort: 4500 + metricsPort: 8080 +# Resource constraints (MicroK8s-optimized) +resources: + gateway: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + routeAgent: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + lighthouse: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/projects/mip-infrastructure.yaml b/projects/mip-infrastructure.yaml index f308333..f62d5d1 100644 --- a/projects/mip-infrastructure.yaml +++ b/projects/mip-infrastructure.yaml @@ -50,5 +50,13 @@ spec: description: Full access to infrastructure resources policies: - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, - *, mip-argo-project-infrastructure/*, allow + get, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, + create, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, + update, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, + delete, mip-argo-project-infrastructure/*, allow + - p, proj:mip-argo-project-infrastructure:infrastructure-admin, applications, + sync, mip-argo-project-infrastructure/*, allow groups: [argocd-admins] diff --git a/projects/static/mip-common/mip-common.yaml b/projects/static/mip-common/mip-common.yaml index be3a23e..33b4af2 100644 --- a/projects/static/mip-common/mip-common.yaml +++ b/projects/static/mip-common/mip-common.yaml @@ -31,16 +31,8 @@ spec: kind: Namespace - group: '' kind: PersistentVolume - - group: rbac.authorization.k8s.io - kind: ClusterRole - - group: rbac.authorization.k8s.io - kind: ClusterRoleBinding - group: apiextensions.k8s.io kind: CustomResourceDefinition - - group: admissionregistration.k8s.io - kind: ValidatingAdmissionWebhook - - group: admissionregistration.k8s.io - kind: MutatingAdmissionWebhook - group: networking.k8s.io kind: IngressClass - group: cert-manager.io @@ -111,7 +103,15 @@ spec: - name: common-admin description: Full access to common resources policies: - - p, proj:mip-argo-project-common:common-admin, applications, *, mip-argo-project-common/*, + - p, proj:mip-argo-project-common:common-admin, applications, get, mip-argo-project-common/*, + allow + - p, proj:mip-argo-project-common:common-admin, applications, create, mip-argo-project-common/*, + allow + - p, proj:mip-argo-project-common:common-admin, applications, update, mip-argo-project-common/*, + allow + - p, proj:mip-argo-project-common:common-admin, applications, delete, mip-argo-project-common/*, + allow + - p, proj:mip-argo-project-common:common-admin, applications, sync, mip-argo-project-common/*, allow groups: [argocd-admins] - name: common-operator diff --git a/projects/static/mip-federations/mip-federations.yaml b/projects/static/mip-federations/mip-federations.yaml index 2b5dca1..50b9503 100644 --- a/projects/static/mip-federations/mip-federations.yaml +++ b/projects/static/mip-federations/mip-federations.yaml @@ -75,7 +75,15 @@ spec: - name: federation-admin description: Full access to federation resources policies: - - p, proj:mip-argo-project-federations:federation-admin, applications, *, + - p, proj:mip-argo-project-federations:federation-admin, applications, get, + mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, create, + mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, update, + mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, delete, + mip-argo-project-federations/*, allow + - p, proj:mip-argo-project-federations:federation-admin, applications, sync, mip-argo-project-federations/*, allow groups: [argocd-admins] - name: federation-developer diff --git a/projects/static/mip-security/mip-security.yaml b/projects/static/mip-security/mip-security.yaml index e0f1247..1350a3b 100644 --- a/projects/static/mip-security/mip-security.yaml +++ b/projects/static/mip-security/mip-security.yaml @@ -48,7 +48,15 @@ spec: - name: security-admin description: Full access to network security policies policies: - - p, proj:mip-argo-project-security:security-admin, applications, *, mip-argo-project-security/*, + - p, proj:mip-argo-project-security:security-admin, applications, get, mip-argo-project-security/*, + allow + - p, proj:mip-argo-project-security:security-admin, applications, create, + mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, update, + mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, delete, + mip-argo-project-security/*, allow + - p, proj:mip-argo-project-security:security-admin, applications, sync, mip-argo-project-security/*, allow groups: [argocd-admins] - name: security-operator diff --git a/projects/static/mip-shared-apps/mip-shared-apps.yaml b/projects/static/mip-shared-apps/mip-shared-apps.yaml index 45678d1..d23e583 100644 --- a/projects/static/mip-shared-apps/mip-shared-apps.yaml +++ b/projects/static/mip-shared-apps/mip-shared-apps.yaml @@ -74,14 +74,22 @@ spec: - name: shared-apps-admin description: Full access to shared applications policies: - - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, *, - mip-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, get, + mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, create, + mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, update, + mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, delete, + mip-argo-project-shared-apps/*, allow + - p, proj:mip-argo-project-shared-apps:shared-apps-admin, applications, sync, + mip-argo-project-shared-apps/*, allow groups: [argocd-admins] - name: shared-apps-developer description: Limited access to shared applications policies: - p, proj:mip-argo-project-shared-apps:shared-apps-developer, applications, - sync, mip-shared-apps/*, allow + sync, mip-argo-project-shared-apps/*, allow - p, proj:mip-argo-project-shared-apps:shared-apps-developer, applications, - get, mip-shared-apps/*, allow + get, mip-argo-project-shared-apps/*, allow groups: [argocd-developers] diff --git a/projects/static/submariner/kustomization.yaml b/projects/static/submariner/kustomization.yaml new file mode 100644 index 0000000..aa4d507 --- /dev/null +++ b/projects/static/submariner/kustomization.yaml @@ -0,0 +1,4 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [submariner-project.yaml] diff --git a/projects/static/submariner/submariner-project.yaml b/projects/static/submariner/submariner-project.yaml new file mode 100644 index 0000000..766d95d --- /dev/null +++ b/projects/static/submariner/submariner-project.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: mip-argo-project-submariner + namespace: argocd-mip-team + # RBAC rules derived from: https://github.com/submariner-io/submariner-charts/tree/main/charts +spec: + description: Project for Submariner deployment + sourceRepos: + - git@github.com:NeuroTech-Platform/mip-infra-staging.git + - https://submariner-io.github.io/submariner-charts/charts + destinations: + - namespace: submariner-k8s-broker + server: https://kubernetes.default.svc + - namespace: submariner-operator + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + - group: submariner.io + kind: Cluster + - group: submariner.io + kind: Endpoint + - group: submariner.io + kind: Gateway + - group: submariner.io + kind: ClusterGlobalEgressIP + - group: submariner.io + kind: GlobalEgressIP + namespaceResourceWhitelist: + - group: apps + kind: Deployment + - group: apps + kind: DaemonSet + - group: batch + kind: Job + - group: '' + kind: ServiceAccount + - group: '' + kind: Service + - group: '' + kind: ConfigMap + - group: '' + kind: Secret + - group: rbac.authorization.k8s.io + kind: Role + - group: rbac.authorization.k8s.io + kind: RoleBinding + - group: submariner.io + kind: Submariner + - group: submariner.io + kind: ServiceDiscovery + - group: submariner.io + kind: Broker + roles: + - name: submariner-admin + description: Admin access to Submariner resources + policies: + - p, proj:mip-argo-project-submariner:submariner-admin, applications, get, + mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, create, + mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, update, + mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, delete, + mip-argo-project-submariner/*, allow + - p, proj:mip-argo-project-submariner:submariner-admin, applications, sync, + mip-argo-project-submariner/*, allow + groups: [argocd-admins] diff --git a/projects/static/submariner/submariner-rbac-project.yaml b/projects/static/submariner/submariner-rbac-project.yaml new file mode 100644 index 0000000..c9a0c67 --- /dev/null +++ b/projects/static/submariner/submariner-rbac-project.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: mip-argo-project-submariner-rbac + namespace: argocd-mip-team +spec: + description: Privileged project for Submariner RBAC resources + sourceRepos: [git@github.com:NeuroTech-Platform/mip-infra-staging.git] + destinations: + - namespace: submariner-k8s-broker + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + namespaceResourceWhitelist: + - group: argoproj.io + kind: Application + roles: + - name: submariner-rbac-admin + description: Admin access to Submariner RBAC + policies: + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, + get, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, + create, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, + update, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, + delete, mip-argo-project-submariner-rbac/*, allow + - p, proj:mip-argo-project-submariner-rbac:submariner-rbac-admin, applications, + sync, mip-argo-project-submariner-rbac/*, allow + groups: [argocd-admins] diff --git a/projects/templates/federation/templates/appproject.yaml b/projects/templates/federation/templates/appproject.yaml index 782f816..9f6c1a2 100644 --- a/projects/templates/federation/templates/appproject.yaml +++ b/projects/templates/federation/templates/appproject.yaml @@ -35,7 +35,11 @@ spec: - name: federation-admin description: "Admin access to {{ .Values.federationName }} resources" policies: - - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, *, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, get, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, create, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, update, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, delete, mip-argo-project-{{ .Values.federationName }}/*, allow + - p, proj:mip-argo-project-{{ .Values.federationName }}:federation-admin, applications, sync, mip-argo-project-{{ .Values.federationName }}/*, allow groups: {{- range .Values.rbac.adminGroups }} - {{ . }}