feat: Flux OpenEBS-Mayastor storage integration

ephemeral/instance.tf

@@ -31,3 +31,15 @@ resource "digitalocean_droplet" "cp" {
     local.prefix,
   ]
 }
+
+# mayastor volume
+resource "digitalocean_volume" "mayastor" {
+  region                  = "fra1"
+  name                    = "rnd-${local.prefix}-spectrum-cp"
+  size                    = "100"
+}
+
+resource "digitalocean_volume_attachment" "mayastor" {
+  droplet_id = digitalocean_droplet.cp[0].id
+  volume_id  = digitalocean_volume.mayastor.id
+}

ephemeral/main.tf

@@ -22,7 +22,8 @@ module "talos" {
         templatefile("${path.root}/patches/registry.yml", {
           docker_username = data.vault_generic_secret.docker.data.username,
           docker_password = data.vault_generic_secret.docker.data.password
-        })
+        },),
+        file("${path.root}/patches/mayastor.yml")
       ]
     },
   ]
@@ -45,5 +46,6 @@ module "spectrum" {
     DOMAIN          = "${local.prefix}.fluence.dev"
     PREFIX          = local.prefix
     LOADBALANCER_IP = digitalocean_droplet.cp[0].ipv4_address
+    DO_DISK         = digitalocean_volume.mayastor.name
   }
 }

ephemeral/patches/mayastor.yml

@@ -0,0 +1,27 @@
+machine:
+  kubelet:
+    extraArgs:
+      node-labels: openebs.io/engine=mayastor
+      max-pods: 500
+    extraMounts:
+            - destination: /var/local # Destination is the absolute path where the mount will be placed in the container.
+              type: bind # Type specifies the mount kind.
+              source: /var/local # Source specifies the source path of the mount.
+              options:
+                - bind
+                - rshared
+                - rw
+  sysctls:
+    vm.nr_hugepages: "1024"
+  nodeLabels:
+    openebs.io/engine: "mayastor"
+cluster:
+  apiServer:
+    admissionControl:
+      - name: PodSecurity
+        configuration:
+          apiVersion: pod-security.admission.config.k8s.io/v1beta1
+          kind: PodSecurityConfiguration
+          exemptions:
+            namespaces:
+              - openebs

flux/clusters/default/kustomization.yml

@@ -4,3 +4,4 @@ resources:
   - ../../core/local-path-provisioner
   - ../../core/metrics-server
   - ../../core/kubelet-serving-cert-approver
+  - ../../core/openebs

flux/clusters/ephemeral/kustomization.yml

@@ -11,7 +11,9 @@ resources:
   - ../../core/kubelet-serving-cert-approver
   - ../../components/kubevirt
   - ../../components/monitoring
+  - openebs-mayastor.yml
   - hubble-ingress.yml
   - grafana-ingress.yml
   #  - cilium-l2.yml
   - lightmare.yml
+  - openebs-do-pool.yml

flux/clusters/ephemeral/openebs-do-pool.yml

@@ -0,0 +1,25 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openebs-do-pool
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: openebs
+  interval: 1m
+  path: "./flux/components/openebs-do-pool"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: spectrum
+    namespace: flux-system
+  patches:
+    - patch: |-
+        - op: replace
+          path: /spec/disks
+          value: ["aio:///dev/disk/by-id/scsi-0DO_Volume_${DO_DISK}"]
+      target:
+        kind: DiskPool
+        name: ebs-pool
+        namespace: openebs

flux/clusters/ephemeral/openebs-mayastor.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openebs
+  namespace: flux-system
+spec:
+  interval: 1m
+  path: "./flux/core/openebs"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: spectrum
+    namespace: flux-system
+

flux/components/openebs-do-pool/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./manifests.yaml

flux/components/openebs-do-pool/manifests.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: mayastor
+parameters:
+  protocol: nvmf
+  repl: "1"
+provisioner: io.openebs.csi-mayastor
+---
+apiVersion: "openebs.io/v1beta2"
+kind: DiskPool
+metadata:
+  name: ebs-pool
+  namespace: openebs
+spec:
+  node: cp-0
+  disks: ["aio:///dev/disk/by-id/placeholder"]

flux/components/openebs-pool/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./manifests.yaml

flux/components/openebs-pool/manifests.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: spectrum-pool
+parameters:
+  protocol: nvmf
+  repl: "1"
+provisioner: io.openebs.csi-mayastor

flux/core/openebs/kustomization.yml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - repository.yml
+  - release.yml

flux/core/openebs/release.yml

@@ -0,0 +1,35 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: openebs
+  namespace: kube-system
+spec:
+  targetNamespace: openebs
+  install:
+    createNamespace: true
+  interval: 5m
+  chart:
+    spec:
+      chart: openebs
+      version: 4.1.3
+      sourceRef:
+        name: openebs
+        kind: HelmRepository
+        namespace: flux-system
+  values:
+    mayastor:
+      csi:
+        node:
+          initContainers:
+            enabled: false
+      etcd:
+        replicaCount: 1
+    engines:
+      replicated:
+        mayastor:
+          enabled: true
+      local:
+        lvm:
+          enabled: false
+        zfs:
+          enabled: false

flux/core/openebs/repository.yml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: openebs
+  namespace: flux-system
+spec:
+  interval: 5m
+  timeout: 3m
+  url: https://openebs.github.io/openebs