WIP add storage migration automation for Azure

helper/azure-pv-pvc-deletion.py

@@ -0,0 +1,58 @@
+# Get the list of all PVs in the cluster
+import subprocess
+import json
+
+list_of_pv = subprocess.run('kubectl get pv -o jsonpath="{range .items[*]}{.metadata.name}{\'\\n\'}{end}"', shell=True, capture_output=True, text=True)
+list_of_pv.check_returncode()  # Check if the command execution was successful
+
+# Split the PV names into a list
+list_of_pv = list_of_pv.stdout.strip().split("\n")
+
+# Loop through each PV and patch the reclaim policy
+for pv in list_of_pv:
+
+    # Trim leading and trailing spaces from the PV name
+    pv = pv.strip()
+
+    # Patch the PV with the new reclaim policy
+    patch_command = f'kubectl patch pv {pv} -p \'{{"spec":{{"persistentVolumeReclaimPolicy":"Delete"}}}}\''
+    subprocess.run(patch_command, shell=True)
+
+    # Print the status of the patched PV
+    print(f"Patched PV: {pv}")
+    get_command = f'kubectl get pv {pv} -o jsonpath="{{.spec.persistentVolumeReclaimPolicy}}"'
+    pv_status = subprocess.run(get_command, shell=True, capture_output=True, text=True)
+    pv_status.check_returncode()  # Check if the command execution was successful
+    pv_status = pv_status.stdout.strip()
+    print(pv_status)
+    print("")
+
+# Delete all PVs
+for pv in list_of_pv:
+    # Trim leading and trailing spaces from the PV name
+    pv = pv.strip()
+    # Delete all PVs
+    pv_delete_command = f'kubectl delete pv {pv} --grace-period=0 --force --wait=false'
+    subprocess.run(pv_delete_command, shell=True)
+    print(f"Deleted PV: {pv}")
+    pv_delete_after_command = f"kubectl patch pv {pv} -p '{{\"metadata\": {json.dumps({'finalizers': None})}}}'"
+    subprocess.run(pv_delete_after_command, shell=True)
+
+# Get the list of all PVCs in the cluster
+pvc_list_command = "kubectl get pvc --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{\"\\n\"}{end}'"
+pvc_list_output = subprocess.check_output(pvc_list_command, shell=True, text=True).strip()
+pvc_list = pvc_list_output.split('\n')
+print(pvc_list)
+
+# Delete all PVCs
+for pvc in pvc_list:
+    # Split PVC and namespace name
+    namespace, pvc_name = pvc.split('/')
+    print(f"PVC: {pvc_name} and namespace: {namespace}")
+    # Delete all PVC
+    patch_command = f"kubectl delete pvc {pvc_name} -n {namespace} --grace-period=0 --force --wait=false"
+    subprocess.run(patch_command, shell=True)
+    # Patch PVC to remove finalizers
+    pvc_delete_after_command = f"kubectl patch pvc {pvc_name} -n {namespace} -p '{{\"metadata\": {json.dumps({'finalizers': None})}}}'"
+    subprocess.run(pvc_delete_after_command, shell=True)
+    print(f"Deleted PVC: {pvc}")

helper/azure-storage-migration.py

@@ -0,0 +1,260 @@
+import argparse
+import subprocess
+import yaml
+import json
+import os
+
+# This script will perform below tasks
+
+# Problem:
+# K8s has deprecated tags `failure-domain.beta.kubernetes.io/zone` and `failure-domain.beta.kubernetes.io/region` to `topology.kubernetes.io/zone` and `topology.kubernetes.io/region` respectively.
+# This creates `volume node affinity conflict` as nodes does not have this tags and in PV manifests this fields are immutable.
+# Here is the link of upstream issue which is closed without proper solution. - https://github.com/Azure/AKS/issues/3563
+# Inspiration taken from official Azure doc - https://learn.microsoft.com/en-us/azure/aks/csi-migrate-in-tree-volumes
+
+# Note: This script will do not make any change to cluster.
+
+# Steps performed in script.
+# 1. Create separate YAML files of all PV and PVC in cluster.
+# 2. Modify those YAMLs to supported External Azure CSI formate and save them in seaparate files.
+# 3. Pass folder path to store original and modified yamls as an argument.
+
+
+
+# Function to create individual files for all PVs in the cluster and store it to defined folder.
+def create_pv_files(output_folder):
+
+    # Run the kubectl command to get the list of PersistentVolumes in JSON format
+    pv_cmd = "kubectl get pv -o json"
+    pv_output = subprocess.check_output(pv_cmd, shell=True).decode("utf-8")
+
+    # Convert the JSON output to YAML
+    pv_list = yaml.safe_load(json.dumps(json.loads(pv_output)))
+
+    # Iterate over the PVs and create a file for each one
+    for pv_data in pv_list["items"]:
+        pv_name = pv_data["metadata"]["name"]
+        file_name = f"{pv_name}.yaml"
+        file_path = os.path.join(output_folder, file_name)
+
+        # Create the file with the PV details in YAML format
+        with open(file_path, "w") as file:
+            yaml.dump(pv_data, file)
+
+        print(f"Created file {file_name} for PV {pv_name} in folder {output_folder}")
+
+# Function to modify PVs generated by above function in a new External Azure CSI format.
+def modify_pv_files(input_folder, output_folder):
+
+    # List all files in the input folder
+    files = os.listdir(input_folder)
+
+    # Iterate over the files and modify the PV data
+    for file_name in files:
+        file_path = os.path.join(input_folder, file_name)
+
+        # Read the source YAML file
+        with open(file_path) as file:
+            data = yaml.safe_load(file)
+
+        # Modify the data according to the desired output
+        modified_data = {
+            "apiVersion": "v1",
+            "kind": "PersistentVolume",
+            "metadata": {
+                "annotations": {
+                    "pv.kubernetes.io/provisioned-by": "disk.csi.azure.com"
+                },
+                "name": data["metadata"]["name"],
+                "labels": data["metadata"].get("labels", {})
+            },
+            "spec": {
+                "accessModes": data["spec"]["accessModes"],
+                "capacity": {
+                    "storage": data["spec"]["capacity"]["storage"],
+                },
+                "claimRef": {
+                    "apiVersion": "v1",
+                    "kind": "PersistentVolumeClaim",
+                    "name": data["spec"]["claimRef"]["name"],
+                    "namespace": data["spec"]["claimRef"]["namespace"]
+                },
+                "csi": {
+                    "driver": "disk.csi.azure.com",
+                    "volumeAttributes": {
+                        "csi.storage.k8s.io/pv/name": data["metadata"]["name"],
+                        "csi.storage.k8s.io/pvc/name": data["spec"]["claimRef"]["name"],
+                        "csi.storage.k8s.io/pvc/namespace": data["spec"]["claimRef"]["namespace"],
+                        "requestedsizegib": data["spec"]["capacity"]["storage"],
+                        "skuname": "Standard_LRS"
+                    },
+                    "volumeHandle": data.get("csi", {}).get("volumeHandle", "")
+                },
+                "persistentVolumeReclaimPolicy": "Delete",
+                "storageClassName": "kubermatic-backup",
+                "nodeAffinity": {
+                    "required": {
+                        "nodeSelectorTerms": [
+                            {
+                                "matchExpressions": []
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+
+        if "labels" in modified_data["metadata"]:
+            labels = modified_data["metadata"]["labels"]
+            if "failure-domain.beta.kubernetes.io/region" in labels:
+                labels["topology.kubernetes.io/region"] = labels.pop("failure-domain.beta.kubernetes.io/region")
+            if "failure-domain.beta.kubernetes.io/zone" in labels:
+                labels["topology.kubernetes.io/zone"] = labels.pop("failure-domain.beta.kubernetes.io/zone")
+
+
+        # Check if 'nodeAffinity' exists and handle the case when it is missing
+        if "nodeAffinity" in data["spec"]:
+            if "matchExpressions" in data["spec"]["nodeAffinity"]["required"]["nodeSelectorTerms"][0]:
+                match_expressions = data["spec"]["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"]
+                modified_match_expressions = []
+                for expression in match_expressions:
+                    if expression["key"] == "failure-domain.beta.kubernetes.io/region":
+                        expression["key"] = "topology.kubernetes.io/region"
+                    elif expression["key"] == "failure-domain.beta.kubernetes.io/zone":
+                        expression["key"] = "topology.kubernetes.io/zone"
+                    modified_match_expressions.append(expression)
+                modified_data["spec"]["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"] = modified_match_expressions
+
+        # Modify storageClassName and sku value accordfingly
+        if data["spec"]["storageClassName"] == "kubermatic-fast":
+            modified_data["spec"]["storageClassName"] =  "kubermatic-fast"
+            modified_data["spec"]["csi"]["volumeAttributes"]["skuname"] =  "StandardSSD_LRS"
+
+        # Restore ReclaimPolicy
+        if data["spec"]["persistentVolumeReclaimPolicy"] == "Retain":
+            modified_data["spec"]["persistentVolumeReclaimPolicy"] =  "Retain"
+
+        # Check if 'azureDisk' exists and set 'volumeHandle' accordingly
+        if "azureDisk" in data["spec"]:
+            modified_data["spec"]["csi"]["volumeHandle"] = data["spec"]["azureDisk"]["diskURI"]
+        elif "csi" in data["spec"]:
+            modified_data["spec"]["csi"]["volumeHandle"] = data["spec"]["csi"]["volumeHandle"]
+
+        # Write the modified data to the output YAML file
+        output_file_path = os.path.join(output_folder, file_name)
+        with open(output_file_path, 'w') as file:
+            yaml.dump(modified_data, file)
+
+        print(f"Modified file {file_name} and saved to {output_file_path}")
+
+# Function to create individual files for all PVCs in the cluster and store it to defined folder.
+def create_pvc_files(output_folder):
+
+    # Run the kubectl command to get the list of PersistentVolumesClaim in JSON format
+    pvc_cmd = "kubectl get pvc --all-namespaces -o json"
+    pvc_output = subprocess.check_output(pvc_cmd, shell=True).decode("utf-8")
+
+    # Convert the JSON output to YAML
+    pvc_list = yaml.safe_load(json.dumps(json.loads(pvc_output)))
+
+    # Iterate over the PVCs and create a file for each one
+    for pvc_data in pvc_list["items"]:
+        pvc_name = pvc_data["metadata"]["namespace"] + "-" + pvc_data["metadata"]["name"]
+        file_name = f"{pvc_name}.yaml"
+        file_path = os.path.join(output_folder, file_name)
+
+        # Create the file with the PVC details in YAML format
+        with open(file_path, "w") as file:
+            yaml.dump(pvc_data, file)
+
+        print(f"Created file {file_name} for PVC {pvc_name} in folder {output_folder}")
+
+# Function to modify PVCs generated by above function in a new External Azure CSI format.
+def modify_pvc_files(input_folder, output_folder):
+    # Remove unwanted fields from the YAML
+    fields_to_remove = [
+        'kubectl.kubernetes.io/last-applied-configuration',
+        'pv.kubernetes.io/bound-by-controller',
+        'pv.kubernetes.io/migrated-to',
+        'volume.beta.kubernetes.io/storage-provisioner',
+        'volume.kubernetes.io/storage-provisioner',
+        'volume.kubernetes.io/storage-resizer',
+        'finalizers',
+        'pv.kubernetes.io/bind-completed',
+        'resourceVersion',
+        'uid',
+        'creationTimestamp'
+    ]
+
+    # List all files in the input folder
+    files = os.listdir(input_folder)
+
+    # Iterate over the files and modify the PV data
+    for file_name in files:
+        file_path = os.path.join(input_folder, file_name)
+
+        # Read the source YAML file
+        with open(file_path) as file:
+            data = yaml.safe_load(file)
+
+        for field in fields_to_remove:
+            data['metadata'].pop(field, None)
+
+        # Remove unwanted fields from the annotations
+        if 'annotations' in data['metadata']:
+            annotations = data['metadata']['annotations']
+            for field in fields_to_remove:
+                annotations.pop(field, None)
+        if 'status' in data:
+            data.pop('status')
+
+    # Write the modified data to the output YAML file
+        output_file_path = os.path.join(output_folder, file_name)
+        with open(output_file_path, 'w') as file:
+            yaml.dump(data, file)
+
+        print(f"Modified file {file_name} and saved to {output_file_path}")
+
+def main(pv_output_folder, pvc_output_folder, modified_pv_folder, modified_pvc_folder):
+
+    # Create the folders if it doesn't exist
+    folder_list = [pv_output_folder,pvc_output_folder,modified_pv_folder,modified_pvc_folder]
+    for folder in folder_list:
+        try:
+            os.makedirs(folder, exist_ok=True)
+            print(f"Created folder: {folder}")
+        except OSError as e:
+            print(f"Error creating folder {folder}: {e}")
+
+    create_pv_files(pv_output_folder)
+    modify_pv_files(pv_output_folder, modified_pv_folder)
+    create_pvc_files(pvc_output_folder)
+    modify_pvc_files(pvc_output_folder, modified_pvc_folder)
+
+    # Call the function to create PV files in the specified folder
+    create_pv_files(pv_output_folder)
+
+    # Call the function to modify PV files in the specified folders
+    modify_pv_files(pv_output_folder, modified_pv_folder)
+
+    # Call the function to create PVC files in the specified folder
+    create_pvc_files(pvc_output_folder)
+
+    # Call the function to modify PV files in the specified folders
+    modify_pvc_files(pvc_output_folder, modified_pvc_folder)
+
+if __name__ == '__main__':
+    #main()
+    parser = argparse.ArgumentParser(description='Convert PV and PVC YAML files')
+    parser.add_argument('--pv-output-folder', help='Folder path to store YAMLs for current PV')
+    parser.add_argument('--pvc-output-folder', help='Folder path to store YAMLs for current PVC')
+    parser.add_argument('--modified-pv-folder', help='Folder path to store modifying PV YAMLs')
+    parser.add_argument('--modified-pvc-folder', help='Folder path to store modifying PVC YAMLs')
+    args = parser.parse_args()
+
+    pv_output_folder = args.pv_output_folder
+    pvc_output_folder = args.pvc_output_folder
+    modified_pv_folder = args.modified_pv_folder
+    modified_pvc_folder = args.modified_pvc_folder
+
+    main(pv_output_folder, pvc_output_folder, modified_pv_folder, modified_pvc_folder)