diff --git a/AGENTS.md b/AGENTS.md index fd28152..9e96d64 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -394,6 +394,15 @@ quotas for runsc and Firecracker use this local-disk filestore. Without an explicit quota, runsc retains its configured memory-backed overlay while Firecracker creates its configured sparse ext4 default. +Terraform-managed Alibaba Cloud node pools instead attach a dedicated 300 GiB +ESSD by default, have ACK format it as XFS, and mount it at `/home/akernel`. +The Aliyun sandboxd configuration leaves `filestore_dir_size` unset and uses +`/home/akernel/filestore` directly, so writable layers and +`/home/akernel/checkpoints` share the native reflink-capable filesystem. Do +not set a bounded filestore size for this profile because that reintroduces a +loop-backed filesystem and disables the high-performance Firecracker C/R +layout. + The bundled node enables YuanRong's local-only sandbox snapshot data plane and stores checkpoint state under the persistent `/home/akernel/checkpoints` mount. RRT receives diff --git a/deploy/README.md b/deploy/README.md index afaedf4..e5dde05 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -313,6 +313,12 @@ through its own LoadBalancer when `install_monitor=true`. Set `install_dragonfly=true` to install the pinned official Dragonfly chart and inject its seed-client proxy into the node runtime configuration. +Terraform-managed Alibaba Cloud nodes also receive a dedicated 300 GiB XFS +disk mounted at `/home/akernel` by default. sandboxd consumes that native +filesystem directly for writable layers and local checkpoints, without a +loop-backed filestore. See the Aliyun guide for capacity, opt-out, and node +replacement details. + Only the AKernel all-in-one image is pushed to the registry selected by `make config`. etcd, Traefik, Grafana, Prometheus, Loki, Tempo, and BusyBox use their pinned official public images by default. Set the per-component image diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index a7e38e0..69f1915 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -312,6 +312,11 @@ node_pool_system_disk_size = 100 node_pool_data_disk_enabled = true node_pool_data_disk_category = "cloud_essd" node_pool_data_disk_size = 100 +node_pool_extra_data_disk_enabled = true +node_pool_extra_data_disk_category = "cloud_essd" +node_pool_extra_data_disk_size = 300 +node_pool_extra_data_disk_mount_path = "/home/akernel" +node_pool_extra_data_disk_fs_type = "xfs" node_pool_key_name = "${node_pool_key_name}" core_namespace = "akernel" diff --git a/deploy/terraform/aliyun/README.md b/deploy/terraform/aliyun/README.md index 80849cf..4dc3c19 100644 --- a/deploy/terraform/aliyun/README.md +++ b/deploy/terraform/aliyun/README.md @@ -112,6 +112,45 @@ terraform apply \ -var 'monitor_storage_class=alicloud-disk-essd' ``` +## AKernel node storage + +New Terraform-managed ACK node pools attach two data disks by default. The +first remains available to ACK for the container runtime. The second is a +dedicated 300 GiB ESSD that ACK formats as XFS and mounts at +`/home/akernel` before the node joins the cluster. The same storage layout is +applied to user-defined `extra_node_pools`; dedicated Dragonfly pools retain +their own storage configuration. + +AKernel stores both `/home/akernel/filestore` and +`/home/akernel/checkpoints` on that native XFS filesystem. The generated +sandboxd configuration intentionally leaves `filestore_dir_size` unset, so +sandboxd uses the directory directly instead of creating a loop-backed ext4 +or XFS filesystem. Keeping the writable layer and checkpoint artifacts on +the same reflink-capable filesystem enables the fast Firecracker checkpoint +and restore path. + +Change the dedicated disk capacity or category with +`node_pool_extra_data_disk_size` and +`node_pool_extra_data_disk_category`. Set +`node_pool_extra_data_disk_enabled=false` to opt out, for example when an +existing cluster already provisions `/home/akernel` itself. The option has no +effect when `create_cluster=false`; existing-cluster users must mount a +suitable host filesystem before deploying the chart. An explicit ext4 +override remains supported for compatibility, but it cannot provide the XFS +reflink checkpoint path. + +Changing these settings does not migrate live sandbox data on existing +nodes. Before replacing an existing node pool, drain its sandboxes and remove +or relocate lifecycle-bound checkpoints. Review the Terraform plan as the +default dedicated disk adds one cloud disk per AKernel node. + +After provisioning a node, verify the effective layout with: + +```bash +findmnt -no SOURCE,FSTYPE,TARGET /home/akernel +xfs_info /home/akernel | grep 'reflink=1' +``` + Install Dragonfly P2P distribution: ```bash diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index 3505976..9b63375 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -32,6 +32,10 @@ resource "null_resource" "input_validation" { condition = !(length(var.node_pool_key_name) > 0 && length(var.node_pool_login_password) > 0) error_message = "node_pool_key_name conflicts with node_pool_login_password. Set only one." } + precondition { + condition = !var.create_cluster || !(var.node_pool_extra_data_disk_enabled && var.node_home_use_csi_ephemeral) + error_message = "Dedicated node data disk and CSI ephemeral home storage are mutually exclusive. Disable one storage mode." + } precondition { condition = length(var.master_image_tag) > 0 && length(var.node_image_tag) > 0 error_message = "master_image_tag and node_image_tag must be set to a published AKernel image tag." @@ -87,78 +91,7 @@ locals { sandboxd_nat_backend = var.sandboxd_nat_backend max_user_namespaces = var.node_pool_max_user_namespaces }) - node_pool_bootstrap_user_data = var.node_pool_extra_data_disk_enabled ? join("\n", [ - local.node_pool_bootstrap_base, - <<-EOT - # Mount extra data disk - set -euo pipefail - MOUNT_PATH="${var.node_pool_extra_data_disk_mount_path}" - FS_TYPE="${var.node_pool_extra_data_disk_fs_type}" - MARKER_PATH="${var.node_pool_extra_data_disk_mount_path}/.storage-ready" - - if mountpoint -q "$MOUNT_PATH" 2>/dev/null; then - echo "[data-disk] $MOUNT_PATH is already mounted, skipping." - exit 0 - fi - - ROOT_SOURCE=$(findmnt -n -o SOURCE /) - ROOT_DEV="" - if [ -n "$ROOT_SOURCE" ]; then - ROOT_DEV=$(lsblk -ndo PKNAME "$ROOT_SOURCE" 2>/dev/null || true) - fi - - TARGET_DEV="" - for dev in /dev/vd[b-z] /dev/xvd[b-z] /dev/sd[b-z] /dev/nvme[0-9]n1 /dev/nvme[1-9][0-9]n1; do - [ -b "$dev" ] || continue - dev_name=$(basename "$dev") - if [ -n "$ROOT_DEV" ] && [ "$dev_name" = "$ROOT_DEV" ]; then - continue - fi - if lsblk -ln -o MOUNTPOINT "$dev" 2>/dev/null | grep -q '/'; then - continue - fi - if pvs "$dev" &>/dev/null 2>&1; then - continue - fi - if blkid "$dev" &>/dev/null; then - EXISTING_FS=$(blkid -s TYPE -o value "$dev" 2>/dev/null || true) - if [ "$EXISTING_FS" = "$FS_TYPE" ]; then - TARGET_DEV="$dev" - break - fi - continue - fi - TARGET_DEV="$dev" - break - done - - if [ -z "$TARGET_DEV" ]; then - echo "[data-disk] No available data disk found, skipping." - exit 0 - fi - - echo "[data-disk] Using device: $TARGET_DEV" - mkdir -p "$MOUNT_PATH" - - if ! blkid "$TARGET_DEV" &>/dev/null; then - echo "[data-disk] Formatting $TARGET_DEV with $FS_TYPE..." - if [ "$FS_TYPE" = "xfs" ]; then - mkfs.xfs -f "$TARGET_DEV" - else - mkfs.ext4 -F "$TARGET_DEV" - fi - fi - - mount -o defaults,noatime "$TARGET_DEV" "$MOUNT_PATH" - UUID=$(blkid -s UUID -o value "$TARGET_DEV") - if ! grep -q "$UUID" /etc/fstab 2>/dev/null; then - echo "UUID=$UUID $MOUNT_PATH $FS_TYPE defaults,noatime 0 0" >> /etc/fstab - fi - echo "[data-disk] Mounted $TARGET_DEV at $MOUNT_PATH ($FS_TYPE) successfully." - touch "$MARKER_PATH" - echo "[data-disk] Created readiness marker: $MARKER_PATH" - EOT - ]) : local.node_pool_bootstrap_base + node_pool_bootstrap_user_data = local.node_pool_bootstrap_base dragonfly_seed_user_data = var.dragonfly_seed_node_pool.mount_local_nvme ? join("\n", [ local.node_pool_bootstrap_base, templatefile("${path.module}/../shared/local-nvme-mount.sh.tftpl", { @@ -202,34 +135,32 @@ locals { traefik_internal_stats_image = length(var.traefik_internal_stats_image) > 0 ? var.traefik_internal_stats_image : "${local.acr_registry}/busybox:1.37.0-musl" core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { - acr_registry = local.acr_registry - acr_secret_enabled = local.acr_secret_enabled - acr_host = local.acr_host - acr_username = var.acr_username - acr_password = var.acr_password - etcd_image_repository = local.etcd_image_repo - etcd_image_tag = var.etcd_image_tag - master_image_repository = local.master_image_repo - master_image_tag = var.master_image_tag - node_image_repository = local.node_image_repo - node_image_tag = var.node_image_tag - traefik_image_repository = local.traefik_image_repo - traefik_image_tag = var.traefik_image_tag - iam_litebus_data_key = var.iam_litebus_data_key - enable_kruise = var.install_prereqs - master_service_type = (var.master_public_access_8888 && !var.traefik_enabled) ? var.master_service_type : "ClusterIP" - traefik_enabled = var.traefik_enabled - sandboxd_nat_backend = var.sandboxd_nat_backend - enable_runc = var.enable_runc - node_secret_create = var.node_secret_create - node_home_use_csi_ephemeral = var.node_home_use_csi_ephemeral - node_home_csi_storage_class = local.effective_node_home_csi_sc - node_home_csi_size = var.node_home_csi_size - node_pool_extra_data_disk_enabled = var.node_pool_extra_data_disk_enabled - node_storage_init_image = var.node_storage_init_image - node_storage_init_mount_path = var.node_pool_extra_data_disk_mount_path - oss_auths = local.oss_auths - registry_auths = local.registry_auths + acr_registry = local.acr_registry + acr_secret_enabled = local.acr_secret_enabled + acr_host = local.acr_host + acr_username = var.acr_username + acr_password = var.acr_password + etcd_image_repository = local.etcd_image_repo + etcd_image_tag = var.etcd_image_tag + master_image_repository = local.master_image_repo + master_image_tag = var.master_image_tag + node_image_repository = local.node_image_repo + node_image_tag = var.node_image_tag + traefik_image_repository = local.traefik_image_repo + traefik_image_tag = var.traefik_image_tag + iam_litebus_data_key = var.iam_litebus_data_key + enable_kruise = var.install_prereqs + master_service_type = (var.master_public_access_8888 && !var.traefik_enabled) ? var.master_service_type : "ClusterIP" + traefik_enabled = var.traefik_enabled + sandboxd_nat_backend = var.sandboxd_nat_backend + enable_runc = var.enable_runc + node_secret_create = var.node_secret_create + node_home_use_csi_ephemeral = var.node_home_use_csi_ephemeral + node_home_csi_storage_class = local.effective_node_home_csi_sc + node_home_csi_size = var.node_home_csi_size + node_host_disk_path = var.create_cluster && var.node_pool_extra_data_disk_enabled ? var.node_pool_extra_data_disk_mount_path : "/home/akernel" + oss_auths = local.oss_auths + registry_auths = local.registry_auths etcd_storage_class = local.effective_storage_class etcd_cpu = var.etcd_resources.cpu @@ -313,14 +244,19 @@ locals { tempo_pvc_size = var.tempo_resources.pvc_size }) + akernel_extra_node_pools = [for pool in var.extra_node_pools : merge(pool, { + use_akernel_data_disk = true + })] + # Dragonfly server pool (manager + scheduler) — fixed size, goes through extra node pools dragonfly_server_pool = var.install_dragonfly && var.dragonfly_server_node_pool.enabled ? [{ - name = var.dragonfly_server_node_pool.name - size = var.dragonfly_server_node_pool.size - instance_types = var.dragonfly_server_node_pool.instance_types - system_disk_size = var.dragonfly_server_node_pool.system_disk_size - data_disk_enabled = true - data_disk_size = var.dragonfly_server_node_pool.data_disk_size + name = var.dragonfly_server_node_pool.name + size = var.dragonfly_server_node_pool.size + instance_types = var.dragonfly_server_node_pool.instance_types + system_disk_size = var.dragonfly_server_node_pool.system_disk_size + data_disk_enabled = true + data_disk_size = var.dragonfly_server_node_pool.data_disk_size + use_akernel_data_disk = false labels = { (var.dragonfly_server_node_pool.node_label_key) = var.dragonfly_server_node_pool.node_label_value } @@ -332,7 +268,7 @@ locals { }] : [] # Seed pool is created as a dedicated resource (with autoscaling), not via extra - all_extra_node_pools = concat(var.extra_node_pools, local.dragonfly_server_pool) + all_extra_node_pools = concat(local.akernel_extra_node_pools, local.dragonfly_server_pool) dragonfly_values = var.install_dragonfly ? templatefile("${path.module}/values-dragonfly.yaml.tmpl", { storage_class = local.effective_storage_class @@ -504,8 +440,11 @@ resource "alicloud_cs_kubernetes_node_pool" "default_with_key" { dynamic "data_disks" { for_each = var.node_pool_extra_data_disk_enabled ? [1] : [] content { - category = var.node_pool_extra_data_disk_category - size = var.node_pool_extra_data_disk_size + category = var.node_pool_extra_data_disk_category + size = var.node_pool_extra_data_disk_size + auto_format = "true" + file_system = var.node_pool_extra_data_disk_fs_type + mount_target = var.node_pool_extra_data_disk_mount_path } } @@ -546,8 +485,11 @@ resource "alicloud_cs_kubernetes_node_pool" "default_with_password" { dynamic "data_disks" { for_each = var.node_pool_extra_data_disk_enabled ? [1] : [] content { - category = var.node_pool_extra_data_disk_category - size = var.node_pool_extra_data_disk_size + category = var.node_pool_extra_data_disk_category + size = var.node_pool_extra_data_disk_size + auto_format = "true" + file_system = var.node_pool_extra_data_disk_fs_type + mount_target = var.node_pool_extra_data_disk_mount_path } } @@ -589,6 +531,16 @@ resource "alicloud_cs_kubernetes_node_pool" "extra" { size = coalesce(each.value.data_disk_size, var.node_pool_data_disk_size) } } + dynamic "data_disks" { + for_each = each.value.use_akernel_data_disk && var.node_pool_extra_data_disk_enabled ? [1] : [] + content { + category = var.node_pool_extra_data_disk_category + size = var.node_pool_extra_data_disk_size + auto_format = "true" + file_system = var.node_pool_extra_data_disk_fs_type + mount_target = var.node_pool_extra_data_disk_mount_path + } + } key_name = length(var.node_pool_key_name) > 0 ? var.node_pool_key_name : null password = length(var.node_pool_key_name) == 0 && length(var.node_pool_login_password) > 0 ? var.node_pool_login_password : null diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index 5261a1b..3950033 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -56,23 +56,23 @@ node_pool_system_disk_size = 100 node_pool_data_disk_enabled = true node_pool_data_disk_category = "cloud_essd" node_pool_data_disk_size = 100 +node_pool_extra_data_disk_enabled = true +node_pool_extra_data_disk_category = "cloud_essd" +node_pool_extra_data_disk_size = 300 +node_pool_extra_data_disk_mount_path = "/home/akernel" +node_pool_extra_data_disk_fs_type = "xfs" # Set one of the following for node login. node_pool_key_name = "" # node_pool_login_password = "" -# --- Node /home/akernel storage mode (choose one) --- -# Default: hostPath from system disk (/home/akernel). No extra config needed. +# --- Alternative node /home/akernel storage modes --- +# The default above is a dedicated XFS cloud disk. Set +# node_pool_extra_data_disk_enabled=false to use the system-disk hostPath. # -# Option A: Extra data disk — dedicated cloud disk per node, mounted to /home/akernel via user_data. -# No PVC scheduling delay, scales with nodes. -# node_pool_extra_data_disk_enabled = true -# node_pool_extra_data_disk_category = "cloud_essd" -# node_pool_extra_data_disk_size = 300 -# node_pool_extra_data_disk_mount_path = "/home/akernel" -# node_pool_extra_data_disk_fs_type = "ext4" # ext4 or xfs -# -# Option B: CSI ephemeral PVC — dynamic cloud disk per pod. +# Alternatively, use a CSI ephemeral PVC per node pod. Disable the dedicated +# data disk when selecting this mode. +# node_pool_extra_data_disk_enabled = false # node_home_use_csi_ephemeral = true # node_home_csi_storage_class = "alicloud-disk-essd" # node_home_csi_size = "300Gi" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 531667b..0ce6d32 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -87,29 +87,6 @@ node: values: - dragonfly-seed - dragonfly-server -%{ if node_pool_extra_data_disk_enabled ~} - initContainers: - - name: wait-for-storage - image: "${node_storage_init_image}" - command: - - /bin/sh - - -c - - | - # NOTE: don't use `mountpoint -q ${node_storage_init_mount_path}` here. Inside this init - # container ${node_storage_init_mount_path} is a kubelet bind mount of the host hostPath, - # so it is *already* a mountpoint regardless of whether the host has finished mounting - # the data disk underneath. Poll the marker file written by the host user_data script - # (data-disk-mount.sh.tftpl) instead — it appears only after mkfs + mount succeed. - echo "[init] Waiting for storage at ${node_storage_init_mount_path}/.storage-ready ..." - while [ ! -f ${node_storage_init_mount_path}/.storage-ready ]; do - sleep 5 - done - echo "[init] Storage ready." - volumeMounts: - - name: home-disk - mountPath: ${node_storage_init_mount_path} - mountPropagation: HostToContainer -%{ endif ~} image: repository: "${node_image_repository}" tag: "${node_image_tag}" @@ -118,6 +95,7 @@ node: ephemeral-storage: "${node_ephemeral}" requests: ephemeral-storage: "${node_ephemeral}" + hostDiskPath: "${node_host_disk_path}" homeDisk: useEphemeralPVC: ${node_home_use_csi_ephemeral} storageClassName: "${node_home_csi_storage_class}" @@ -193,8 +171,10 @@ node: [plugin.runtime] image_lib_dir="/home/akernel/images" + # The Aliyun module mounts a dedicated filesystem at /home/akernel. + # Leave filestore_dir_size unset so sandboxd consumes the native XFS + # directory instead of creating a loop-backed bounded filesystem. filestore_dir="/home/akernel/filestore" - filestore_dir_size="200G" filestore_xfs_enabled=false loop_device_dir="/dev" overlay_tmpfs_size="10G" diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index 0f86b2c..40ba488 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -237,8 +237,8 @@ variable "node_pool_data_disk_size" { variable "node_pool_extra_data_disk_enabled" { type = bool - description = "Whether to attach an extra data disk for akernel hostPath storage (separate from the container runtime data disk)." - default = false + description = "Whether to attach and mount a dedicated data disk for AKernel hostPath storage, separate from the container runtime data disk." + default = true } variable "node_pool_extra_data_disk_category" { @@ -255,19 +255,29 @@ variable "node_pool_extra_data_disk_size" { variable "node_pool_extra_data_disk_mount_path" { type = string - description = "Path to auto-format and mount the extra data disk. Typically /home/akernel for akernel node hostPath storage." + description = "Path where ACK formats and mounts the dedicated AKernel data disk." default = "/home/akernel" + + validation { + condition = startswith(var.node_pool_extra_data_disk_mount_path, "/") && var.node_pool_extra_data_disk_mount_path != "/" + error_message = "node_pool_extra_data_disk_mount_path must be an absolute path other than /." + } } variable "node_pool_extra_data_disk_fs_type" { type = string - description = "Filesystem type for the extra data disk (ext4 or xfs)." - default = "ext4" + description = "Filesystem type for the dedicated AKernel data disk. XFS enables the high-performance reflink checkpoint path; ext4 is retained for compatibility." + default = "xfs" + + validation { + condition = contains(["ext4", "xfs"], var.node_pool_extra_data_disk_fs_type) + error_message = "node_pool_extra_data_disk_fs_type must be ext4 or xfs." + } } variable "node_storage_init_image" { type = string - description = "Image for the storage readiness init container. Only used when node_pool_extra_data_disk_enabled=true." + description = "Deprecated compatibility variable. ACK now formats and mounts the dedicated data disk before joining the node." default = "busybox:1.36" }