diff --git a/.github/vale/styles/config/vocabularies/Aiven/accept.txt b/.github/vale/styles/config/vocabularies/Aiven/accept.txt index cd8f39d28..5fe0918fb 100644 --- a/.github/vale/styles/config/vocabularies/Aiven/accept.txt +++ b/.github/vale/styles/config/vocabularies/Aiven/accept.txt @@ -23,6 +23,7 @@ Astacus australia Auth0 autojoin +autoscale autoscaler autoscaling autovacuum @@ -54,6 +55,7 @@ config Config Confluent connect +Cooldown Coralogix Couchbase CPU/MS diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md new file mode 100644 index 000000000..02c4f4c18 --- /dev/null +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -0,0 +1,267 @@ +--- +title: Enable autoscaling for Aiven for Apache Kafka® Diskless Topics +sidebar_label: Enable autoscaling +limited: true +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import ConsoleLabel from "@site/src/components/ConsoleIcons" +import {ConsoleIcon} from "@site/src/components/ConsoleIcons" +import RelatedPages from "@site/src/components/RelatedPages"; + +Autoscaling adjusts the capacity of your Aiven for Apache Kafka® service based on CPU usage. +It helps maintain performance during traffic spikes and reduces resource use when +demand is low. Autoscaling is available only for Diskless Topics services deployed in +Bring Your Own Cloud (BYOC) using a plan that is part of the supported autoscaling +plan set. + +## Why use autoscaling + +- **Maintain performance**: Services scale up during sustained high CPU load. +- **Control costs**: Services scale down when demand decreases. +- **Stay flexible**: You can set minimum and maximum plans. Scaling happens within + those limits. +- **Get notified**: You receive an email notification whenever autoscaling scales the + service up or down. + +## How autoscaling works + +Autoscaling monitors CPU usage across your Kafka service and adjusts the service plan +based on predefined thresholds. It supports plan-based scaling only for services that +use Diskless Topics in BYOC. + +- **Scale up**: When CPU usage is consistently high. +- **Scale down**: When CPU usage is consistently low. +- **Plan switching**: The service scales by switching to the next larger or smaller + autoscaling plan. +- **Scaling in groups**: Nodes are added or removed in fixed groups (for example, + three nodes at a time), not individually. +- **Cooldown period**: The autoscaler waits at least 10 minutes after each scaling action + before evaluating the service again. +- **Health check**: Scaling occurs only when all brokers are healthy and in the + **RUNNING** state. + +## Key considerations and limitations + +- **Service compatibility**: Autoscaling is only available for Kafka services using + Diskless Topics in BYOC that are created with a supported autoscaling plan. Only + specific plans are eligible for autoscaling. You must select one of these supported + plans when creating the service. +- **Limited availability**: Contact the [Aiven support team](mailto:support@aiven.io) to + request access and enable autoscaling for your project. +- **Enable only at service creation**: You must enable autoscaling when you + [create a Diskless Topics service in BYOC](/docs/products/diskless/howto/create-diskless-topic). + You cannot enable autoscaling for existing services. +- **Aiven Console limitation**: Autoscaling is not available in the Aiven Console. Use + the CLI, Terraform, or API. +- **Plan changes**: While autoscaling is enabled, you cannot change the service plan + manually. To move the service to another plan (for example, `business-*`), first + disable autoscaling. To re-enable autoscaling, the service must be on an autoscaling + plan. +- **Thresholds**: Autoscaling is based on predefined CPU thresholds. You cannot configure + these values. The only configurable limits are `min_plan` and `max_plan`. +- **Billing**: Autoscaling uses special autoscaling plans. When a service scales, you + are billed according to the new plan. Each plan change appears as a separate line item + on your invoice. + +## Prerequisites + +- Autoscaling access granted for your project by Aiven support +- Access to one of the following: + - [Aiven API](https://api.aiven.io/doc/) + - [Aiven CLI client](/docs/tools/cli) + - [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest) +- An Aiven for Apache Kafka® Diskless Topics (BYOC) service created with an autoscaling + plan. For example: + + - `` + - `` + + Example using the CLI: + + ```bash + avn service create kafka-autoscale-demo \ + --service-type kafka \ + --cloud google-europe-west3 \ + --plan + ``` + + This creates a Kafka service named `kafka-autoscale-demo` in the `google-europe-west3` + region using the `` plan. + +## Enable autoscaling + +To enable autoscaling, create an autoscaler integration endpoint and link it to your +Kafka service with Diskless Topics in BYOC. + + + + +Enable autoscaling with the [Aiven CLI](/docs/tools/cli): + +1. Create an autoscaler integration endpoint: + + ```bash + avn service integration-endpoint-create \ + --project YOUR_PROJECT_NAME \ + --endpoint-name kafka-autoscaler \ + --endpoint-type autoscaler_service + ``` + +1. Get the endpoint ID: + + ```bash + avn service integration-endpoint-list --project YOUR_PROJECT_NAME + ``` + +1. Link your Kafka service to the autoscaler endpoint: + + ```bash + avn service integration-create \ + --project YOUR_PROJECT_NAME \ + --integration-type autoscaler_service \ + --dest-endpoint-id ENDPOINT_ID \ + --source-service YOUR_SERVICE_NAME \ + --user-config-json '{"autoscaling":{"min_plan":"","max_plan":""}}' + ``` + + Parameters: + + - `autoscaling.min_plan`: The smallest plan the service can scale down to. + - `autoscaling.max_plan`: The largest plan the service can scale up to. + + + + +Use the +[`aiven_service_integration_endpoint`](https://registry.terraform.io/providers/aiven/aiven/latest/docs/resources/service_integration_endpoint) +and +[`aiven_service_integration`](https://registry.terraform.io/providers/aiven/aiven/latest/docs/resources/service_integration) +resources to enable autoscaling. + +```hcl +resource "aiven_service_integration_endpoint" "autoscaler" { + project = var.project_name + endpoint_name = "kafka-autoscaler" + endpoint_type = "autoscaler_service" +} + +resource "aiven_service_integration" "autoscaling" { + project = var.project_name + integration_type = "autoscaler_service" + source_service_name = aiven_kafka.kafka.service_name + dest_endpoint_id = aiven_service_integration_endpoint.autoscaler.id + + user_config = { + autoscaling = { + min_plan = "" + max_plan = "" + } + } +} +``` + +Parameters: + +- `min_plan`: The smallest plan the service can scale down to. +- `max_plan`: The largest plan the service can scale up to. + +:::note[Terraform limitation] +With the current Terraform design, you must first create your Kafka service with a plan. +After the service exists, create the autoscaler endpoint and integration in a +separate `terraform apply` step. +::: + + + + +Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): + +1. Create an autoscaler integration endpoint with `endpoint_type` set to `autoscaler_service`: + + ```bash + curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration_endpoint \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "endpoint_name": "kafka-autoscaler", + "endpoint_type": "autoscaler_service" + }' + ``` + +1. Link your Kafka service to the new endpoint by calling `ServiceIntegrationCreate`: + + ```bash + curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "source_service": "SERVICE_NAME", + "integration_type": "autoscaler_service", + "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", + "user_config": { + "autoscaling": { + "min_plan": "", + "max_plan": "" + } + } + }' + ``` + + Parameters: + + - `source_service`: The Kafka service to autoscale. + - `dest_endpoint_id`: The ID of the autoscaler endpoint you created. + - `user_config.autoscaling.min_plan`: The smallest plan the service can scale down to. + - `user_config.autoscaling.max_plan`: The largest plan the service can scale up to. + + + + +## Disable autoscaling + +To disable autoscaling, remove the autoscaler integration from your Kafka service +with Diskless Topics in BYOC. + + + + +List integrations to find the autoscaler integration ID: + +```bash +avn service integration-list --project PROJECT_NAME SERVICE_NAME +``` + +Then delete it: + +```bash +avn service integration-delete --project PROJECT_NAME INTEGRATION_ID +``` + + + + +1. Open your Terraform configuration. +1. Remove the `aiven_service_integration` resource for the autoscaler. +1. Apply the changes. + + + + +Delete the service integration that links your Kafka service to the autoscaler endpoint: + +```bash +curl --request DELETE \ + --url https://api.aiven.io/v1/project/{project_name}/integration/{integration_id} \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' +``` + + + + + + +- [Get started with Diskless Topics](/docs/products/diskless/get-started) diff --git a/sidebars.ts b/sidebars.ts index 999faeba6..f9fb7f3a7 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1233,7 +1233,10 @@ const sidebars: SidebarsConfig = { { type: 'category', label: 'How to', - items: ['products/diskless/howto/create-diskless-topic'], + items: [ + 'products/diskless/howto/create-diskless-topic', + 'products/diskless/howto/enable-autoscaling', + ], }, ], },