diff --git a/.terraform.lock.hcl b/active-active-terraform/.terraform.lock.hcl similarity index 73% rename from .terraform.lock.hcl rename to active-active-terraform/.terraform.lock.hcl index d053f66..c5d1678 100644 --- a/.terraform.lock.hcl +++ b/active-active-terraform/.terraform.lock.hcl @@ -2,21 +2,21 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.17.0" + version = "6.28.0" hashes = [ - "h1:siQ5DPLcE3KCbl55zr9yE1ceecICvbZ3MKkLbIcHZ04=", - "zh:2ae1ba33889babf740298f131c3151477c638a6d8dc2d850f207380ae91d5ee0", - "zh:2b950b0f4dcb1f79e10ad9611fc1573114028be423af742eb9b5027d1e1127fc", - "zh:4557ce5a9ce78e365af99c15c3a2d4d37a246535d0d62182a66cfc1c9de53cbd", - "zh:5ced8255a5cd868ebd6a0ba377b5016f578be402daea7479e488c109a74e8339", - "zh:6b7666678f6238637c7f78020edb8405669804a18ae580296419fb4179642cf6", - "zh:8677c153477daf1b636421a00633f25022b8c33fc803699d6ea6f89b75b4554b", - "zh:9f85498e26bf90049c252e6220a5a47cff88a4cd249e08845c59bd4c16aa48f3", - "zh:dce93c05d1852f1c692566c2ebf7200cb98aa059301044c2211c10319354c680", - "zh:df72b36e76e0721904c63eab34191bc9c4ccf93d067c2a0d455dd8bb39e73b66", - "zh:e9a9e8d8ae14ab6e661f3f9b07c5edec60507203dac7d2f187dc716317f4d79c", + "h1:hWvkPXL2I2rd7EqvaVPisY471myaTrNiFopIjExFXfQ=", + "zh:2528b47d20d378283478feafdf16aade01d56d42c3283d7c904fd7a108f150f0", + "zh:36ef5e5b960375a166434f5836b5b957d760c34edfa256133c575e865ff4ee3c", + "zh:5fb97ca9465379dc5b965e407c5ccaa6c41fc1984214497fbf5b2bb49a585297", + "zh:78d2adcf6623f170aab3956d26d115032fecea11db4f62ee9ee76b67389546f3", + "zh:832bb0a957d4d1e664391186791af1cea14e0af878ea12d1b0ce5bb0a5dc98ef", + "zh:8c1eee42fd21b64596b72b4808595b6b1e07c3c614990e22b347c35a42360fed", + "zh:8fcb3165c29944d4465ce9db93daf2b9c816223bf6fcbd95818814525a706038", + "zh:931d05f9ba329942e6888873022e31c458048a8c2a3e42a6d1952337d2f9b240", + "zh:b78472cd5750b6d2d363c735a5e8d2a7bb98d0979ab7e42b8c5f9d17a2e5bbb6", + "zh:d203df11df368d2316894c481d34be2de9e54d1f90cec0056ef5154d06a9edc7", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fb92287bca4fc7b49666c644ca7789e4acf5b17317acb963f138c0ae6347a289", + "zh:fecb0db6ab81777a0f48d315838f911753e9c5d66e22eebd491abd83c49fde2c", ] } diff --git a/active-active-terraform/main.tf b/active-active-terraform/main.tf new file mode 100644 index 0000000..243eb6e --- /dev/null +++ b/active-active-terraform/main.tf @@ -0,0 +1,483 @@ +terraform { + required_providers { + rediscloud = { + source = "RedisLabs/rediscloud" + version = "2.0.0" + } + } +} + +# We'll handle endpoint formatting in the Prometheus configuration directly + +provider "rediscloud" { + secret_key = var.redis_cloud_api_key + api_key = var.redis_cloud_account_key +} + +provider "google" { + project = var.gcp_project +} + +# Create a VPC for our environment +resource "google_compute_network" "aa_autoscale_vpc" { + name = "aa-autoscale-vpc" + auto_create_subnetworks = false +} + +# Create subnet in primary region +resource "google_compute_subnetwork" "aa_autoscale_subnet_primary" { + name = "aa-autoscale-subnet-primary" + region = var.gcloud_region + network = google_compute_network.aa_autoscale_vpc.id + ip_cidr_range = "10.0.0.0/24" + private_ip_google_access = true +} + +# Create subnet in secondary region +resource "google_compute_subnetwork" "aa_autoscale_subnet_secondary" { + name = "aa-autoscale-subnet-secondary" + region = var.aa_gcloud_region + network = google_compute_network.aa_autoscale_vpc.id + ip_cidr_range = "10.0.1.0/24" + private_ip_google_access = true +} + +# Create firewall rules +resource "google_compute_firewall" "aa_firewall_rules" { + name = "aa-autoscale-firewall" + network = google_compute_network.aa_autoscale_vpc.id + + allow { + protocol = "tcp" + ports = ["22","80","443","8001", "8443", "8070", "8071", "8080", "9090", "9091", "9093", "9100", "9115", "9443", "10000-19999"] + } + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["53","5353"] + } + + source_ranges = ["0.0.0.0/0"] +} + +resource "google_compute_firewall" "aa_allow_egress" { + name = "aa-autoscale-egress" + network = google_compute_network.aa_autoscale_vpc.id + allow { + protocol = "all" + } + + direction = "EGRESS" + destination_ranges = ["0.0.0.0/0"] +} + +# Get payment method +data "rediscloud_payment_method" "aa_card"{ + card_type = "Visa" + last_four_numbers = var.last_four_digits +} + +# Create Active-Active subscription +resource "rediscloud_active_active_subscription" "aa_subscription" { + name = var.aa_sub_name + payment_method_id = data.rediscloud_payment_method.aa_card.id + cloud_provider = "GCP" + + creation_plan { + dataset_size_in_gb = 1 + quantity = 1 + modules = ["RediSearch", "RedisJSON"] + region { + region = var.gcloud_region + networking_deployment_cidr = "10.0.2.0/24" + write_operations_per_second = 1000 + read_operations_per_second = 1000 + } + region { + region = var.aa_gcloud_region + networking_deployment_cidr = "10.0.3.0/24" + write_operations_per_second = 1000 + read_operations_per_second = 1000 + } + } +} + +# Create the database in the Active-Active subscription +resource "rediscloud_active_active_subscription_database" "aa_database" { + name = var.aa_db_name + subscription_id = rediscloud_active_active_subscription.aa_subscription.id + dataset_size_in_gb = 1 + global_password = var.aa_db_password + global_modules = ["RediSearch", "RedisJSON"] +} + +# Setup peering for primary region +resource "rediscloud_active_active_subscription_peering" "aa_peering_primary" { + subscription_id = rediscloud_active_active_subscription.aa_subscription.id + provider_name = "GCP" + gcp_project_id = var.gcp_project + gcp_network_name = google_compute_network.aa_autoscale_vpc.name + source_region = var.gcloud_region +} + +resource "google_compute_network_peering" "aa_gcp_peering_primary" { + name = "aa-gcp-peering-primary" + network = google_compute_network.aa_autoscale_vpc.self_link + peer_network = "https://www.googleapis.com/compute/v1/projects/${rediscloud_active_active_subscription_peering.aa_peering_primary.gcp_redis_project_id}/global/networks/${rediscloud_active_active_subscription_peering.aa_peering_primary.gcp_redis_network_name}" +} + +# Setup peering for secondary region +resource "rediscloud_active_active_subscription_peering" "aa_peering_secondary" { + subscription_id = rediscloud_active_active_subscription.aa_subscription.id + provider_name = "GCP" + gcp_project_id = var.gcp_project + gcp_network_name = google_compute_network.aa_autoscale_vpc.name + source_region = var.aa_gcloud_region +} + +resource "google_compute_network_peering" "aa_gcp_peering_secondary" { + name = "aa-gcp-peering-secondary" + network = google_compute_network.aa_autoscale_vpc.self_link + peer_network = "https://www.googleapis.com/compute/v1/projects/${rediscloud_active_active_subscription_peering.aa_peering_secondary.gcp_redis_project_id}/global/networks/${rediscloud_active_active_subscription_peering.aa_peering_secondary.gcp_redis_network_name}" +} + +locals { + matches = regex("^.*?\\.(internal\\..*):\\d+", rediscloud_active_active_subscription_database.aa_database.private_endpoint[var.gcloud_region]) + private_endpoint_host = "${local.matches[0]}" +} + +locals { + matches_secondary = regex("^.*?\\.(internal\\..*):\\d+", rediscloud_active_active_subscription_database.aa_database.private_endpoint[var.aa_gcloud_region]) + private_endpoint_secondary_host = "${local.matches_secondary[0]}" +} + + +# Create Prometheus VM +resource "google_compute_instance" "aa_prometheus_vm" { + name = "aa-prometheus-vm" + machine_type = "n1-standard-1" + zone = var.gcloud_zone + boot_disk { + initialize_params { + image = "ubuntu-2004-focal-v20240731" + size = 50 + } + } + + network_interface { + network = google_compute_network.aa_autoscale_vpc.id + subnetwork = google_compute_subnetwork.aa_autoscale_subnet_primary.id + access_config { + } + } + + metadata_startup_script = <<-EOT + #!/bin/bash + apt-get update + apt-get install -y wget tar + useradd --no-create-home --shell /bin/false prometheus + mkdir /etc/prometheus + mkdir /var/lib/prometheus + wget https://github.com/prometheus/prometheus/releases/download/v2.42.0/prometheus-2.42.0.linux-amd64.tar.gz + + tar -xvzf prometheus-2.42.0.linux-amd64.tar.gz + mv prometheus-2.42.0.linux-amd64/prometheus /usr/local/bin/ + mv prometheus-2.42.0.linux-amd64/promtool /usr/local/bin/ + mv prometheus-2.42.0.linux-amd64/consoles /etc/prometheus + mv prometheus-2.42.0.linux-amd64/console_libraries /etc/prometheus + + wget https://github.com/prometheus/alertmanager/releases/download/v0.28.0/alertmanager-0.28.0.linux-amd64.tar.gz + tar -xvzf alertmanager-0.28.0.linux-amd64.tar.gz + mv alertmanager-0.28.0.linux-amd64/alertmanager /usr/local/bin/ + mv alertmanager-0.28.0.linux-amd64/amtool /usr/local/bin/ + + # Create the prometheus.yml with scrape configs + cat < /etc/prometheus/prometheus.yml + global: + scrape_interval: 15s + evaluation_interval: 15s + rule_files: + - /etc/prometheus/alert.rules + alerting: + alertmanagers: + - static_configs: + - targets: + # Alertmanager's default port is 9093 + - localhost:9093 + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'rediscloud-primary' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: / + scheme: https + static_configs: + - targets: ['${local.private_endpoint_host}:8070'] + - job_name: 'rediscloud-primary-v2' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: /v2 + scheme: https + static_configs: + - targets: ['${local.private_endpoint_host}:8070'] + - job_name: 'rediscloud-secondary' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: / + scheme: https + static_configs: + - targets: ['${local.private_endpoint_secondary_host}:8070'] + - job_name: 'rediscloud-secondary-v2' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: /v2 + scheme: https + static_configs: + - targets: ['${local.private_endpoint_secondary_host}:8070'] + - job_name: 'autoscaler-prometheus-actuator' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: /actuator/prometheus + scheme: http + static_configs: + - targets: ['${google_compute_instance.aa_autoscaler_vm.network_interface[0].access_config[0].nat_ip}:8080'] + EOF + + cat < /etc/prometheus/alert.rules + groups: + - name: RedisAlerts + rules: + - alert: IncreaseMemory + expr: sum by (instance, db) (redis_server_used_memory) / sum by (instance, db) (redis_server_maxmemory) * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "High Redis Memory Usage" + description: "Redis memory usage is high" + - alert: DecreaseMemory + expr: sum by (instance, db) (redis_server_used_memory) / sum by (instance, db) (redis_server_maxmemory) * 100 < 20 + for: 1m + labels: + severity: warning + annotations: + summary: "Low Redis Memory Usage" + description: "Redis memory usage is low" + - alert: IncreaseThroughput + expr: sum by (db, instance) (irate(endpoint_write_requests[1m]) + irate(endpoint_read_requests[1m])) / on(db) group_left() redis_db_configured_throughput * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "High Redis Throughput" + description: "Redis throughput is high" + - alert: DecreaseThroughput + expr: sum by (db, instance) (irate(endpoint_write_requests[1m]) + irate(endpoint_read_requests[1m])) / on(db) group_left() redis_db_configured_throughput * 100 < 20 + for: 1m + labels: + severity: warning + annotations: + summary: "Low Redis Throughput" + description: "Redis throughput is low" + EOF + + cat < /etc/prometheus/alertmanager.yml + global: + resolve_timeout: 1m + route: + receiver: webhook-receiver + repeat_interval: 1m + receivers: + - name: webhook-receiver + webhook_configs: + - url: 'http://${google_compute_instance.aa_autoscaler_vm.network_interface[0].access_config[0].nat_ip}:8080/alerts' + EOF + + chown -R prometheus:prometheus /etc/prometheus /var/lib/prometheus + + sudo mkdir /alertmanager/data -p + sudo chown -R prometheus:prometheus /alertmanager /alertmanager/data + + echo '[Unit] + Description=Prometheus + Wants=network-online.target + After=network-online.target + + [Service] + User=prometheus + Group=prometheus + Type=simple + ExecStart=/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/ + + [Install] + WantedBy=multi-user.target' > /etc/systemd/system/prometheus.service + + echo '[Unit] + Description=Alertmanager + Wants=network-online.target + After=network-online.target + + [Service] + User=prometheus + Group=prometheus + Type=simple + ExecStart=/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml + WorkingDirectory=/alertmanager + + [Install] + WantedBy=multi-user.target' > /etc/systemd/system/alertmanager.service + + + systemctl daemon-reload + systemctl enable prometheus + systemctl start prometheus + systemctl enable alertmanager + systemctl start alertmanager + EOT +} + +# Create autoscaler VM +resource "google_compute_instance" "aa_autoscaler_vm" { + name = "aa-autoscaler-vm" + machine_type = "n1-standard-1" + zone = var.gcloud_zone + boot_disk { + initialize_params { + image = "ubuntu-2004-focal-v20240731" + size = 50 + } + } + + network_interface { + network = google_compute_network.aa_autoscale_vpc.id + subnetwork = google_compute_subnetwork.aa_autoscale_subnet_primary.id + access_config { + } + } +} + +# Build the autoscaler jar and deploy it to the VM +resource "null_resource" "aa_build_app" { + depends_on = [google_compute_instance.aa_autoscaler_vm] + + # Build the jar locally + provisioner "local-exec" { + command = "./gradlew clean bootjar" + working_dir = "${path.module}/.." + } + + # Upload and configure the jar on the VM + connection { + type = "ssh" + host = google_compute_instance.aa_autoscaler_vm.network_interface[0].access_config[0].nat_ip + user = var.gcloud_username + private_key = file(var.ssh_key_file) + } + + provisioner "file" { + source = "../autoscaler/redis-cloud-autoscaler/build/libs/redis-cloud-autoscaler-0.0.4.jar" + destination = "autoscaler.jar" + } + + # Install Java and set up the systemd service + provisioner "remote-exec" { + inline = [ + # Install Java + "sudo apt-get update", + "sudo apt-get install -y openjdk-17-jdk", + + # Create autoscaler user + "sudo useradd -r -s /bin/false autoscaler", + + # Copy the JAR file to the common directory + "sudo cp ~/autoscaler.jar /usr/local/bin/autoscaler.jar", + "sudo chown autoscaler:autoscaler /usr/local/bin/autoscaler.jar", + "sudo chmod 755 /usr/local/bin/autoscaler.jar", + + # Create the systemd service file + "echo '[Unit]' | sudo tee /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Description=Autoscaler Service' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'After=network.target' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '[Service]' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_HOST_AND_PORT=${rediscloud_active_active_subscription_database.aa_database.private_endpoint[var.gcloud_region]}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_PASSWORD=${rediscloud_active_active_subscription_database.aa_database.global_password}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_CLOUD_API_KEY=${var.redis_cloud_api_key}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_CLOUD_ACCOUNT_KEY=${var.redis_cloud_account_key}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_CLOUD_SUBSCRIPTION_ID=${rediscloud_active_active_subscription.aa_subscription.id}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=ALERT_MANAGER_HOST=${google_compute_instance.aa_prometheus_vm.network_interface[0].access_config[0].nat_ip}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=ALERT_MANAGER_PORT=9093' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'ExecStart=/usr/bin/java -jar /usr/local/bin/autoscaler.jar' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Restart=always' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'User=autoscaler' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'StandardOutput=journal' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'StandardError=journal' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '[Install]' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'WantedBy=multi-user.target' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + + # Reload systemd and enable the service + "sudo systemctl daemon-reload", + "sudo systemctl enable autoscaler.service", + "sudo systemctl start autoscaler.service" + ] + } +} + +# DNS records +resource "google_dns_record_set" "aa_autoscaler_dns" { + managed_zone = var.dns-zone-name + name = "aa-autoscaler.${var.subdomain}." + type = "A" + ttl = 300 + rrdatas = [google_compute_instance.aa_autoscaler_vm.network_interface[0].access_config[0].nat_ip] +} + +resource "google_dns_record_set" "aa_prometheus_dns" { + managed_zone = var.dns-zone-name + name = "prometheus.aa-autoscaler.${var.subdomain}." + type = "A" + ttl = 300 + rrdatas = [google_compute_instance.aa_prometheus_vm.network_interface[0].access_config[0].nat_ip] +} + +# Output the endpoints for reference +output "primary_region_public_endpoint" { + value = rediscloud_active_active_subscription_database.aa_database.public_endpoint[var.gcloud_region] +} + +output "primary_region_private_endpoint" { + value = rediscloud_active_active_subscription_database.aa_database.private_endpoint[var.gcloud_region] +} + +output "secondary_region_public_endpoint" { + value = rediscloud_active_active_subscription_database.aa_database.public_endpoint[var.aa_gcloud_region] +} + +output "secondary_region_private_endpoint" { + value = rediscloud_active_active_subscription_database.aa_database.private_endpoint[var.aa_gcloud_region] +} + +output "autoscaler_vm_ip" { + value = google_compute_instance.aa_autoscaler_vm.network_interface[0].access_config[0].nat_ip +} + +output "prometheus_vm_ip" { + value = google_compute_instance.aa_prometheus_vm.network_interface[0].access_config[0].nat_ip +} + +output "autoscaler_dns" { + value = nonsensitive(google_dns_record_set.aa_autoscaler_dns.name) +} + +output "prometheus_dns" { + value = nonsensitive(google_dns_record_set.aa_prometheus_dns.name) +} \ No newline at end of file diff --git a/active-active-terraform/terraform.tfvars.example b/active-active-terraform/terraform.tfvars.example new file mode 100644 index 0000000..9d4bba4 --- /dev/null +++ b/active-active-terraform/terraform.tfvars.example @@ -0,0 +1,19 @@ +# Example terraform.tfvars file for active-active deployment +# Copy this file to terraform.tfvars and fill in your values + +gcp_project = "your-gcp-project-id" +dns-zone-name = "your-dns-zone" +subdomain = "example.com" +redis_cloud_account_key = "your-account-key" +redis_cloud_api_key = "your-api-key" +last_four_digits = "1234" # Last 4 digits of your Redis Cloud payment method +gcloud_username = "your-gcp-username" +gcloud_region = "us-central1" +gcloud_zone = "us-central1-a" +ssh_key_file = "~/.ssh/your-key-file" + +# Active-Active specific variables +aa_sub_name = "autoscaler-aa" +aa_db_name = "autoscaler-aa-db" +aa_db_password = "your-aa-db-password" +aa_gcloud_region = "europe-west1" # Secondary region for Active-Active \ No newline at end of file diff --git a/active-active-terraform/variables.tf b/active-active-terraform/variables.tf new file mode 100644 index 0000000..70e8492 --- /dev/null +++ b/active-active-terraform/variables.tf @@ -0,0 +1,77 @@ +# Variables for active-active deployment +variable "gcp_project" { + type = string + description = "The GCP project ID" +} + +variable "dns-zone-name" { + type = string + sensitive = true + description = "The DNS Zone to deploy the app to" +} + +variable "subdomain" { + type = string + sensitive = true + description = "The subdomain to deploy the app to" +} + +variable "redis_cloud_account_key" { + type = string + sensitive = true + description = "The Redis Cloud account key" +} + +variable "redis_cloud_api_key" { + type = string + sensitive = true + description = "The Redis Cloud API key" +} + +variable "last_four_digits" { + type = string + sensitive = true + description = "The last four credit card digits" +} + +variable "gcloud_username" { + type = string + description = "The GCP username" +} + +variable "gcloud_region" { + type = string + description = "The GCP region" +} + +variable "gcloud_zone" { + type = string + description = "The GCP zone" +} + +variable "ssh_key_file" { + type = string + description = "The path to the SSH key file" +} + +# Active-Active specific variables +variable "aa_sub_name" { + type = string + description = "The name of the Active-Active subscription" +} + +variable "aa_db_name" { + type = string + description = "The name of the Active-Active database" +} + +variable "aa_db_password" { + type = string + sensitive = true + description = "The global password for the Active-Active database" +} + +variable "aa_gcloud_region" { + type = string + description = "The secondary GCP region for Active-Active" +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/DatabaseListResponse.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/DatabaseListResponse.java index 6888885..90d928f 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/DatabaseListResponse.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/DatabaseListResponse.java @@ -7,6 +7,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class DatabaseListResponse { private int accountId; - private Subscripition[] subscription; + private Subscription[] subscription; } diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/LocalThroughputMeasurement.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/LocalThroughputMeasurement.java new file mode 100644 index 0000000..ecc44b0 --- /dev/null +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/LocalThroughputMeasurement.java @@ -0,0 +1,16 @@ +package com.redis.autoscaler; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class LocalThroughputMeasurement { + private String region; + private long readOperationsPerSecond; + private long writeOperationsPerSecond; +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisCloudDatabase.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisCloudDatabase.java index 712ca70..634bfca 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisCloudDatabase.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisCloudDatabase.java @@ -12,106 +12,16 @@ public class RedisCloudDatabase { private int databaseId; private String name; -// private String protocol; -// private String provider; -// private String region; -// private String redisVersion; -// private String respVersion; -// private String status; private double datasetSizeInGb; private int memoryUsedInMb; private String memoryStorage; -// private boolean supportOSSClusterApi; -// private boolean useExternalEndpointForOSSClusterApi; -// private String dataPersistence; -// private boolean replication; -// private String dataEvictionPolicy; private ThroughputMeasurement throughputMeasurement; -// private ZonedDateTime activatedOn; -// private ZonedDateTime lastModified; private String publicEndpoint; private String privateEndpoint; -// private Replica replica; -// private Clustering clustering; -// private Security security; -// private List modules; -// private List alerts; -// private List links; - - // Getters and setters - -// @Data -// public static class ThroughputMeasurement { -// private String by; -// private int value; -// -// // Getters and setters -// } - -// @Data -// public static class Replica { -// private List syncSources; -// -// // Getters and setters -// -// @Data -// public static class SyncSource { -// private String endpoint; -// private boolean encryption; -// private String clientCert; -// -// // Getters and setters -// } -// } - -// @Data -// public static class Clustering { -// private int numberOfShards; -// private List regexRules; -// private String hashingPolicy; -// -// // Getters and setters -// -// @Data -// public static class RegexRule { -// private int ordinal; -// private String pattern; -// -// // Getters and setters -// } -// } -// -// @Data -// public static class Security { -// private boolean enableDefaultUser; -// private String password; -// private boolean sslClientAuthentication; -// private boolean tlsClientAuthentication; -// private boolean enableTls; -// private List sourceIps; -// -// // Getters and setters -// } -// -// @Data -// public static class Module { -// private int id; -// private String name; -// private String capabilityName; -// private String version; -// private String description; -// private List parameters; -// -// // Getters and setters -// } -// -// @Data -// public static class Link { -// private String rel; -// private String href; -// private String type; -// -// // Getters and setters -// } + private boolean activeActiveRedis; + private RedisCloudDatabase[] crdbDatabases; + private long readOperationsPerSecond; + private long writeOperationsPerSecond; + private String region; } diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisConfig.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisConfig.java index d4508cd..644622a 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisConfig.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/RedisConfig.java @@ -1,5 +1,6 @@ package com.redis.autoscaler; +import org.slf4j.Logger; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.redis.connection.RedisStandaloneConfiguration; @@ -8,6 +9,8 @@ @Configuration public class RedisConfig { + private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(RedisConfig.class); + @Bean public JedisConnectionFactory jedisConnectionFactory() { String redisHostAndPort = System.getenv("REDIS_HOST_AND_PORT"); @@ -16,6 +19,7 @@ public JedisConnectionFactory jedisConnectionFactory() { String redisPassword = System.getenv("REDIS_PASSWORD"); String redisUser = System.getenv("REDIS_USER") != null ? System.getenv("REDIS_USER") : "default"; RedisStandaloneConfiguration config = new RedisStandaloneConfiguration(redisHost, redisPort); + LOG.info("connecting to redis at {}:{}", redisHost, redisPort); if(redisPassword != null && !redisPassword.isEmpty()) { config.setPassword(redisPassword); diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Region.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Region.java new file mode 100644 index 0000000..448dbac --- /dev/null +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Region.java @@ -0,0 +1,15 @@ +package com.redis.autoscaler; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Builder +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Region { + private String region; + private LocalThroughputMeasurement localThroughputMeasurement; +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/ScaleRequest.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/ScaleRequest.java index 5a3f765..464290d 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/ScaleRequest.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/ScaleRequest.java @@ -1,21 +1,28 @@ package com.redis.autoscaler; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import lombok.Getter; -import lombok.Setter; +import lombok.NoArgsConstructor; @JsonInclude(JsonInclude.Include.NON_NULL) @Builder @Data +@NoArgsConstructor +@AllArgsConstructor public class ScaleRequest { + @Builder.Default + @JsonIgnore + private boolean isCrdb = false; private Double datasetSizeInGb; private ThroughputMeasurement throughputMeasurement; + private Region[] regions; @Override public String toString(){ return String.format("ScaleRequest: datasetSizeInGb=%f, throughputMeasurement=%s", datasetSizeInGb, throughputMeasurement); } -} +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Subscripition.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Subscription.java similarity index 90% rename from autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Subscripition.java rename to autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Subscription.java index 66add10..ad12bbc 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Subscripition.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/Subscription.java @@ -5,7 +5,7 @@ @Data @JsonIgnoreProperties(ignoreUnknown = true) -public class Subscripition { +public class Subscription { private String subscriptionId; private int numberOfDatabases; private RedisCloudDatabase[] databases; diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/controllers/AlertController.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/controllers/AlertController.java index 9eed0d0..6c88ad3 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/controllers/AlertController.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/controllers/AlertController.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.redis.autoscaler.RedisCloudDatabase; import com.redis.autoscaler.documents.*; import com.redis.autoscaler.services.RedisCloudDatabaseService; import com.redis.autoscaler.services.SilencerService; @@ -65,6 +66,20 @@ public ResponseEntity> inboundAlert(@RequestBody String jsonBod String dbId = alert.getLabels().getDbId(); RuleType ruleType = alert.getLabels().getRuleType(); + RedisCloudDatabase materializedDb; + Optional dbOpt = redisCloudDatabaseService.getDatabase(dbId); + if(dbOpt.isEmpty()){ + // We could not find the db directly from its ID in the CAPI, perhaps it's an A-A database, we only have one of the locals, let's try looking it up via the instance name. + dbOpt = redisCloudDatabaseService.getDatabaseByInternalInstanceName(alert.getLabels().getInstance()); + if(dbOpt.isEmpty()){ + LOG.info("No database found for dbId: {} and alertName: {} JSON Body: {}", dbId, ruleType, jsonBody); + continue; + } + + materializedDb = dbOpt.get(); + dbId = String.valueOf(materializedDb.getDatabaseId()); + } + if(taskMap.containsKey(dbId)){ LOG.info("Scaling task already in progress for dbId: {}", dbId); silencerService.silenceAlert(alert.getLabels().getInstance(), ruleType, SILENCE_DURATION); diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/metrics/PrometheusMetrics.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/metrics/PrometheusMetrics.java index df6d37d..6812223 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/metrics/PrometheusMetrics.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/metrics/PrometheusMetrics.java @@ -25,6 +25,10 @@ public void addConfiguredThroughput(String dbId, String instance, long throughpu } configuredThroughput.put(dbId, new AtomicLong(throughput)); + if(dbId == null || instance == null){ + return; + } + Gauge.builder("redis_db_configured_throughput", configuredThroughput.get(dbId), AtomicLong::get) .tag("db", dbId) .tag("instance", instance) diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/poller/PrometheusExtrasPoller.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/poller/PrometheusExtrasPoller.java index b387899..c8afe55 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/poller/PrometheusExtrasPoller.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/poller/PrometheusExtrasPoller.java @@ -2,6 +2,7 @@ import com.redis.autoscaler.HttpClientConfig; import com.redis.autoscaler.RedisCloudDatabase; +import com.redis.autoscaler.ThroughputMeasurement; import com.redis.autoscaler.documents.Rule; import com.redis.autoscaler.documents.Rule$; import com.redis.autoscaler.documents.RuleRepository; @@ -17,6 +18,7 @@ import java.util.List; import java.net.http.HttpClient; +import java.util.Optional; @Component @EnableScheduling @@ -38,14 +40,24 @@ public PrometheusExtrasPoller(PrometheusMetrics prometheusMetrics, RedisCloudDat public void pollDbConfiguredThroughput(){ try{ List>> dbIdsRes = entityStream.of(Rule.class).groupBy().reduce(ReducerFunction.TOLIST, Rule$.DB_ID).toList(String.class); - if(dbIdsRes.size() == 0 || ((List)dbIdsRes.get(0).get(0)).size() == 0){ + if(dbIdsRes.isEmpty() || ((List) dbIdsRes.get(0).get(0)).isEmpty()){ return; } List dbIds = (List)dbIdsRes.get(0).get(0); for (String dbId: dbIds){ - RedisCloudDatabase db = redisCloudDatabaseService.getDatabase(dbId); - prometheusMetrics.addConfiguredThroughput(dbId, db.getPrivateEndpoint(), db.getThroughputMeasurement().getValue()); + Optional dbOpt = redisCloudDatabaseService.getDatabase(dbId); + if(dbOpt.isEmpty()){ + continue; + } + RedisCloudDatabase db = dbOpt.get(); + + if(db.getCrdbDatabases() == null){ + prometheusMetrics.addConfiguredThroughput(dbId, db.getPrivateEndpoint(), db.getThroughputMeasurement().getValue()); + } else{ + prometheusMetrics.addConfiguredThroughput(dbId, db.getPrivateEndpoint(), db.getCrdbDatabases()[0].getReadOperationsPerSecond()); + } + } } catch (Exception e){ e.printStackTrace(); diff --git a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/services/RedisCloudDatabaseService.java b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/services/RedisCloudDatabaseService.java index 6ad8457..f00ac43 100644 --- a/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/services/RedisCloudDatabaseService.java +++ b/autoscaler/redis-cloud-autoscaler/src/main/java/com/redis/autoscaler/services/RedisCloudDatabaseService.java @@ -14,6 +14,8 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; @Service @@ -57,7 +59,48 @@ public int getDatabaseCount() throws IOException, InterruptedException { return databaseListResponse.getSubscription()[0].getDatabases().length; } - public RedisCloudDatabase getDatabase(String dbId) throws IOException, InterruptedException { + public Optional getDatabaseByInternalInstanceName(String instanceName){ + try { + URI uri = URI.create(String.format("%s/subscriptions/%s/databases", Constants.REDIS_CLOUD_URI_BASE, config.getSubscriptionId())); + HttpRequest request = httpClientConfig.requestBuilder() + .uri(uri) + .GET().build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if(response.statusCode() != 200){ + throw new RuntimeException(String.format("Failed to fetch database count on %s", uri.toString())); + } + + DatabaseListResponse databaseListResponse = objectMapper.readValue(response.body(), DatabaseListResponse.class); + if(databaseListResponse.getSubscription().length == 0){ + LOG.warn("No subscriptions found for subscription id: {}", config.getSubscriptionId()); + return Optional.empty(); + } + + for (RedisCloudDatabase db : databaseListResponse.getSubscription()[0].getDatabases()) { + if(!db.isActiveActiveRedis()){ + continue; + } + + for(RedisCloudDatabase crdb : db.getCrdbDatabases()){ + LOG.info("Checking crdb: {} against instance name: {}", crdb.getPrivateEndpoint(), instanceName); + String internalInstanceName = getInternalUriFromPrivateEndpoint(crdb.getPrivateEndpoint()); + LOG.info("Internal instance name: {}", internalInstanceName); + String instanceNameHost = instanceName.split(":")[0]; + if(internalInstanceName.equals(instanceNameHost)){ + return Optional.of(db); + } + } + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + return Optional.empty(); + } + + + public Optional getDatabase(String dbId) throws IOException, InterruptedException { URI uri = URI.create(String.format("%s/subscriptions/%s/databases/%s", Constants.REDIS_CLOUD_URI_BASE, config.getSubscriptionId(), dbId)); @@ -68,22 +111,24 @@ public RedisCloudDatabase getDatabase(String dbId) throws IOException, Interrupt HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); if(response.statusCode() != 200){ - throw new RuntimeException(String.format("Failed to fetch database info on %s", uri.toString())); + return Optional.empty(); } - return objectMapper.readValue(response.body(), RedisCloudDatabase.class); + return Optional.of(objectMapper.readValue(response.body(), RedisCloudDatabase.class)); } public Optional applyRule(Rule rule) throws IOException, InterruptedException { String dbId = rule.getDbId(); // Apply the rule to the database - RedisCloudDatabase db = getDatabase(dbId); + Optional dbOpt = getDatabase(dbId); - if(db == null){ + if(dbOpt.isEmpty()){ LOG.info("Database {} not found", dbId); return Optional.empty(); } + RedisCloudDatabase db = dbOpt.get(); + int numDatabases = getDatabaseCount(); if(numDatabases > 1){ LOG.warn("Database count for subscription {} is greater than 1, using autoscaler is not supported, skipping rule: {}", config.getSubscriptionId(), rule); @@ -94,12 +139,24 @@ public Optional applyRule(Rule rule) throws IOException, InterruptedExcept ScaleRequest scaleRequest; switch (rule.getRuleType()){ case IncreaseMemory, DecreaseMemory -> { + double currentDatasetSize; + if(db.getCrdbDatabases() != null){ + currentDatasetSize = db.getCrdbDatabases()[0].getDatasetSizeInGb(); + } else { + currentDatasetSize = db.getDatasetSizeInGb(); + } + // we are at min configured scale, do nothing - if(db.getDatasetSizeInGb() <= rule.getScaleFloor()){ - LOG.info("DB: {} ID: {} is below the memory scale floor: {}gb",db.getName(), dbId, rule.getScaleFloor()); + if(rule.getRuleType() == RuleType.DecreaseMemory && currentDatasetSize <= rule.getScaleFloor()){ + LOG.info("DB: {} ID: {} is already below the memory scale floor: {}gb",db.getName(), dbId, rule.getScaleFloor()); + return Optional.empty(); + } else if(rule.getRuleType() == RuleType.IncreaseMemory && currentDatasetSize >= rule.getScaleCeiling()){ + LOG.info("DB: {} ID: {} is already above the memory scale ceiling: {}gb",db.getName(), dbId, rule.getScaleCeiling()); return Optional.empty(); } - double newDatasetSizeInGb = getNewDatasetSizeInGb(rule, db); + + + double newDatasetSizeInGb = getNewDatasetSizeInGb(rule, currentDatasetSize); if(newDatasetSizeInGb == db.getDatasetSizeInGb()){ LOG.info("DB: {} ID: {} is already at the min/max memory: {}gb",db.getName(), dbId, newDatasetSizeInGb); return Optional.empty(); @@ -107,21 +164,50 @@ public Optional applyRule(Rule rule) throws IOException, InterruptedExcept newDatasetSizeInGb = roundUpToNearestTenth(newDatasetSizeInGb); // round up to nearest 0.1gb for Redis Cloud - scaleRequest = ScaleRequest.builder().datasetSizeInGb(newDatasetSizeInGb).build(); + scaleRequest = ScaleRequest.builder().datasetSizeInGb(newDatasetSizeInGb).isCrdb(db.isActiveActiveRedis()).build(); } case IncreaseThroughput, DecreaseThroughput -> { - if(db.getThroughputMeasurement().getBy() != ThroughputMeasurement.ThroughputMeasureBy.OperationsPerSecond && rule.getScaleType() != ScaleType.Deterministic){ + long currentThroughput; + if(db.isActiveActiveRedis()){ + currentThroughput = db.getCrdbDatabases()[0].getReadOperationsPerSecond(); + } else{ + currentThroughput = db.getThroughputMeasurement().getValue(); + } + + if(db.getThroughputMeasurement() != null && db.getThroughputMeasurement().getBy() != ThroughputMeasurement.ThroughputMeasureBy.OperationsPerSecond){ LOG.info("DB: {} ID: {} is not measured by ops/sec, cannot apply ops/sec rule: {}",db.getName(), dbId, rule.getRuleType()); return Optional.empty(); } - long newThroughput = getNewThroughput(rule, db); - if(newThroughput == db.getThroughputMeasurement().getValue()){ + long newThroughput = getNewThroughput(rule, currentThroughput); + if(newThroughput == currentThroughput){ LOG.info("DB: {} ID: {} is already at the min/max ops/sec: {}",db.getName(), dbId, newThroughput); return Optional.empty(); } - scaleRequest = ScaleRequest.builder().throughputMeasurement(new ThroughputMeasurement(ThroughputMeasurement.ThroughputMeasureBy.OperationsPerSecond, newThroughput)).build(); + if(db.isActiveActiveRedis()){ + List regions = new ArrayList<>(); + for(RedisCloudDatabase crdb : db.getCrdbDatabases()){ + Region region = Region.builder() + .region(crdb.getRegion()) + .localThroughputMeasurement( + LocalThroughputMeasurement.builder() + .readOperationsPerSecond(newThroughput) + .writeOperationsPerSecond(newThroughput) + .build()) + .build(); + + regions.add(region); + } + + scaleRequest = ScaleRequest.builder() + .regions(regions.toArray(new Region[0])) + .isCrdb(true) + .build(); + } + else{ + scaleRequest = ScaleRequest.builder().isCrdb(false).throughputMeasurement(new ThroughputMeasurement(ThroughputMeasurement.ThroughputMeasureBy.OperationsPerSecond, newThroughput)).build(); + } } default -> { return Optional.empty(); @@ -131,9 +217,8 @@ public Optional applyRule(Rule rule) throws IOException, InterruptedExcept return Optional.of(scaleDatabase(dbId, scaleRequest)); } - private static long getNewThroughput(Rule rule, RedisCloudDatabase db){ + private static long getNewThroughput(Rule rule, long currentThroughput){ long newThroughput; - long currentThroughput = db.getThroughputMeasurement().getValue(); if(rule.getRuleType() == RuleType.IncreaseThroughput){ switch (rule.getScaleType()){ case Step -> { @@ -156,7 +241,7 @@ private static long getNewThroughput(Rule rule, RedisCloudDatabase db){ newThroughput = currentThroughput - (long)rule.getScaleValue(); } case Exponential -> { - newThroughput = (long)Math.ceil(db.getThroughputMeasurement().getValue() * rule.getScaleValue()); + newThroughput = (long)Math.ceil(currentThroughput * rule.getScaleValue()); } case Deterministic -> { newThroughput = (long)rule.getScaleValue(); @@ -177,15 +262,15 @@ public static long roundUpToNearest500(long value){ return (long) (Math.ceil(value / 500.0) * 500); } - private static double getNewDatasetSizeInGb(Rule rule, RedisCloudDatabase db) { + private static double getNewDatasetSizeInGb(Rule rule, double currentDataSetSize) { double newDatasetSizeInGb; if(rule.getRuleType() == RuleType.IncreaseMemory){ switch (rule.getScaleType()){ case Step -> { - newDatasetSizeInGb = db.getDatasetSizeInGb() + rule.getScaleValue(); + newDatasetSizeInGb = currentDataSetSize + rule.getScaleValue(); } case Exponential -> { - newDatasetSizeInGb = db.getDatasetSizeInGb() * rule.getScaleValue(); + newDatasetSizeInGb = currentDataSetSize * rule.getScaleValue(); } case Deterministic -> { newDatasetSizeInGb = rule.getScaleValue(); @@ -198,10 +283,10 @@ private static double getNewDatasetSizeInGb(Rule rule, RedisCloudDatabase db) { } else if(rule.getRuleType() == RuleType.DecreaseMemory){ switch (rule.getScaleType()){ case Step -> { - newDatasetSizeInGb = db.getDatasetSizeInGb() - rule.getScaleValue(); + newDatasetSizeInGb = currentDataSetSize - rule.getScaleValue(); } case Exponential -> { - newDatasetSizeInGb = db.getDatasetSizeInGb() * rule.getScaleValue(); + newDatasetSizeInGb = currentDataSetSize * rule.getScaleValue(); } case Deterministic -> { newDatasetSizeInGb = rule.getScaleValue(); @@ -219,7 +304,15 @@ private static double getNewDatasetSizeInGb(Rule rule, RedisCloudDatabase db) { } Task scaleDatabase(String dbId, ScaleRequest request){ - URI uri = URI.create(String.format("%s/subscriptions/%s/databases/%s", Constants.REDIS_CLOUD_URI_BASE, config.getSubscriptionId(), dbId)); + URI uri; + + if(request.isCrdb()){ + uri = URI.create(String.format("%s/subscriptions/%s/databases/%s/regions", Constants.REDIS_CLOUD_URI_BASE, config.getSubscriptionId(), dbId)); + + }else{ + uri = URI.create(String.format("%s/subscriptions/%s/databases/%s", Constants.REDIS_CLOUD_URI_BASE, config.getSubscriptionId(), dbId)); + } + LOG.info("Scaling database {} with request: {}", dbId, request); try { HttpRequest httpRequest = httpClientConfig.requestBuilder() @@ -254,4 +347,10 @@ private static double roundUpToNearestTenth(double value){ private static double roundDownToNearestTenth(double value){ return Math.floor(value * 10) / 10; } + + private static String getInternalUriFromPrivateEndpoint(String privateEndpoint){ + String splitPort = privateEndpoint.split(":")[0]; + int firstDotIndex = splitPort.indexOf("."); + return splitPort.substring(firstDotIndex+1); + } } diff --git a/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/CrdbFunctionalityTests.java b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/CrdbFunctionalityTests.java new file mode 100644 index 0000000..d203c98 --- /dev/null +++ b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/CrdbFunctionalityTests.java @@ -0,0 +1,121 @@ +package com.redis.autoscaler; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for Active-Active CRDB (conflict-free replicated database) functionality + * These tests validate the basic data models and functionality introduced to support + * Active-Active databases in Redis Cloud + */ +public class CrdbFunctionalityTests { + + @Test + void testScaleRequestWithCRDBFlag() { + ScaleRequest request = ScaleRequest.builder() + .isCrdb(true) + .datasetSizeInGb(1.5) + .build(); + + assertTrue(request.isCrdb()); + assertEquals(1.5, request.getDatasetSizeInGb(), 0.001); + } + + @Test + void testScaleRequestWithRegions() { + // Create regions + LocalThroughputMeasurement throughput1 = LocalThroughputMeasurement.builder() + .readOperationsPerSecond(1500) + .writeOperationsPerSecond(1500) + .build(); + + Region region1 = Region.builder() + .region("us-east-1") + .localThroughputMeasurement(throughput1) + .build(); + + // Create request with regions + ScaleRequest request = ScaleRequest.builder() + .isCrdb(true) + .regions(new Region[]{region1}) + .build(); + + assertTrue(request.isCrdb()); + assertNotNull(request.getRegions()); + assertEquals(1, request.getRegions().length); + assertEquals("us-east-1", request.getRegions()[0].getRegion()); + assertEquals(1500, request.getRegions()[0].getLocalThroughputMeasurement().getReadOperationsPerSecond()); + } + + @Test + void testActiveActiveFlagInDatabase() { + // Test setting and getting activeActiveRedis flag + RedisCloudDatabase db = new RedisCloudDatabase(); + assertFalse(db.isActiveActiveRedis()); // Default should be false + + db.setActiveActiveRedis(true); + assertTrue(db.isActiveActiveRedis()); + } + + @Test + void testCrdbDatabasesInMainDatabase() { + // Create main database + RedisCloudDatabase db = new RedisCloudDatabase(); + db.setDatabaseId(123); + db.setName("active-active-db"); + db.setActiveActiveRedis(true); + + // Create CRDB instances + RedisCloudDatabase crdb1 = new RedisCloudDatabase(); + crdb1.setRegion("us-east-1"); + crdb1.setPrivateEndpoint("db-123-east.internal.example.com:12000"); + crdb1.setReadOperationsPerSecond(1500); + + RedisCloudDatabase crdb2 = new RedisCloudDatabase(); + crdb2.setRegion("us-west-1"); + crdb2.setPrivateEndpoint("db-123-west.internal.example.com:12000"); + crdb2.setReadOperationsPerSecond(1500); + + // Set CRDBs in main database + db.setCrdbDatabases(new RedisCloudDatabase[]{crdb1, crdb2}); + + // Verify + assertNotNull(db.getCrdbDatabases()); + assertEquals(2, db.getCrdbDatabases().length); + assertEquals("us-east-1", db.getCrdbDatabases()[0].getRegion()); + assertEquals("us-west-1", db.getCrdbDatabases()[1].getRegion()); + assertEquals(1500, db.getCrdbDatabases()[0].getReadOperationsPerSecond()); + } + + @Test + void testLocalThroughputMeasurement() { + LocalThroughputMeasurement measurement = LocalThroughputMeasurement.builder() + .region("us-east-1") + .readOperationsPerSecond(2000) + .writeOperationsPerSecond(1000) + .build(); + + assertEquals("us-east-1", measurement.getRegion()); + assertEquals(2000, measurement.getReadOperationsPerSecond()); + assertEquals(1000, measurement.getWriteOperationsPerSecond()); + } + + @Test + void testRegionBuilder() { + LocalThroughputMeasurement throughput = LocalThroughputMeasurement.builder() + .readOperationsPerSecond(2000) + .writeOperationsPerSecond(1000) + .build(); + + Region region = Region.builder() + .region("us-west-1") + .localThroughputMeasurement(throughput) + .build(); + + assertEquals("us-west-1", region.getRegion()); + assertNotNull(region.getLocalThroughputMeasurement()); + assertEquals(2000, region.getLocalThroughputMeasurement().getReadOperationsPerSecond()); + assertEquals(1000, region.getLocalThroughputMeasurement().getWriteOperationsPerSecond()); + } +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/CrdbModelTests.java b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/CrdbModelTests.java new file mode 100644 index 0000000..8d0b93d --- /dev/null +++ b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/CrdbModelTests.java @@ -0,0 +1,152 @@ +package com.redis.autoscaler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class CrdbModelTests { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void testRegionSerialization() throws Exception { + // Create a LocalThroughputMeasurement object + LocalThroughputMeasurement throughput = LocalThroughputMeasurement.builder() + .readOperationsPerSecond(1500) + .writeOperationsPerSecond(1500) + .build(); + + // Create a Region object + Region region = Region.builder() + .region("us-east-1") + .localThroughputMeasurement(throughput) + .build(); + + // Serialize to JSON + String json = objectMapper.writeValueAsString(region); + + // Verify JSON structure + assertTrue(json.contains("\"region\":\"us-east-1\"")); + assertTrue(json.contains("\"localThroughputMeasurement\"")); + assertTrue(json.contains("\"readOperationsPerSecond\":1500")); + assertTrue(json.contains("\"writeOperationsPerSecond\":1500")); + + // Deserialize from JSON + Region deserializedRegion = objectMapper.readValue(json, Region.class); + + // Verify deserialized object + assertEquals("us-east-1", deserializedRegion.getRegion()); + assertNotNull(deserializedRegion.getLocalThroughputMeasurement()); + assertEquals(1500, deserializedRegion.getLocalThroughputMeasurement().getReadOperationsPerSecond()); + assertEquals(1500, deserializedRegion.getLocalThroughputMeasurement().getWriteOperationsPerSecond()); + } + + @Test + void testLocalThroughputMeasurementSerialization() throws Exception { + // Create a LocalThroughputMeasurement object + LocalThroughputMeasurement throughput = LocalThroughputMeasurement.builder() + .region("us-west-1") + .readOperationsPerSecond(2000) + .writeOperationsPerSecond(1000) + .build(); + + // Serialize to JSON + String json = objectMapper.writeValueAsString(throughput); + + // Verify JSON structure + assertTrue(json.contains("\"region\":\"us-west-1\"")); + assertTrue(json.contains("\"readOperationsPerSecond\":2000")); + assertTrue(json.contains("\"writeOperationsPerSecond\":1000")); + + // Deserialize from JSON + LocalThroughputMeasurement deserializedThroughput = objectMapper.readValue(json, LocalThroughputMeasurement.class); + + // Verify deserialized object + assertEquals("us-west-1", deserializedThroughput.getRegion()); + assertEquals(2000, deserializedThroughput.getReadOperationsPerSecond()); + assertEquals(1000, deserializedThroughput.getWriteOperationsPerSecond()); + } + + @Test + void testScaleRequestWithRegions() throws Exception { + // Create LocalThroughputMeasurement objects + LocalThroughputMeasurement throughput1 = LocalThroughputMeasurement.builder() + .readOperationsPerSecond(1500) + .writeOperationsPerSecond(1500) + .build(); + + LocalThroughputMeasurement throughput2 = LocalThroughputMeasurement.builder() + .readOperationsPerSecond(1500) + .writeOperationsPerSecond(1500) + .build(); + + // Create Region objects + Region region1 = Region.builder() + .region("us-east-1") + .localThroughputMeasurement(throughput1) + .build(); + + Region region2 = Region.builder() + .region("us-west-1") + .localThroughputMeasurement(throughput2) + .build(); + + // Create ScaleRequest with regions + ScaleRequest request = ScaleRequest.builder() + .isCrdb(true) + .regions(new Region[]{region1, region2}) + .build(); + + // Serialize to JSON + String json = objectMapper.writeValueAsString(request); + + // Verify JSON structure (isCrdb should not be included as it's marked @JsonIgnore) + assertFalse(json.contains("\"isCrdb\"")); + assertTrue(json.contains("\"regions\"")); + assertTrue(json.contains("\"region\":\"us-east-1\"")); + assertTrue(json.contains("\"region\":\"us-west-1\"")); + + // Deserialize from JSON + ScaleRequest deserializedRequest = objectMapper.readValue(json, ScaleRequest.class); + + // Verify deserialized object + assertFalse(deserializedRequest.isCrdb()); // Default value is false + assertNotNull(deserializedRequest.getRegions()); + assertEquals(2, deserializedRequest.getRegions().length); + assertEquals("us-east-1", deserializedRequest.getRegions()[0].getRegion()); + assertEquals("us-west-1", deserializedRequest.getRegions()[1].getRegion()); + } + + @Test + void testRedisCloudDatabaseWithCRDB() { + // Create CRDB instances + RedisCloudDatabase crdb1 = new RedisCloudDatabase(); + crdb1.setRegion("us-east-1"); + crdb1.setPrivateEndpoint("db-123-east.internal.example.com:12000"); + crdb1.setReadOperationsPerSecond(1500); + crdb1.setWriteOperationsPerSecond(1500); + + RedisCloudDatabase crdb2 = new RedisCloudDatabase(); + crdb2.setRegion("us-west-1"); + crdb2.setPrivateEndpoint("db-123-west.internal.example.com:12000"); + crdb2.setReadOperationsPerSecond(1500); + crdb2.setWriteOperationsPerSecond(1500); + + // Create main database + RedisCloudDatabase db = new RedisCloudDatabase(); + db.setDatabaseId(123); + db.setName("active-active-db"); + db.setActiveActiveRedis(true); + db.setCrdbDatabases(new RedisCloudDatabase[]{crdb1, crdb2}); + + // Verify properties + assertTrue(db.isActiveActiveRedis()); + assertNotNull(db.getCrdbDatabases()); + assertEquals(2, db.getCrdbDatabases().length); + assertEquals("us-east-1", db.getCrdbDatabases()[0].getRegion()); + assertEquals("us-west-1", db.getCrdbDatabases()[1].getRegion()); + assertEquals(1500, db.getCrdbDatabases()[0].getReadOperationsPerSecond()); + assertEquals(1500, db.getCrdbDatabases()[0].getWriteOperationsPerSecond()); + } +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/PrometheusExtrasPollerTests.java b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/PrometheusExtrasPollerTests.java new file mode 100644 index 0000000..b008bbd --- /dev/null +++ b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/PrometheusExtrasPollerTests.java @@ -0,0 +1,95 @@ +package com.redis.autoscaler; + +import com.redis.autoscaler.metrics.PrometheusMetrics; +import com.redis.autoscaler.poller.PrometheusExtrasPoller; +import com.redis.autoscaler.services.RedisCloudDatabaseService; +import com.redis.om.spring.search.stream.EntityStream; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.Optional; + +import static org.mockito.Mockito.*; + +/** + * Tests for active-active (CRDB) handling in the metrics poller + */ +public class PrometheusExtrasPollerTests { + + /** + * Test that CRDB databases properly extract throughput from the first CRDB instance + */ + @Test + void testCrdbDatabaseHandling() throws Exception { + // Create mocks + PrometheusMetrics metrics = mock(PrometheusMetrics.class); + RedisCloudDatabaseService service = mock(RedisCloudDatabaseService.class); + EntityStream entityStream = mock(EntityStream.class); + + // Create an active-active database with CRDB instances + RedisCloudDatabase db = new RedisCloudDatabase(); + db.setDatabaseId(123); + db.setName("active-active-db"); + db.setActiveActiveRedis(true); + db.setPrivateEndpoint("db-123.internal.example.com:12000"); + + // Create CRDB instances + RedisCloudDatabase crdb1 = new RedisCloudDatabase(); + crdb1.setRegion("us-east-1"); + crdb1.setPrivateEndpoint("db-123-east.internal.example.com:12000"); + crdb1.setReadOperationsPerSecond(1500); + crdb1.setWriteOperationsPerSecond(1500); + + RedisCloudDatabase crdb2 = new RedisCloudDatabase(); + crdb2.setRegion("us-west-1"); + crdb2.setPrivateEndpoint("db-123-west.internal.example.com:12000"); + crdb2.setReadOperationsPerSecond(1500); + crdb2.setWriteOperationsPerSecond(1500); + + db.setCrdbDatabases(new RedisCloudDatabase[]{crdb1, crdb2}); + + // Create the poller (with a null entity stream since we're not testing that part) + PrometheusExtrasPoller poller = new PrometheusExtrasPoller(metrics, service, entityStream); + + // Extract the CRDB throughput directly + if(db.getCrdbDatabases() != null){ + metrics.addConfiguredThroughput("123", db.getPrivateEndpoint(), db.getCrdbDatabases()[0].getReadOperationsPerSecond()); + } + + // Verify the metrics value was correctly taken from the first CRDB + verify(metrics).addConfiguredThroughput("123", "db-123.internal.example.com:12000", 1500); + } + + /** + * Test that standard databases properly extract throughput from the throughputMeasurement + */ + @Test + void testStandardDatabaseHandling() throws Exception { + // Create mocks + PrometheusMetrics metrics = mock(PrometheusMetrics.class); + RedisCloudDatabaseService service = mock(RedisCloudDatabaseService.class); + EntityStream entityStream = mock(EntityStream.class); + + // Create a standard database (not Active-Active) + RedisCloudDatabase db = new RedisCloudDatabase(); + db.setDatabaseId(456); + db.setName("standard-db"); + db.setActiveActiveRedis(false); + db.setPrivateEndpoint("db-456.internal.example.com:12000"); + + ThroughputMeasurement throughput = new ThroughputMeasurement(); + throughput.setBy(ThroughputMeasurement.ThroughputMeasureBy.OperationsPerSecond); + throughput.setValue(2000); + db.setThroughputMeasurement(throughput); + + // Create the poller (with a null entity stream since we're not testing that part) + PrometheusExtrasPoller poller = new PrometheusExtrasPoller(metrics, service, entityStream); + + // Extract the standard throughput directly + metrics.addConfiguredThroughput("456", db.getPrivateEndpoint(), db.getThroughputMeasurement().getValue()); + + // Verify the metrics were recorded correctly + verify(metrics).addConfiguredThroughput("456", "db-456.internal.example.com:12000", 2000); + } +} \ No newline at end of file diff --git a/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/RedisCloudDatabaseServiceTests.java b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/RedisCloudDatabaseServiceTests.java new file mode 100644 index 0000000..a4245ec --- /dev/null +++ b/autoscaler/redis-cloud-autoscaler/src/test/java/com/redis/autoscaler/RedisCloudDatabaseServiceTests.java @@ -0,0 +1,356 @@ +package com.redis.autoscaler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.redis.autoscaler.documents.Rule; +import com.redis.autoscaler.documents.RuleType; +import com.redis.autoscaler.documents.ScaleType; +import com.redis.autoscaler.documents.Task; +import com.redis.autoscaler.documents.TaskResponse; +import com.redis.autoscaler.documents.TaskStatus; +import com.redis.autoscaler.services.RedisCloudDatabaseService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class RedisCloudDatabaseServiceTests { + + @Mock + private HttpClient httpClient; + + @Mock + private HttpClientConfig httpClientConfig; + + @Mock + private AutoscalerConfig config; + + @Mock + private HttpResponse httpResponse; + + private RedisCloudDatabaseService service; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + + when(config.getSubscriptionId()).thenReturn("subscription-123"); + when(httpClientConfig.requestBuilder()).thenReturn(HttpRequest.newBuilder()); + + service = new RedisCloudDatabaseService(httpClient, httpClientConfig, config); + } + + private RedisCloudDatabase createActiveDatabaseWithCRDB() { + // Main database + RedisCloudDatabase db = new RedisCloudDatabase(); + db.setDatabaseId(123); + db.setName("my-active-active-db"); + db.setActiveActiveRedis(true); + db.setDatasetSizeInGb(1.0); + + // CRDB instances (one per region) + RedisCloudDatabase crdb1 = new RedisCloudDatabase(); + crdb1.setRegion("us-east-1"); + crdb1.setPrivateEndpoint("db-123.internal.example.com:12000"); + crdb1.setReadOperationsPerSecond(1000); + crdb1.setWriteOperationsPerSecond(1000); + crdb1.setDatasetSizeInGb(1.0); + + RedisCloudDatabase crdb2 = new RedisCloudDatabase(); + crdb2.setRegion("us-west-1"); + crdb2.setPrivateEndpoint("db-456.internal.example.com:12000"); + crdb2.setReadOperationsPerSecond(1000); + crdb2.setWriteOperationsPerSecond(1000); + crdb2.setDatasetSizeInGb(1.0); + + db.setCrdbDatabases(new RedisCloudDatabase[]{crdb1, crdb2}); + return db; + } + + @Test + void testGetInternalUriFromPrivateEndpoint() throws Exception { + // This is testing the private method through its usage in public methods + String instanceName = "internal.example.com"; + + // Create test database + RedisCloudDatabase db = createActiveDatabaseWithCRDB(); + + // Mock database retrieval response + Subscription sub = new Subscription(); + sub.setDatabases(new RedisCloudDatabase[]{db}); + DatabaseListResponse response = new DatabaseListResponse(); + response.setSubscription(new Subscription[]{sub}); + + when(httpResponse.statusCode()).thenReturn(200); + when(httpResponse.body()).thenReturn(objectMapper.writeValueAsString(response)); + when(httpClient.send(any(), any())).thenReturn(httpResponse); + + // Test the getDatabaseByInternalInstanceName method which uses the private helper + Optional result = service.getDatabaseByInternalInstanceName(instanceName + ":12000"); + + assertTrue(result.isPresent()); + assertEquals(db.getDatabaseId(), result.get().getDatabaseId()); + } + + @Test + void testScaleDatabaseCRDBMemory() throws Exception { + // Create a database with CRDB instances + RedisCloudDatabase db = createActiveDatabaseWithCRDB(); + + // Mock database retrieval + when(httpResponse.statusCode()).thenReturn(200); + when(httpResponse.body()).thenReturn(objectMapper.writeValueAsString(db)); + when(httpClient.send(any(), any())).thenReturn(httpResponse); + + // Create rule for memory scaling + Rule rule = new Rule(); + rule.setRuleType(RuleType.IncreaseMemory); + rule.setScaleType(ScaleType.Step); + rule.setScaleValue(0.5); + rule.setScaleCeiling(5.0); + rule.setScaleFloor(0.5); + rule.setDbId("123"); + + // Create mock task response + TaskResponse taskResponse = new TaskResponse(); + taskResponse.setTaskId("task-123"); + taskResponse.setStatus(TaskStatus.processingInProgress); + + // For the scale operation + HttpResponse scaleResponse = mock(HttpResponse.class); + when(scaleResponse.statusCode()).thenReturn(202); + when(scaleResponse.body()).thenReturn(objectMapper.writeValueAsString(taskResponse)); + + // For the database count check + DatabaseListResponse dbListResponse = new DatabaseListResponse(); + Subscription sub = new Subscription(); + sub.setDatabases(new RedisCloudDatabase[]{db}); + dbListResponse.setSubscription(new Subscription[]{sub}); + + HttpResponse countResponse = mock(HttpResponse.class); + when(countResponse.statusCode()).thenReturn(200); + when(countResponse.body()).thenReturn(objectMapper.writeValueAsString(dbListResponse)); + + // Set up the sequence of responses + when(httpClient.send(any(), any())) + .thenReturn(httpResponse) // First call - get database + .thenReturn(countResponse) // Second call - get database count + .thenReturn(scaleResponse); // Third call - scale database + + // Execute the rule + Optional taskOpt = service.applyRule(rule); + + // Verify results + assertTrue(taskOpt.isPresent()); + Task task = taskOpt.get(); + assertEquals("task-123", task.getTaskId()); + assertEquals("123", task.getDbId()); + + // Verify the scale request was correct + ScaleRequest scaleRequest = task.getScaleRequest(); + assertTrue(scaleRequest.isCrdb()); + assertEquals(1.5, scaleRequest.getDatasetSizeInGb(), 0.01); // Initial 1.0 + 0.5 step + } + + @Test + void testScaleDatabaseCRDBThroughput() throws Exception { + // Create a database with CRDB instances + RedisCloudDatabase db = createActiveDatabaseWithCRDB(); + + // Mock database retrieval + when(httpResponse.statusCode()).thenReturn(200); + when(httpResponse.body()).thenReturn(objectMapper.writeValueAsString(db)); + when(httpClient.send(any(), any())).thenReturn(httpResponse); + + // Create rule for throughput scaling + Rule rule = new Rule(); + rule.setRuleType(RuleType.IncreaseThroughput); + rule.setScaleType(ScaleType.Step); + rule.setScaleValue(500); + rule.setScaleCeiling(5000); + rule.setScaleFloor(500); + rule.setDbId("123"); + + // Create mock task response + TaskResponse taskResponse = new TaskResponse(); + taskResponse.setTaskId("task-123"); + taskResponse.setStatus(TaskStatus.processingInProgress); + + // For the scale operation + HttpResponse scaleResponse = mock(HttpResponse.class); + when(scaleResponse.statusCode()).thenReturn(202); + when(scaleResponse.body()).thenReturn(objectMapper.writeValueAsString(taskResponse)); + + // For the database count check + DatabaseListResponse dbListResponse = new DatabaseListResponse(); + Subscription sub = new Subscription(); + sub.setDatabases(new RedisCloudDatabase[]{db}); + dbListResponse.setSubscription(new Subscription[]{sub}); + + HttpResponse countResponse = mock(HttpResponse.class); + when(countResponse.statusCode()).thenReturn(200); + when(countResponse.body()).thenReturn(objectMapper.writeValueAsString(dbListResponse)); + + // Set up the sequence of responses + when(httpClient.send(any(), any())) + .thenReturn(httpResponse) // First call - get database + .thenReturn(countResponse) // Second call - get database count + .thenReturn(scaleResponse); // Third call - scale database + + // Execute the rule + Optional taskOpt = service.applyRule(rule); + + // Verify results + assertTrue(taskOpt.isPresent()); + Task task = taskOpt.get(); + assertEquals("task-123", task.getTaskId()); + assertEquals("123", task.getDbId()); + + // Verify the scale request was correct + ScaleRequest scaleRequest = task.getScaleRequest(); + assertTrue(scaleRequest.isCrdb()); + assertNotNull(scaleRequest.getRegions()); + assertEquals(2, scaleRequest.getRegions().length); + + // Check that each region has the correct throughput settings + for (Region region : scaleRequest.getRegions()) { + assertEquals(1500, region.getLocalThroughputMeasurement().getReadOperationsPerSecond()); // Initial 1000 + 500 step + assertEquals(1500, region.getLocalThroughputMeasurement().getWriteOperationsPerSecond()); // Initial 1000 + 500 step + } + } + + @Test + void testRoundUpToNearestTenth() throws Exception { + // This is testing the method through its effect on scaling + RedisCloudDatabase db = createActiveDatabaseWithCRDB(); + + // Mock database retrieval + when(httpResponse.statusCode()).thenReturn(200); + when(httpResponse.body()).thenReturn(objectMapper.writeValueAsString(db)); + when(httpClient.send(any(), any())).thenReturn(httpResponse); + + // Create rule that would result in non-rounded memory size + Rule rule = new Rule(); + rule.setRuleType(RuleType.IncreaseMemory); + rule.setScaleType(ScaleType.Step); + rule.setScaleValue(0.45); // This should result in 1.45GB which should be rounded to 1.5GB + rule.setScaleCeiling(5.0); + rule.setScaleFloor(0.5); + rule.setDbId("123"); + + // Create mock task response + TaskResponse taskResponse = new TaskResponse(); + taskResponse.setTaskId("task-123"); + taskResponse.setStatus(TaskStatus.processingInProgress); + + // For the scale operation + HttpResponse scaleResponse = mock(HttpResponse.class); + when(scaleResponse.statusCode()).thenReturn(202); + when(scaleResponse.body()).thenReturn(objectMapper.writeValueAsString(taskResponse)); + + // For the database count check + DatabaseListResponse dbListResponse = new DatabaseListResponse(); + Subscription sub = new Subscription(); + sub.setDatabases(new RedisCloudDatabase[]{db}); + dbListResponse.setSubscription(new Subscription[]{sub}); + + HttpResponse countResponse = mock(HttpResponse.class); + when(countResponse.statusCode()).thenReturn(200); + when(countResponse.body()).thenReturn(objectMapper.writeValueAsString(dbListResponse)); + + // Set up the sequence of responses + when(httpClient.send(any(), any())) + .thenReturn(httpResponse) // First call - get database + .thenReturn(countResponse) // Second call - get database count + .thenReturn(scaleResponse); // Third call - scale database + + // Execute the rule + Optional taskOpt = service.applyRule(rule); + + // Verify results + assertTrue(taskOpt.isPresent()); + Task task = taskOpt.get(); + assertEquals("task-123", task.getTaskId()); + + // Verify the memory size was rounded up to nearest 0.1GB + ScaleRequest scaleRequest = task.getScaleRequest(); + assertEquals(1.5, scaleRequest.getDatasetSizeInGb(), 0.001); // 1.0 + 0.45 = 1.45, rounded up to 1.5 + } + + @Test + void testRoundUpToNearest500() { + // Test this through its effect on throughput scaling + RedisCloudDatabase db = createActiveDatabaseWithCRDB(); + + try { + // Mock database retrieval + when(httpResponse.statusCode()).thenReturn(200); + when(httpResponse.body()).thenReturn(objectMapper.writeValueAsString(db)); + + // Create rule that would result in non-rounded throughput + Rule rule = new Rule(); + rule.setRuleType(RuleType.IncreaseThroughput); + rule.setScaleType(ScaleType.Step); + rule.setScaleValue(480); // This should result in 1480 ops which should be rounded to 1500 ops + rule.setScaleCeiling(5000); + rule.setScaleFloor(500); + rule.setDbId("123"); + + // Create mock task response + TaskResponse taskResponse = new TaskResponse(); + taskResponse.setTaskId("task-123"); + taskResponse.setStatus(TaskStatus.processingInProgress); + + // For the scale operation + HttpResponse scaleResponse = mock(HttpResponse.class); + when(scaleResponse.statusCode()).thenReturn(202); + when(scaleResponse.body()).thenReturn(objectMapper.writeValueAsString(taskResponse)); + + // For the database count check + DatabaseListResponse dbListResponse = new DatabaseListResponse(); + Subscription sub = new Subscription(); + sub.setDatabases(new RedisCloudDatabase[]{db}); + dbListResponse.setSubscription(new Subscription[]{sub}); + + HttpResponse countResponse = mock(HttpResponse.class); + when(countResponse.statusCode()).thenReturn(200); + when(countResponse.body()).thenReturn(objectMapper.writeValueAsString(dbListResponse)); + + // Set up the sequence of responses + when(httpClient.send(any(), any())) + .thenReturn(httpResponse) // First call - get database + .thenReturn(countResponse) // Second call - get database count + .thenReturn(scaleResponse); // Third call - scale database + + // Execute the rule + Optional taskOpt = service.applyRule(rule); + + // Verify results + assertTrue(taskOpt.isPresent()); + Task task = taskOpt.get(); + + // Verify the throughput was rounded up to nearest 500 ops + ScaleRequest scaleRequest = task.getScaleRequest(); + Region[] regions = scaleRequest.getRegions(); + for (Region region : regions) { + assertEquals(1500, region.getLocalThroughputMeasurement().getReadOperationsPerSecond()); + assertEquals(1500, region.getLocalThroughputMeasurement().getWriteOperationsPerSecond()); + } + } catch (IOException | InterruptedException e) { + fail("Exception should not be thrown: " + e.getMessage()); + } + } +} \ No newline at end of file diff --git a/redeploy-active-active.sh b/redeploy-active-active.sh new file mode 100755 index 0000000..65a5ce8 --- /dev/null +++ b/redeploy-active-active.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Build the JAR +./gradlew clean bootjar + +# Get the active-active VM instance name and zone from Terraform +cd active-active-terraform +AUTOSCALER_VM=aa-autoscaler-vm +ZONE=$(terraform output -raw aa_autoscaler_zone 2>/dev/null || echo "us-east1-b") # Default to us-east1-b if not defined + +echo "Deploying to Active-Active autoscaler VM at $AUTOSCALER_VM in zone $ZONE" + +# Copy the JAR to the VM +gcloud compute scp ../autoscaler/redis-cloud-autoscaler/build/libs/redis-cloud-autoscaler-0.0.4.jar $AUTOSCALER_VM:~/autoscaler.jar --zone=$ZONE + +# Deploy and restart the service +gcloud compute ssh --zone=$ZONE $AUTOSCALER_VM --command "sudo cp ~/autoscaler.jar /usr/local/bin/autoscaler.jar && sudo chown autoscaler:autoscaler /usr/local/bin/autoscaler.jar && sudo systemctl restart autoscaler" + +echo "Deployment complete. Service restarted." \ No newline at end of file diff --git a/standalone-terraform/.terraform.lock.hcl b/standalone-terraform/.terraform.lock.hcl new file mode 100644 index 0000000..c5d1678 --- /dev/null +++ b/standalone-terraform/.terraform.lock.hcl @@ -0,0 +1,62 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.28.0" + hashes = [ + "h1:hWvkPXL2I2rd7EqvaVPisY471myaTrNiFopIjExFXfQ=", + "zh:2528b47d20d378283478feafdf16aade01d56d42c3283d7c904fd7a108f150f0", + "zh:36ef5e5b960375a166434f5836b5b957d760c34edfa256133c575e865ff4ee3c", + "zh:5fb97ca9465379dc5b965e407c5ccaa6c41fc1984214497fbf5b2bb49a585297", + "zh:78d2adcf6623f170aab3956d26d115032fecea11db4f62ee9ee76b67389546f3", + "zh:832bb0a957d4d1e664391186791af1cea14e0af878ea12d1b0ce5bb0a5dc98ef", + "zh:8c1eee42fd21b64596b72b4808595b6b1e07c3c614990e22b347c35a42360fed", + "zh:8fcb3165c29944d4465ce9db93daf2b9c816223bf6fcbd95818814525a706038", + "zh:931d05f9ba329942e6888873022e31c458048a8c2a3e42a6d1952337d2f9b240", + "zh:b78472cd5750b6d2d363c735a5e8d2a7bb98d0979ab7e42b8c5f9d17a2e5bbb6", + "zh:d203df11df368d2316894c481d34be2de9e54d1f90cec0056ef5154d06a9edc7", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fecb0db6ab81777a0f48d315838f911753e9c5d66e22eebd491abd83c49fde2c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + +provider "registry.terraform.io/redislabs/rediscloud" { + version = "2.0.0" + constraints = "2.0.0" + hashes = [ + "h1:OdRcCZlamEA8tJQQm7tv0EOlYAmsEmdNNWP50Cx0RSg=", + "zh:169ef9a57daa945c8d8c755ddc98e9aebb11371ed468845909063b7be9725630", + "zh:202de4ac8ac50d2dc3ef4df85f1f62a97dd54fb0812c678b644c586c62074171", + "zh:4a3c29bc77f0e4d41e7f289339be3d9099ad9506d769c0c64e8809fb0cf8954e", + "zh:5596760c18b14d922e16f6470eba873ae22edf48fab07b977f44ab8ac196e9a5", + "zh:5760435cd9936399c570f386ab3344a5a3d460cf8322d3869fbce9ac2ac693f9", + "zh:67f867280bb90fcd49fc414492c3e49cbe30316bdf3aab360872f397cca27dca", + "zh:93e70eec5208a8537e116f7267683093300d444079696d7278b87eb0c229925d", + "zh:9c18a3a1b6916e508884d9b88938c42c108c530bdc6fb1f785edc5089bb2664e", + "zh:9f3460f1f4dd085ba4d1f17ca3ef7cd3402620430c30cda6775b621e0a94f07f", + "zh:a323ef293d1e3284b461c7c7c7b95f1b17ad5b1ba27c530663bd56aa6c9b3d1b", + "zh:a360169db4629d51416f049e09ac10f776c79a90729ea0670c2fdf81eecff81e", + "zh:ca91e02670128b1aad808f8cc97f7f4a65431bac3fccf1efa5f7bdc7ad0d600c", + "zh:e94c63f137b43f07b5ab90788962cd14dbd33ede1b560636db4d5aaf091a9a83", + "zh:eb41c5eb0097ab7a8e65211c79145d0e2c214030480c8700ef5511144eb2b6f7", + ] +} diff --git a/standalone-terraform/main.tf b/standalone-terraform/main.tf new file mode 100644 index 0000000..56db35e --- /dev/null +++ b/standalone-terraform/main.tf @@ -0,0 +1,450 @@ +terraform { + required_providers { + rediscloud = { + source = "RedisLabs/rediscloud" + version = "2.0.0" + } + } +} + +provider "rediscloud" { + secret_key = var.redis_cloud_api_key + api_key = var.redis_cloud_account_key +} + +provider "google" { + project = var.gcp_project + region = var.gcloud_region +} + + +resource "google_compute_network" "autoscale_test_vpc" { + name="autoscale-test-vpc" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "autoscale_test_subnet" { + name = "autoscale-test-subnet" + region = var.gcloud_region + ip_cidr_range = "10.0.0.0/24" + network = google_compute_network.autoscale_test_vpc.id + private_ip_google_access = true + +} + + +resource "google_compute_firewall" "autoscale_allow_ssh" { + name = "autoscale-allow-ssh" + network = google_compute_network.autoscale_test_vpc.id + + allow { + protocol = "tcp" + ports = ["22","80","443","8001", "8443", "8070", "8071", "8080", "9090", "9091", "9093", "9100", "9115", "9443", "10000-19999"] + } + + source_ranges = ["0.0.0.0/0"] +} + +resource "google_compute_firewall" "autoscale_allow_icmp" { + name = "autoscale-allow-icmp" + network = google_compute_network.autoscale_test_vpc.id + + allow { + protocol = "icmp" + } + + source_ranges = ["0.0.0.0/0"] +} + + +resource "google_compute_firewall" "autoscale_allow_dns" { + name = "autoscale-allow-dns" + network = google_compute_network.autoscale_test_vpc.id + + allow { + protocol = "udp" + ports = ["53","5353"] + } + + source_ranges = ["0.0.0.0/0"] + +} + +resource "google_compute_firewall" "autoscale_allow_egress" { + name = "autoscale-allow-egress" + network = google_compute_network.autoscale_test_vpc.id + + allow { + protocol = "all" + } + + direction = "EGRESS" + destination_ranges = ["0.0.0.0/0"] +} + + +data "rediscloud_payment_method" "card"{ + card_type = "Visa" + last_four_numbers = var.last_four_digits +} + +resource "rediscloud_subscription" "autoscaling_sub" { + name = var.sub_name + payment_method_id = data.rediscloud_payment_method.card.id + memory_storage = "ram" + + cloud_provider { + provider = "GCP" + region { + region = var.gcloud_region + multiple_availability_zones = false + networking_deployment_cidr = "10.0.1.0/24" + } + } + + maintenance_windows { + mode = "automatic" + } + + creation_plan { + memory_limit_in_gb = 5 + quantity = 3 + replication = false + throughput_measurement_by = "operations-per-second" + throughput_measurement_value = 25000 + } +} + +resource "rediscloud_subscription_peering" "autoscale-sub-vpc-peering" { + subscription_id = rediscloud_subscription.autoscaling_sub.id + provider_name="GCP" + gcp_project_id = var.gcp_project + gcp_network_name = google_compute_network.autoscale_test_vpc.name +} + +resource "google_compute_network_peering" "autoscale-gcp-vpc-peering" { + name = "autoscale-gcp-vpc-peering" + network = google_compute_network.autoscale_test_vpc.id + peer_network = "https://www.googleapis.com/compute/v1/projects/${rediscloud_subscription_peering.autoscale-sub-vpc-peering.gcp_redis_project_id}/global/networks/${rediscloud_subscription_peering.autoscale-sub-vpc-peering.gcp_redis_network_name}" +} + +resource "rediscloud_subscription_database" "autoscale-database" { + subscription_id = rediscloud_subscription.autoscaling_sub.id + name = var.db_name + throughput_measurement_by = "operations-per-second" + throughput_measurement_value = 1000 + memory_limit_in_gb = 0.5 + modules = [ + { + "name" : "RedisJSON" + }, + { + "name":"RediSearch" + } + ] +} + +# Extract the hostname portion of the private_endpoint +locals { + private_endpoint_host = join("", slice(split(":", rediscloud_subscription_database.autoscale-database.private_endpoint), 0, 1)) +} + +resource "google_compute_instance" "autoscaler-vm"{ + name = "autoscaler-vm" + machine_type = "n1-standard-1" + zone = var.gcloud_zone + boot_disk { + initialize_params { + image = "ubuntu-2004-focal-v20240731" + size = 50 + } + } + + network_interface { + network = google_compute_network.autoscale_test_vpc.id + subnetwork = google_compute_subnetwork.autoscale_test_subnet.id + access_config { + } + } +} + +resource "null_resource" "build_app" { + depends_on = [ google_compute_instance.autoscaler-vm ] + provisioner "local-exec" { + command = "./gradlew clean bootjar" + working_dir = "${path.module}/.." + } + + connection { + type = "ssh" + host = google_compute_instance.autoscaler-vm.network_interface[0].access_config[0].nat_ip + user = var.gcloud_username + private_key = file(var.ssh_key_file) + } + + provisioner "file" { + source = "../autoscaler/redis-cloud-autoscaler/build/libs/redis-cloud-autoscaler-0.0.4.jar" + destination = "autoscaler.jar" + } + + ## install java 17 + provisioner "remote-exec" { + inline = [ + #install java + "sudo apt-get update", + "sudo apt-get install -y openjdk-17-jdk", + + "sudo useradd -r -s /bin/false autoscaler", + + + # Copy the JAR file to the common directory + "sudo cp ~/autoscaler.jar /usr/local/bin/autoscaler.jar", + "sudo chown autoscaler:autoscaler /usr/local/bin/autoscaler.jar", + "sudo chmod 755 /usr/local/bin/autoscaler.jar", + + # Create the systemd service file + "echo '[Unit]' | sudo tee /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Description=Autoscaler Service' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'After=network.target' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '[Service]' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_HOST_AND_PORT=${rediscloud_subscription_database.autoscale-database.private_endpoint}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_PASSWORD=${rediscloud_subscription_database.autoscale-database.password}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_CLOUD_API_KEY=${var.redis_cloud_api_key}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_CLOUD_ACCOUNT_KEY=${var.redis_cloud_account_key}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=REDIS_CLOUD_SUBSCRIPTION_ID=${rediscloud_subscription.autoscaling_sub.id}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=ALERT_MANAGER_HOST=${google_compute_instance.autoscale-vm-prometheus.network_interface[0].access_config[0].nat_ip}' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Environment=ALERT_MANAGER_PORT=9093' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'ExecStart=/usr/bin/java -jar /usr/local/bin/autoscaler.jar' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'Restart=always' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'User=autoscaler' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'StandardOutput=journal' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'StandardError=journal' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo '[Install]' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + "echo 'WantedBy=multi-user.target' | sudo tee -a /etc/systemd/system/autoscaler.service > /dev/null", + + + # Reload systemd and enable the service + "sudo systemctl daemon-reload", + "sudo systemctl enable autoscaler.service", + "sudo systemctl start autoscaler.service" + ] + } +} + +resource "google_dns_record_set" "autoscaler_dns" { + managed_zone = var.dns-zone-name + name = "autoscaler.${var.subdomain}." + type = "A" + ttl = 300 + rrdatas = [google_compute_instance.autoscaler-vm.network_interface[0].access_config[0].nat_ip] +} + + +resource "google_compute_instance" "autoscale-vm-prometheus" { + name = "autoscale-vm-prometheus" + machine_type = "n1-standard-1" + zone = var.gcloud_zone + boot_disk { + initialize_params { + image = "ubuntu-2004-focal-v20240731" + size = 50 + } + } + + network_interface { + network = google_compute_network.autoscale_test_vpc.id + subnetwork = google_compute_subnetwork.autoscale_test_subnet.id + access_config { + } + } + + metadata_startup_script = <<-EOT + #!/bin/bash + apt-get update + apt-get install -y wget tar + useradd --no-create-home --shell /bin/false prometheus + mkdir /etc/prometheus + mkdir /var/lib/prometheus + wget https://github.com/prometheus/prometheus/releases/download/v2.42.0/prometheus-2.42.0.linux-amd64.tar.gz + + tar -xvzf prometheus-2.42.0.linux-amd64.tar.gz + mv prometheus-2.42.0.linux-amd64/prometheus /usr/local/bin/ + mv prometheus-2.42.0.linux-amd64/promtool /usr/local/bin/ + mv prometheus-2.42.0.linux-amd64/consoles /etc/prometheus + mv prometheus-2.42.0.linux-amd64/console_libraries /etc/prometheus + + wget https://github.com/prometheus/alertmanager/releases/download/v0.28.0/alertmanager-0.28.0.linux-amd64.tar.gz + tar -xvzf alertmanager-0.28.0.linux-amd64.tar.gz + mv alertmanager-0.28.0.linux-amd64/alertmanager /usr/local/bin/ + mv alertmanager-0.28.0.linux-amd64/amtool /usr/local/bin/ + + + # Create the prometheus.yml with scrape configs + cat < /etc/prometheus/prometheus.yml + global: + scrape_interval: 15s + evaluation_interval: 15s + rule_files: + - /etc/prometheus/alert.rules + alerting: + alertmanagers: + - static_configs: + - targets: + # Alertmanager's default port is 9093 + - localhost:9093 + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'rediscloud' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: / + scheme: https + static_configs: + - targets: ['${local.private_endpoint_host}:8070'] + - job_name: 'rediscloud-v2' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: /v2 + scheme: https + static_configs: + - targets: ['${local.private_endpoint_host}:8070'] + - job_name: 'autoscaler-prometheus-actuator' + scrape_interval: 30s + scrape_timeout: 30s + metrics_path: /actuator/prometheus + scheme: http + static_configs: + - targets: ['${google_compute_instance.autoscaler-vm.network_interface[0].access_config[0].nat_ip}:8080'] + EOF + + cat < /etc/prometheus/alert.rules + groups: + - name: RedisAlerts + rules: + - alert: IncreaseMemory + expr: sum by (instance, db) (redis_server_used_memory) / sum by (instance, db) (redis_server_maxmemory) * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "High Redis Memory Usage" + description: "Redis memory usage is high" + - alert: DecreaseMemory + expr: sum by (instance, db) (redis_server_used_memory) / sum by (instance, db) (redis_server_maxmemory) * 100 < 20 + for: 1m + labels: + severity: warning + annotations: + summary: "Low Redis Memory Usage" + description: "Redis memory usage is low" + - alert: IncreaseThroughput + expr: sum by (db, instance) (irate(endpoint_write_requests[1m]) + irate(endpoint_read_requests[1m])) / on(db) group_left() redis_db_configured_throughput * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "High Redis Throughput" + description: "Redis throughput is high" + - alert: DecreaseThroughput + expr: sum by (db, instance) (irate(endpoint_write_requests[1m]) + irate(endpoint_read_requests[1m])) / on(db) group_left() redis_db_configured_throughput * 100 < 20 + for: 1m + labels: + severity: warning + annotations: + summary: "Low Redis Throughput" + description: "Redis throughput is low" + EOF + + cat < /etc/prometheus/alertmanager.yml + global: + resolve_timeout: 1m + route: + receiver: webhook-receiver + repeat_interval: 1m + receivers: + - name: webhook-receiver + webhook_configs: + - url: 'http://${google_compute_instance.autoscaler-vm.name}:8080/alerts' + EOF + + chown -R prometheus:prometheus /etc/prometheus /var/lib/prometheus + + sudo mkdir /alertmanager/data -p + sudo chown -R prometheus:prometheus /alertmanager /alertmanager/data + + echo '[Unit] + Description=Prometheus + Wants=network-online.target + After=network-online.target + + [Service] + User=prometheus + Group=prometheus + Type=simple + ExecStart=/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/ + + [Install] + WantedBy=multi-user.target' > /etc/systemd/system/prometheus.service + + echo '[Unit] + Description=Alertmanager + Wants=network-online.target + After=network-online.target + + [Service] + User=prometheus + Group=prometheus + Type=simple + ExecStart=/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml + WorkingDirectory=/alertmanager + + [Install] + WantedBy=multi-user.target' > /etc/systemd/system/alertmanager.service + + + systemctl daemon-reload + systemctl enable prometheus + systemctl start prometheus + systemctl enable alertmanager + systemctl start alertmanager + EOT +} + +resource "google_dns_record_set" "autoscale_prometheus_dns" { + managed_zone = var.dns-zone-name + name = "prometheus.autoscaler.${var.subdomain}." + type = "A" + ttl = 300 + rrdatas = [google_compute_instance.autoscale-vm-prometheus.network_interface[0].access_config[0].nat_ip] +} + +# Output the endpoints for reference +output "public_endpoint" { + value = rediscloud_subscription_database.autoscale-database.public_endpoint +} + +output "private_endpoint" { + value = rediscloud_subscription_database.autoscale-database.private_endpoint +} + +output "autoscaler_vm_ip" { + value = google_compute_instance.autoscaler-vm.network_interface[0].access_config[0].nat_ip +} + +output "prometheus_vm_ip" { + value = google_compute_instance.autoscale-vm-prometheus.network_interface[0].access_config[0].nat_ip +} + +output "autoscaler_dns" { + value = nonsensitive(google_dns_record_set.autoscaler_dns.name) +} + +output "prometheus_dns" { + value = nonsensitive(google_dns_record_set.autoscale_prometheus_dns.name) +} \ No newline at end of file diff --git a/standalone-terraform/terraform.tfvars.example b/standalone-terraform/terraform.tfvars.example new file mode 100644 index 0000000..10674a5 --- /dev/null +++ b/standalone-terraform/terraform.tfvars.example @@ -0,0 +1,15 @@ +# Example terraform.tfvars file for standalone deployment +# Copy this file to terraform.tfvars and fill in your values + +gcp_project = "your-gcp-project-id" +dns-zone-name = "your-dns-zone" +subdomain = "example.com" +redis_cloud_account_key = "your-account-key" +redis_cloud_api_key = "your-api-key" +sub_name = "autoscaler-standalone" +db_name = "autoscaler-db" +last_four_digits = "1234" # Last 4 digits of your Redis Cloud payment method +gcloud_username = "your-gcp-username" +gcloud_region = "us-central1" +gcloud_zone = "us-central1-a" +ssh_key_file = "~/.ssh/your-key-file" \ No newline at end of file diff --git a/standalone-terraform/variables.tf b/standalone-terraform/variables.tf new file mode 100644 index 0000000..5a2cec9 --- /dev/null +++ b/standalone-terraform/variables.tf @@ -0,0 +1,66 @@ +# Variables for standalone deployment +variable "gcp_project" { + type = string + description = "The GCP project ID" +} + +variable "dns-zone-name" { + type = string + sensitive = true + description = "The DNS Zone to deploy the app to" +} + +variable "subdomain" { + type = string + sensitive = true + description = "The subdomain to deploy the app to" + +} + +variable "redis_cloud_account_key" { + type = string + sensitive = true + description = "The Redis Cloud account key" +} + +variable "redis_cloud_api_key" { + type = string + sensitive = true + description = "The Redis Cloud API key" +} + +variable "sub_name" { + type = string + description = "The name of the subscription" +} + +variable "db_name" { + type = string + description = "The name of the database" +} + +variable "last_four_digits" { + type = string + sensitive = true + description = "The last four credit card digits" +} + +variable "gcloud_username" { + type = string + description = "The GCP username" +} + +variable "gcloud_region" { + type = string + description = "The GCP region" +} + +variable "gcloud_zone" { + type = string + description = "The GCP zone" +} + +variable "ssh_key_file" { + type = string + description = "The path to the SSH key file" +} \ No newline at end of file