From d0f6859f0499f198ca978a8a806f82daf69e947f Mon Sep 17 00:00:00 2001 From: Travis Clarke Date: Tue, 1 Oct 2019 15:36:22 -0700 Subject: [PATCH] Introduce terraform for creating "maintained" clusters --- .gitignore | 3 + config/README.md | 1 + .../k8s-prow/cluster-configuration.tf | 170 ++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 config/clusters/k8s-prow/cluster-configuration.tf diff --git a/.gitignore b/.gitignore index a6a83d53dbc1..5ce63a70b24f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ node_modules/ # Files generated by pj-on-kind.sh pj.yaml pod.yaml +# Terraform files +**/.terraform/* +*.tfstate* diff --git a/config/README.md b/config/README.md index c7cb5cc97124..5ddba80eea00 100644 --- a/config/README.md +++ b/config/README.md @@ -8,4 +8,5 @@ the tools housed within this repo. - [`jobs/`](./jobs): Prow job configs for [prow.k8s.io](https://prow.k8s.io) - [`testgrids/`](./testgrids): Testgrid configuration for [testgrid.k8s.io](https://testgrid.k8s.io) - [`tests/`](./tests): validation tests for the configs +- [`clusters/`](./clusters): GCP cluster infrastructure for [prow.k8s.io](https://prow.k8s.io) diff --git a/config/clusters/k8s-prow/cluster-configuration.tf b/config/clusters/k8s-prow/cluster-configuration.tf new file mode 100644 index 000000000000..657c03e6fe59 --- /dev/null +++ b/config/clusters/k8s-prow/cluster-configuration.tf @@ -0,0 +1,170 @@ +/* +This file defines the configuration for the `k8s-prow` cluster: + - GCP container cluster + - GCP container node pools +*/ + +variable "project_name" { + type = "string" + default = "k8s-prow" +} + +variable "cluster_name" { + type = "string" + default = "prow" +} + +variable "cluster_region" { + type = "string" + default = "us-central1-f" +} + +# Configure the Google Cloud provider +provider "google" { + project = "${var.project_name}" + region = "${var.cluster_region}" +} + +# Configure the Google Cloud beta provider (required for defining taints) +provider "google-beta" { + project = "${var.project_name}" + region = "${var.cluster_region}" +} + +resource "google_container_cluster" "cluster" { + name = "${var.cluster_name}" + location = "${var.cluster_region}" + + # Whether the ABAC authorizer is enabled for this cluster. When enabled, identities + # in the system, including service accounts, nodes, and controllers, will have statically + # granted permissions beyond those provided by the RBAC configuration or IAM. + # Set to `false` to utilize RBAC. + enable_legacy_abac = true + + # Disable basic and client certificate authorization for the cluster + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } +} + + +# The "default-pool" pool is for running Prow service component nodes (e.g. Deck, Plank, Sinker, etc.) +resource "google_container_node_pool" "default_nodes" { + name = "default-pool" + location = "${google_container_cluster.cluster.location}" + cluster = "${google_container_cluster.cluster.name}" + node_count = 8 + + # Auto repair, and auto upgrade nodes to match the master version + management { + auto_repair = false + auto_upgrade = true + } + + node_config { + machine_type = "n1-standard-4" + disk_size_gb = "100" + labels = { + role = "prow" + } + oauth_scopes = [ + # Compute Engine (rw) + "https://www.googleapis.com/auth/compute", + # Storage (full) + "https://www.googleapis.com/auth/devstorage.full_control", + # Storage (ro) + "https://www.googleapis.com/auth/devstorage.read_only", + # Service Control (enabled) + "https://www.googleapis.com/auth/servicecontrol", + # Service Management (rw) + "https://www.googleapis.com/auth/service.management", + # Service Management (ro) + "https://www.googleapis.com/auth/service.management.readonly", + # Stackdriver Logging (wo) + "https://www.googleapis.com/auth/logging.write", + # Stackdriver Monitoring (full) + "https://www.googleapis.com/auth/monitoring", + ] + } +} + +# The "ghproxy" pool is for running the GitHub reverse proxy cache (i.e. GHproxy) +resource "google_container_node_pool" "ghproxy_nodes" { + provider = "google-beta" + + name = "ghproxy" + location = "${google_container_cluster.cluster.location}" + cluster = "${google_container_cluster.cluster.name}" + node_count = 1 + + + # Auto repair, and auto upgrade nodes to match the master version + management { + auto_repair = true + auto_upgrade = true + } + + # The node configuration of the pool. + node_config { + machine_type = "n1-standard-8" + disk_size_gb = "100" + labels = { + dedicated = "ghproxy" + } + taint { + key = "dedicated" + value = "ghproxy" + effect = "NO_SCHEDULE" + } + oauth_scopes = [ + # Compute Engine (rw) + "https://www.googleapis.com/auth/compute", + # Storage (ro) + "https://www.googleapis.com/auth/devstorage.read_only", + # Service Control (enabled) + "https://www.googleapis.com/auth/servicecontrol", + # Service Management (rw) + "https://www.googleapis.com/auth/service.management", + # Stackdriver Logging (wo) + "https://www.googleapis.com/auth/logging.write", + # Stackdriver Monitoring (full) + "https://www.googleapis.com/auth/monitoring", + ] + } +} + +resource "google_container_node_pool" "n1_standard_8_nodes" { + name = "n1-standard-8" + location = "${google_container_cluster.cluster.location}" + cluster = "${google_container_cluster.cluster.name}" + node_count = 8 + + # Auto repair, and auto upgrade nodes to match the master version + management { + auto_repair = true + auto_upgrade = true + } + + # The node configuration of the pool. + node_config { + machine_type = "n1-standard-8" + disk_size_gb = "200" + + oauth_scopes = [ + # Compute Engine (rw) + "https://www.googleapis.com/auth/compute", + # Storage (ro) + "https://www.googleapis.com/auth/devstorage.read_only", + # Service Control (enabled) + "https://www.googleapis.com/auth/servicecontrol", + # Service Management (rw) + "https://www.googleapis.com/auth/service.management", + # Stackdriver Logging (wo) + "https://www.googleapis.com/auth/logging.write", + # Stackdriver Monitoring (full) + "https://www.googleapis.com/auth/monitoring", + ] + } +}