Enable EKS Pod Identity for Hub-Spoke Patterns (#55)

Signed-off-by: Carlos Santana <[email protected]>

Author: csantanapr

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/README.md

@@ -50,16 +50,6 @@ Access ArgoCD's UI, run the command from the output:
 terraform output -raw access_argocd
 ```
 
-## Verify that ArgoCD Service Accouts has the annotation for IRSA
-```shell
-kubectl get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"'
-kubectl get sa -n argocd argocd-server  -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"'
-```
-The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters
-```text
-"arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub"
-```
-
 ## Deploy the Spoke EKS Cluster
 Initialize Terraform and deploy the EKS clusters:
 Is recommended to use a new terminal window for each cluster

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/main.tf

@@ -73,10 +73,9 @@ locals {
     enable_ack_emrcontainers                     = try(var.addons.enable_ack_emrcontainers, false)
     enable_ack_sfn                               = try(var.addons.enable_ack_sfn, false)
     enable_ack_eventbridge                       = try(var.addons.enable_ack_eventbridge, false)
-    enable_aws_argocd                            = try(var.addons.enable_aws_argocd, false)
   }
   oss_addons = {
-    enable_argocd                          = try(var.addons.enable_argocd, false)
+    enable_argocd                          = try(var.addons.enable_argocd, true)
     enable_argo_rollouts                   = try(var.addons.enable_argo_rollouts, false)
     enable_argo_events                     = try(var.addons.enable_argo_events, false)
     enable_argo_workflows                  = try(var.addons.enable_argo_workflows, false)
@@ -107,7 +106,6 @@ locals {
       aws_vpc_id       = module.vpc.vpc_id
     },
     {
-      argocd_iam_role_arn = module.argocd_irsa.iam_role_arn
       argocd_namespace    = local.argocd_namespace
     },
     {
@@ -149,43 +147,49 @@ module "gitops_bridge_bootstrap" {
 ################################################################################
 # ArgoCD EKS Access
 ################################################################################
-module "argocd_irsa" {
-  source = "aws-ia/eks-blueprints-addon/aws"
-
-  create_release             = false
-  create_role                = true
-  role_name_use_prefix       = false
-  role_name                  = "${module.eks.cluster_name}-argocd-hub"
-  assume_role_condition_test = "StringLike"
-  create_policy              = false
-  role_policies = {
-    ArgoCD_EKS_Policy = aws_iam_policy.irsa_policy.arn
-  }
-  oidc_providers = {
-    this = {
-      provider_arn    = module.eks.oidc_provider_arn
-      namespace       = local.argocd_namespace
-      service_account = "argocd-*"
+data "aws_iam_policy_document" "eks_assume" {
+  statement {
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["pods.eks.amazonaws.com"]
     }
+    actions = ["sts:AssumeRole","sts:TagSession"]
   }
-  tags = local.tags
-
 }
-
-resource "aws_iam_policy" "irsa_policy" {
-  name        = "${module.eks.cluster_name}-argocd-irsa"
-  description = "IAM Policy for ArgoCD Hub"
-  policy      = data.aws_iam_policy_document.irsa_policy.json
-  tags        = local.tags
+resource "aws_iam_role" "argocd_hub" {
+  name               = "${module.eks.cluster_name}-argocd-hub"
+  assume_role_policy = data.aws_iam_policy_document.eks_assume.json
 }
-
-data "aws_iam_policy_document" "irsa_policy" {
+data "aws_iam_policy_document" "aws_assume_policy" {
   statement {
     effect    = "Allow"
     resources = ["*"]
-    actions   = ["sts:AssumeRole"]
+    actions   = ["sts:AssumeRole","sts:TagSession"]
   }
 }
+resource "aws_iam_policy" "aws_assume_policy" {
+  name        = "${module.eks.cluster_name}-argocd-aws-assume"
+  description = "IAM Policy for ArgoCD Hub"
+  policy      = data.aws_iam_policy_document.aws_assume_policy.json
+  tags        = local.tags
+}
+resource "aws_iam_role_policy_attachment" "aws_assume_policy" {
+  role       = aws_iam_role.argocd_hub.name
+  policy_arn = aws_iam_policy.aws_assume_policy.arn
+}
+resource "aws_eks_pod_identity_association" "argocd_app_controller" {
+  cluster_name    = module.eks.cluster_name
+  namespace       = "argocd"
+  service_account = "argocd-application-controller"
+  role_arn        = aws_iam_role.argocd_hub.arn
+}
+resource "aws_eks_pod_identity_association" "argocd_api_server" {
+  cluster_name    = module.eks.cluster_name
+  namespace       = "argocd"
+  service_account = "argocd-server"
+  role_arn        = aws_iam_role.argocd_hub.arn
+}
 
 ################################################################################
 # EKS Blueprints Addons
@@ -228,7 +232,7 @@ module "eks_blueprints_addons" {
 #tfsec:ignore:aws-eks-enable-control-plane-logging
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "~> 19.13"
+  version = "~> 20.5"
 
   cluster_name                   = local.name
   cluster_version                = local.cluster_version
@@ -238,6 +242,10 @@ module "eks" {
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.private_subnets
 
+  # Cluster access entry
+  # To add the current caller identity as an administrator
+  enable_cluster_creator_admin_permissions = true
+
   eks_managed_node_groups = {
     initial = {
       instance_types = ["t3.medium"]
@@ -249,6 +257,10 @@ module "eks" {
   }
   # EKS Addons
   cluster_addons = {
+    eks-pod-identity-agent = {
+      most_recent = true
+      before_compute = true
+    }
     vpc-cni = {
       # Specify the VPC CNI addon should be deployed before compute to ensure
       # the addon is configured before data plane compute resources are created

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/outputs.tf

@@ -36,9 +36,10 @@ output "access_argocd" {
 
 output "argocd_iam_role_arn" {
   description = "IAM Role for ArgoCD Cluster Hub, use to connect to spoke clusters"
-  value       = module.argocd_irsa.iam_role_arn
+  value       = aws_iam_role.argocd_hub.arn
 }
 
+
 output "cluster_name" {
   description = "Cluster Hub name"
   value       = module.eks.cluster_name

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/variables.tf

@@ -11,18 +11,15 @@ variable "region" {
 variable "kubernetes_version" {
   description = "Kubernetes version"
   type        = string
-  default     = "1.28"
+  default     = "1.29"
 }
 variable "addons" {
   description = "Kubernetes addons"
   type        = any
   default = {
     enable_aws_load_balancer_controller = true
     enable_metrics_server               = true
-    # Enable argocd with IRSA
-    enable_aws_argocd = true
-    # Disable argocd without IRSA
-    enable_argocd = false
+    enable_argocd = true
   }
 }
 # Addons Git

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/versions.tf

@@ -4,15 +4,15 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 4.67.0"
+      version = ">= 5.39.1"
     }
     helm = {
       source  = "hashicorp/helm"
-      version = ">= 2.10.1"
+      version = ">= 2.12.1"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = "2.22.0"
+      version = ">= 2.26.0"
     }
   }
 

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/main.tf

@@ -111,10 +111,9 @@ locals {
     enable_ack_emrcontainers                     = try(var.addons.enable_ack_emrcontainers, false)
     enable_ack_sfn                               = try(var.addons.enable_ack_sfn, false)
     enable_ack_eventbridge                       = try(var.addons.enable_ack_eventbridge, false)
-    enable_aws_argocd                            = try(var.addons.enable_aws_argocd, false)
   }
   oss_addons = {
-    enable_argocd                          = try(var.addons.enable_argocd, false)
+    enable_argocd                          = try(var.addons.enable_argocd, true)
     enable_argo_rollouts                   = try(var.addons.enable_argo_rollouts, false)
     enable_argo_events                     = try(var.addons.enable_argo_events, false)
     enable_argo_workflows                  = try(var.addons.enable_argo_workflows, false)
@@ -187,7 +186,7 @@ resource "time_sleep" "wait_for_argocd_namespace_and_crds" {
   depends_on = [module.gitops_bridge_bootstrap_hub]
 }
 module "gitops_bridge_bootstrap_spoke" {
-  source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0"
+  source = "gitops-bridge-dev/gitops-bridge/helm"
 
   install = false # Not installing argocd via helm on spoke cluster
   cluster = {
@@ -208,7 +207,7 @@ module "gitops_bridge_bootstrap_spoke" {
 # GitOps Bridge: Bootstrap for Hub Cluster
 ################################################################################
 module "gitops_bridge_bootstrap_hub" {
-  source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0"
+  source = "gitops-bridge-dev/gitops-bridge/helm"
 
   # The ArgoCD remote cluster secret is deploy on hub cluster not on spoke clusters
   providers = {
@@ -240,20 +239,21 @@ module "gitops_bridge_bootstrap_hub" {
 ################################################################################
 # ArgoCD EKS Access
 ################################################################################
-resource "aws_iam_role" "spoke" {
-  name               = "${module.eks.cluster_name}-argocd-spoke"
-  assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
-}
-
 data "aws_iam_policy_document" "assume_role_policy" {
   statement {
-    actions = ["sts:AssumeRole"]
+    actions = ["sts:AssumeRole","sts:TagSession"]
     principals {
       type        = "AWS"
       identifiers = [data.terraform_remote_state.cluster_hub.outputs.argocd_iam_role_arn]
     }
   }
 }
+resource "aws_iam_role" "spoke" {
+  name               = "${module.eks.cluster_name}-argocd-spoke"
+  assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
+}
+
+
 
 
 
@@ -298,7 +298,7 @@ module "eks_blueprints_addons" {
 #tfsec:ignore:aws-eks-enable-control-plane-logging
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "~> 19.13"
+  version = "~> 20.5"
 
   cluster_name                   = local.name
   cluster_version                = local.cluster_version
@@ -308,17 +308,25 @@ module "eks" {
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.private_subnets
 
-  manage_aws_auth_configmap = true
-  aws_auth_roles = [
-    # Granting access to ArgoCD from hub cluster
-    {
-      rolearn  = aws_iam_role.spoke.arn
-      username = "gitops-role"
-      groups = [
-        "system:masters"
-      ]
-    },
-  ]
+  # Cluster access entry
+  # To add the current caller identity as an administrator
+  enable_cluster_creator_admin_permissions = true
+
+  access_entries = {
+    # One access entry with a policy associated
+    argocd = {
+      principal_arn     = aws_iam_role.spoke.arn
+
+      policy_associations = {
+        argocd = {
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+          access_scope = {
+            type       = "cluster"
+          }
+        }
+      }
+    }
+  }
 
   eks_managed_node_groups = {
     initial = {
@@ -331,6 +339,10 @@ module "eks" {
   }
   # EKS Addons
   cluster_addons = {
+    eks-pod-identity-agent = {
+      most_recent = true
+      before_compute = true
+    }
     vpc-cni = {
       # Specify the VPC CNI addon should be deployed before compute to ensure
       # the addon is configured before data plane compute resources are created

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/versions.tf

@@ -4,19 +4,19 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 4.67.0"
-    }
-    helm = {
-      source  = "hashicorp/helm"
-      version = ">= 2.10.1"
+      version = ">= 5.39.1"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = "2.22.0"
+      version = ">= 2.26.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">= 2.12.1"
     }
     time = {
       source  = "hashicorp/time"
-      version = ">= 0.9.1"
+      version = ">= 0.10.0"
     }
   }
 

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/dev.tfvars

@@ -1,6 +1,6 @@
 vpc_cidr = "10.1.0.0/16"
 region = "us-west-2"
-kubernetes_version = "1.28"
+kubernetes_version = "1.29"
 addons = {
   enable_aws_load_balancer_controller = true
   enable_metrics_server               = true

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/prod.tfvars

@@ -1,6 +1,6 @@
 vpc_cidr = "10.3.0.0/16"
 region = "us-west-2"
-kubernetes_version = "1.28"
+kubernetes_version = "1.29"
 addons = {
   enable_aws_load_balancer_controller = true
   enable_metrics_server               = true

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/staging.tfvars

@@ -1,6 +1,6 @@
 vpc_cidr = "10.2.0.0/16"
 region = "us-west-2"
-kubernetes_version = "1.28"
+kubernetes_version = "1.29"
 addons = {
   enable_aws_load_balancer_controller = true
   enable_metrics_server               = true

argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md

@@ -50,16 +50,6 @@ Access ArgoCD's UI, run the command from the output:
 terraform output -raw access_argocd
 ```
 
-## Verify that ArgoCD Service Accouts has the annotation for IRSA
-```shell
-kubectl get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"'
-kubectl get sa -n argocd argocd-server  -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"'
-```
-The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters
-```text
-"arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub"
-```
-
 ## Deploy the Spoke EKS Cluster
 Initialize Terraform and deploy the EKS clusters:
 ```shell