Merge pull request #19 from hbarros-caylent/CA-93-cw-logs

hbarros-caylent · web-flow · commit 08f45c9a116f · 2021-11-18T13:15:09.000-05:00
CA-93 cloudwatch logs integration
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Tamr Terraform AWS Elasticsearch Repo
 
+## v3.1.0 - November 15th 2021
+* Adds variables `logs_retention_in_days` and `log_types` that enables the publishing of ElasticSearch Logs into CloudWatch
+
 ## v3.0.0 - August 9th 2021
 * Requires ES service role to be created as a prerequisite to using this module (see README)
 * Removes input variables `linked_service_role` and `create_new_service_role`
diff --git a/README.md b/README.md
@@ -65,6 +65,8 @@ No provider.
 | instance\_count | Number of instances to launch in the ES domain | `number` | `2` | no |
 | instance\_type | Instance type of data nodes in the domain | `string` | `"c5.large.elasticsearch"` | no |
 | kms\_key\_id | The KMS key id to encrypt the Elasticsearch domain with.<br>  If not specified then it defaults to using the aws/es service KMS key | `string` | `null` | no |
+| log\_retention\_in\_days | Specifies the number of days you want to retain log events.<br>  Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653, and 0.<br>  If you select 0, the events in the log group are always retained and never expire. | `number` | `0` | no |
+| log\_types | A list of log types that will be published to CloudWatch. Valid values are SEARCH\_SLOW\_LOGS, INDEX\_SLOW\_LOGS, ES\_APPLICATION\_LOGS and AUDIT\_LOGS. | `list(string)` | <pre>[<br>  "ES_APPLICATION_LOGS",<br>  "SEARCH_SLOW_LOGS",<br>  "INDEX_SLOW_LOGS"<br>]</pre> | no |
 | node\_to\_node\_encryption\_enabled | Whether to enable node-to-node encryption | `bool` | `true` | no |
 | revoke\_rules\_on\_delete | Whether to revoke rules from the SG upon deletion | `bool` | `true` | no |
 | security\_group\_ids | List of security group IDs to be applied to the ES domain | `list(string)` | `[]` | no |
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-3.0.0
+3.1.0
diff --git a/examples/minimal/README.md b/examples/minimal/README.md
@@ -15,6 +15,8 @@ No requirements.
 |------|-------------|------|---------|:--------:|
 | name-prefix | A string to prepend to names of resources created by this example | `any` | n/a | yes |
 | create\_new\_service\_role | Whether to create a new IAM service linked role for ES. This only needs to happen once per account. If false, linked\_service\_role is required | `bool` | `false` | no |
+| log\_retention\_in\_days | Specifies the number of days you want to retain log events.<br>  Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653, and 0.<br>  If you select 0, the events in the log group are always retained and never expire. | `number` | `0` | no |
+| log\_types | A list of log types that will be published to CloudWatch. Valid values are SEARCH\_SLOW\_LOGS, INDEX\_SLOW\_LOGS, ES\_APPLICATION\_LOGS and AUDIT\_LOGS. | `list(string)` | <pre>[<br>  "ES_APPLICATION_LOGS",<br>  "SEARCH_SLOW_LOGS",<br>  "INDEX_SLOW_LOGS"<br>]</pre> | no |
 | tags | A map of tags to add to all resources created by this example. | `map(string)` | <pre>{<br>  "Author": "Tamr",<br>  "Environment": "Example"<br>}</pre> | no |
 
 ## Outputs
diff --git a/examples/minimal/main.tf b/examples/minimal/main.tf
@@ -34,11 +34,11 @@ resource "aws_iam_service_linked_role" "es" {
 }
 
 module "tamr-es-cluster" {
-  depends_on         = [aws_iam_service_linked_role.es]
-  source             = "../../"
-  vpc_id             = aws_vpc.es_vpc.id
-  domain_name        = format("%s-elasticsearch", var.name-prefix)
-  subnet_ids         = [aws_subnet.es_subnet.id]
-  security_group_ids = module.aws-sg.security_group_ids
-  tags               = var.tags
+  depends_on            = [aws_iam_service_linked_role.es]
+  source                = "../../"
+  vpc_id                = aws_vpc.es_vpc.id
+  domain_name           = format("%s-elasticsearch", var.name-prefix)
+  subnet_ids            = [aws_subnet.es_subnet.id]
+  security_group_ids    = module.aws-sg.security_group_ids
+  tags                  = var.tags
 }
diff --git a/main.tf b/main.tf
@@ -22,4 +22,18 @@ module "tamr-es-cluster" {
   tls_security_policy             = var.tls_security_policy
   node_to_node_encryption_enabled = var.node_to_node_encryption_enabled
   arn_partition                   = var.arn_partition
+  log_publishing_options          = module.tamr-es-coudwatch-log-groups.log_publishing_options
+
+  depends_on = [
+    module.tamr-es-coudwatch-log-groups
+  ]
+}
+
+module "tamr-es-coudwatch-log-groups" {
+  source = "./modules/cloudwatch-logs"
+
+  domain_name           = var.domain_name
+  tags                  = local.effective_tags
+  log_types             = var.log_types
+  log_retention_in_days = var.log_retention_in_days
 }
diff --git a/modules/aws-es/README.md b/modules/aws-es/README.md
@@ -48,6 +48,7 @@ This modules creates:
 | instance\_count | Number of instances to launch in the ES domain | `number` | `2` | no |
 | instance\_type | Instance type of data nodes in the domain | `string` | `"c5.large.elasticsearch"` | no |
 | kms\_key\_id | The KMS key id to encrypt the Elasticsearch domain with.<br>  If not specified then it defaults to using the aws/es service KMS key | `string` | `null` | no |
+| log\_publishing\_options | Set of objects containing values for publishing logs to cloudwatch | <pre>list(object({<br>    log_group_arn = string<br>    log_type      = string<br>  }))</pre> | `[]` | no |
 | node\_to\_node\_encryption\_enabled | Whether to enable node-to-node encryption | `bool` | `true` | no |
 | security\_group\_ids | List of security group IDs to be applied to the ES domain | `list(string)` | `[]` | no |
 | snapshot\_start\_hour | Hour when an automated daily snapshot of the indices is taken | `number` | `0` | no |
diff --git a/modules/aws-es/main.tf b/modules/aws-es/main.tf
@@ -1,3 +1,4 @@
+#tfsec:ignore:aws-elastic-search-enable-domain-logging tfsec:ignore:aws-elastic-search-enable-logging
 resource "aws_elasticsearch_domain" "tamr-es-cluster" {
   domain_name           = var.domain_name
   elasticsearch_version = var.es_version
@@ -38,6 +39,15 @@ resource "aws_elasticsearch_domain" "tamr-es-cluster" {
     enabled = var.node_to_node_encryption_enabled
   }
 
+  dynamic "log_publishing_options" {
+    for_each = var.log_publishing_options
+    content {
+      cloudwatch_log_group_arn = log_publishing_options.value["log_group_arn"]
+      enabled                  = true
+      log_type                 = log_publishing_options.value["log_type"]
+    }
+  }
+
   tags = var.tags
 }
 
diff --git a/modules/aws-es/variables.tf b/modules/aws-es/variables.tf
@@ -118,3 +118,12 @@ variable "arn_partition" {
   EOF
   default     = "aws"
 }
+
+variable "log_publishing_options" {
+  type = list(object({
+    log_group_arn = string
+    log_type      = string
+  }))
+  description = "Set of objects containing values for publishing logs to cloudwatch"
+  default     = []
+}
diff --git a/modules/cloudwatch-logs/README.md b/modules/cloudwatch-logs/README.md
@@ -0,0 +1,36 @@
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | n/a |
+| random | n/a |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| domain\_name | The name to give to the ES domain | `string` | `"tamr-es-cluster"` | no |
+| log\_retention\_in\_days | Specifies the number of days you want to retain log events.<br>  Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653, and 0.<br>  If you select 0, the events in the log group are always retained and never expire. | `number` | `0` | no |
+| log\_types | A list of log types that will be published to CloudWatch. Valid values are SEARCH\_SLOW\_LOGS, INDEX\_SLOW\_LOGS, ES\_APPLICATION\_LOGS and AUDIT\_LOGS. | `list(string)` | <pre>[<br>  "ES_APPLICATION_LOGS",<br>  "SEARCH_SLOW_LOGS",<br>  "INDEX_SLOW_LOGS"<br>]</pre> | no |
+| tags | A map of tags to add to CloudWatch resources. | `map(string)` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| log\_publishing\_options | n/a |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+# References
+This repo is based on:
+* [terraform standard module structure](https://www.terraform.io/docs/modules/index.html#standard-module-structure)
+* [templated terraform module](https://github.com/tmknom/template-terraform-module)
+
+# License
+Apache 2 Licensed. See LICENSE for full details.
diff --git a/modules/cloudwatch-logs/main.tf b/modules/cloudwatch-logs/main.tf
@@ -0,0 +1,37 @@
+#tfsec:ignore:aws-cloudwatch-log-group-customer-key
+resource "aws_cloudwatch_log_group" "es-logs" {
+  for_each = toset(var.log_types)
+
+  name_prefix = format("%s-%s", var.domain_name, each.value)
+
+  retention_in_days = var.log_retention_in_days
+  tags              = var.tags
+}
+
+data "aws_iam_policy_document" "es-tamr-log-publishing-policy" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:PutLogEventsBatch",
+    ]
+
+    resources = [for i in aws_cloudwatch_log_group.es-logs : "${i.arn}:*"]
+
+    principals {
+      identifiers = ["es.amazonaws.com"]
+      type        = "Service"
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_resource_policy" "es-tamr-log-publishing-policy" {
+  count           = length(var.log_types) > 0 ? 1 : 0
+  policy_document = data.aws_iam_policy_document.es-tamr-log-publishing-policy.json
+  policy_name     = "es-tamr-log-publishing-policy-${random_string.suffix.id}"
+}
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+}
diff --git a/modules/cloudwatch-logs/outputs.tf b/modules/cloudwatch-logs/outputs.tf
@@ -0,0 +1,3 @@
+output "log_publishing_options" {
+  value = [for key, log_group in aws_cloudwatch_log_group.es-logs : { "log_group_arn" = log_group.arn, "log_type" = key }]
+}
diff --git a/modules/cloudwatch-logs/variables.tf b/modules/cloudwatch-logs/variables.tf
@@ -0,0 +1,27 @@
+variable "domain_name" {
+  default     = "tamr-es-cluster"
+  type        = string
+  description = "The name to give to the ES domain"
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "A map of tags to add to CloudWatch resources."
+  default     = {}
+}
+
+variable "log_types" {
+  type        = list(string)
+  description = "A list of log types that will be published to CloudWatch. Valid values are SEARCH_SLOW_LOGS, INDEX_SLOW_LOGS, ES_APPLICATION_LOGS and AUDIT_LOGS."
+  default     = ["ES_APPLICATION_LOGS", "SEARCH_SLOW_LOGS", "INDEX_SLOW_LOGS"]
+}
+
+variable "log_retention_in_days" {
+  type        = number
+  description = <<EOF
+  Specifies the number of days you want to retain log events.
+  Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653, and 0.
+  If you select 0, the events in the log group are always retained and never expire.
+  EOF
+  default     = 0
+}
diff --git a/variables.tf b/variables.tf
@@ -171,3 +171,19 @@ variable "arn_partition" {
   EOF
   default     = "aws"
 }
+
+variable "log_types" {
+  type        = list(string)
+  description = "A list of log types that will be published to CloudWatch. Valid values are SEARCH_SLOW_LOGS, INDEX_SLOW_LOGS, ES_APPLICATION_LOGS and AUDIT_LOGS."
+  default     = ["ES_APPLICATION_LOGS", "SEARCH_SLOW_LOGS", "INDEX_SLOW_LOGS"]
+}
+
+variable "log_retention_in_days" {
+  type        = number
+  description = <<EOF
+  Specifies the number of days you want to retain log events.
+  Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653, and 0.
+  If you select 0, the events in the log group are always retained and never expire.
+  EOF
+  default     = 0
+}
