BI Fixes (#128)

* Fix: schema name in fivetran does not allow some characters

* fix: must define ROLE ARN for Snowflake Source

* Adds SQL Script to generate read-only user for RDS

* Bug: Snowflake Network Policy doesn't work, remove for now

* Adjust readmes and variable names

* must pass role arn in analytics module to fivetran

Author: swiknaba

analytics/fivetran.tf

@@ -15,6 +15,7 @@ module "fivetran" {
   destination_user_name       = var.destination_user_name
   destination_password        = var.destination_password
   destination_connection_type = var.destination_connection_type
+  destination_role_arn        = var.destination_role_arn
 
   environment = local.environment
   project     = local.project

analytics/variables.tf

@@ -30,7 +30,13 @@ variable "time_zone_offset" {
 }
 
 variable "destination_user_name" {
-  type = string
+  type    = string
+  default = "FIVETRAN_USER"
+}
+
+variable "destination_role_arn" {
+  type    = string
+  default = "FIVETRAN_ROLE"
 }
 
 variable "destination_host" {

fivetran/README.md

@@ -22,7 +22,8 @@ module "fivetran" {
   project             = "my-project"
   environment         = "production"
 
-  destination_user_name        = "${project}-${environment}-bot"
+  destination_user_name        = "FIVETRAN_USER"
+  destination_role_arn         = "FIVETRAN_ROLE"
   destination_host             = "${SNOWFLAKE_ACCOUNT_LOCATOR}.eu-central-1.snowflakecomputing.com" # `eu-central-1` if you run on AWS in EU region
   destination_connection_type  = "Directly"
   destination_password         = "XXX"

fivetran/connectors.tf

@@ -42,7 +42,7 @@ resource "fivetran_connector" "github" {
   run_setup_tests   = true
 
   destination_schema {
-    name = "github_${each.value.organisation}" # name shown on Fivetran UI
+    name = "github_${replace(each.value.organisation, "/[^0-9A-Za-z_]/", "_")}" # name shown on Fivetran UI
   }
 
   config {

fivetran/main.tf

@@ -18,6 +18,7 @@ resource "fivetran_destination" "main" {
     port            = var.destination_port
     database        = var.destination_database_name
     user            = var.destination_user_name
+    role_arn        = var.destination_role_arn
     password        = var.destination_password
     auth            = "PASSWORD"
     connection_type = var.destination_connection_type

fivetran/rds-readonly-role.sql

@@ -0,0 +1,10 @@
+-- creates a read-only role for fivetran to connect to RDS
+
+CREATE ROLE {project}_{environment}_fivetran;
+ALTER ROLE {project}_{environment}_fivetran WITH LOGIN ENCRYPTED PASSWORD 'generate-a-secure-password';
+
+GRANT CONNECT ON DATABASE {project}_{environment} TO {project}_{environment}_fivetran;
+GRANT USAGE ON SCHEMA public TO {project}_{environment}_fivetran;
+
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO {project}_{environment}_fivetran;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO {project}_{environment}_fivetran;

fivetran/variables.tf

@@ -29,6 +29,10 @@ variable "destination_user_name" {
   type = string
 }
 
+variable "destination_role_arn" {
+  type = string
+}
+
 variable "destination_host" {
   description = "e.g. 'your-account.snowflakecomputing.com'"
   type        = string

snowflake/cloud/README.md

@@ -7,7 +7,12 @@ Note: Since this module consist of the snowflake network policy, the Snowflake u
 ```terraform
 # main.tf
 module "snowflake_cloud" {
-  source = "github.com/dbl-works/terraform//awesome-module?ref=v2022.08.05"
+  source = "github.com/dbl-works/terraform//snowflake/cloud?ref=v2022.08.05"
+
+  providers = {
+    snowflake                = snowflake
+    snowflake.security_admin = snowflake.security_admin
+  }
 
   warehouse_name = "WH_FIVETRAN" # Snowflake mostly uses upcase by their convention
 
@@ -27,9 +32,11 @@ module "snowflake_cloud" {
   ]
 
   # optional
-  suspend_compute_after_seconds = 57      # on AWS, the minimum charge is 60 seconds
-  warehouse_size                = "large" # 8 credits/hour/cluster for "large"
-  warehouse_cluster_count       = 1
+  suspend_compute_after_seconds    = 57      # on AWS, the minimum charge is 60 seconds
+  warehouse_size                   = "large" # 8 credits/hour/cluster for "large"
+  warehouse_cluster_count          = 1
+  multi_cluster_warehouses_enabled = false # must be enabled in the Snowflake account (via UI)
+
   # Default value of this variable is the fivetrans IP address in the EU region + using GCP as cloud provider
   # If you are using fivetrans, check the list of IP addresses here: https://fivetran.com/docs/getting-started/ips#euregions
   allowed_ip_list               = ["35.235.32.144/29"]
@@ -75,7 +82,7 @@ provider "snowflake" {
   region   = var.snowflake_region
 
   # For auth exactly one option must be set.
-  private_key_passphrase = var.snowflake_private_key_path
+  private_key_path = var.snowflake_private_key_path
 }
 
 provider "snowflake" {
@@ -87,7 +94,7 @@ provider "snowflake" {
   region   = var.snowflake_region
 
   # For auth exactly one option must be set.
-  private_key_passphrase = var.snowflake_private_key_path
+  private_key_path = var.snowflake_private_key_path
 }
 ```
 

snowflake/cloud/main.tf

@@ -14,33 +14,56 @@ resource "snowflake_warehouse" "main" {
   auto_suspend = var.suspend_compute_after_seconds
   auto_resume  = true
 
-  max_cluster_count = var.warehouse_cluster_count
-  min_cluster_count = 1         # if set to less than max count, auto-scaling is enabled
-  scaling_policy    = "ECONOMY" # Conserves credits by favoring keeping running clusters fully-loaded
+  max_cluster_count = var.multi_cluster_warehouses_enabled ? var.warehouse_cluster_count : null
+  min_cluster_count = var.multi_cluster_warehouses_enabled ? 1 : null         # if set to less than max count, auto-scaling is enabled
+  scaling_policy    = var.multi_cluster_warehouses_enabled ? "ECONOMY" : null # Conserves credits by favoring keeping running clusters fully-loaded
 }
 
 locals {
   network_policy_name = "IpNetworkPolicy"
 }
 
+
+#
+# when using "SECURITYADMIN" + "SYSADMIN" roles, which should be suffcient, see https://docs.snowflake.com/en/sql-reference/sql/create-network-policy.html
+#
+# │ Error: error creating network policy IpNetworkPolicy: 003001 (42501): SQL access control error:
+# │ Insufficient privileges to operate on account 'IL49394'
+# │
+# │   with module.snowflake_cloud.snowflake_network_policy.policy,
+# │   on .terraform/modules/snowflake_cloud/main.tf line 27, in resource "snowflake_network_policy" "policy":
+# │   27: resource "snowflake_network_policy" "policy" {
+# │
+# ╵
+# ╷
+# │ Error: error creating attachment for network policy IpNetworkPolicy: error setting network policy IpNetworkPolicy on account: 003001 (42501): SQL access control error:
+# │ Insufficient privileges to operate on account 'IL49394'
+# │
+# │   with module.snowflake_cloud.snowflake_network_policy_attachment.attach,
+# │   on .terraform/modules/snowflake_cloud/main.tf line 39, in resource "snowflake_network_policy_attachment" "attach":
+# │   39: resource "snowflake_network_policy_attachment" "attach" {
+
+
+
 # https://docs.snowflake.com/en/user-guide/network-policies.html
-resource "snowflake_network_policy" "policy" {
-  # The identifier must start with an alphabetic character
-  # and cannot contain spaces or special characters unless the
-  # entire identifier string is enclosed in double quotes (e.g. "My object").
+# resource "snowflake_network_policy" "policy" {
+#   # The identifier must start with an alphabetic character
+#   # and cannot contain spaces or special characters unless the
+#   # entire identifier string is enclosed in double quotes (e.g. "My object").
 
-  name    = local.network_policy_name
-  comment = "Network policy to allow or deny access to a single IP address or a list of addresses."
+#   name    = local.network_policy_name
+#   comment = "Network policy to allow or deny access to a single IP address or a list of addresses."
 
-  allowed_ip_list = var.allowed_ip_list
-  blocked_ip_list = var.blocked_ip_list
-}
+#   allowed_ip_list = var.allowed_ip_list
+#   blocked_ip_list = var.blocked_ip_list
+# }
 
-resource "snowflake_network_policy_attachment" "attach" {
-  network_policy_name = local.network_policy_name
-  # A Snowflake account can only have one network policy set globally at any given time.
-  # This resource does not enforce one-policy-per-account, it is the user's responsibility to enforce this. If multiple network policy resources have set_for_account: true,
-  # the final policy set on the account will be non-deterministic.
-  set_for_account = true
-  users           = var.snowflake_users
-}
+# resource "snowflake_network_policy_attachment" "attach" {
+#   network_policy_name = local.network_policy_name
+#   # A Snowflake account can only have one network policy set globally at any given time.
+#   # This resource does not enforce one-policy-per-account, it is the user's responsibility to enforce this.
+#   # If multiple network policy resources have set_for_account: true,
+#   # the final policy set on the account will be non-deterministic.
+#   set_for_account = true
+#   users           = var.snowflake_users
+# }

snowflake/cloud/variables.tf

@@ -22,6 +22,13 @@ variable "warehouse_cluster_count" {
   default     = 1
 }
 
+# has to be enabled in the Snowflake account. Disabled by default.
+variable "multi_cluster_warehouses_enabled" {
+  description = "https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/warehouse#max_cluster_count"
+  type        = bool
+  default     = false
+}
+
 #
 # A retention period of 0 days for an object effectively disables Time Travel for the object.
 #