S3 encryption

.circleci/config.yml

@@ -1,69 +1,175 @@
 version: 2
 
+# TODO: centralize full configuration. Figure out how
+#   ?? Each step as a separate script that is downloaded and run ??
+#   ?? CircleCI feature request to supoort include from remote sources
+# More Markdown terraform_testing
+# Python testing. Add doc and test that too
+# circleci/python: Both 2 and 3?
+# if src/requirements.txt get version from *.tf and test
+#   Style+: flake8 + hacking?, prospector?
+#   Security: bandit, RATS,
+
+# This file uses YAML anchors to deduplicate steps
+# see https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
+# and https://learnxinyminutes.com/docs/yaml/
+
+.steps_template: &steps_terraform_static_analysis
+  steps:
+    - checkout
+    - run:
+        name: "Check: Validate tf files (terraform validate)"
+        command: |
+            find . -type f -name "*.tf" -exec dirname {} \;|sort -u | while read m; do (terraform validate -check-variables=false "$m" && echo "√ $m") || exit 1 ; done
+    - run:
+        name: "Check: Terraform formatting (terraform fmt)"
+        command: |
+            if [ `terraform fmt --list=true -diff=true -write=false | tee format-issues | wc -c` -ne 0 ]; then
+              echo "Some terraform files need be formatted, run 'terraform fmt' to fix"
+              echo "Formatting issues:"
+              cat format-issues
+              exit 1
+            fi
+    - run:
+        name: "Install: tflint"
+        command: |
+            apk update
+            apk add jq wget
+            # Get latest version of tflint (v0.7.0 test if still need to exclude modules. Any other changes)
+            pkg_arch=linux_amd64
+            dl_url=$(curl -s https://api.github.com/repos/wata727/tflint/releases/latest | jq -r ".assets[] | select(.name | test(\"${pkg_arch}\")) | .browser_download_url")
+            wget ${dl_url}
+            unzip tflint_linux_amd64.zip
+            mkdir -p /usr/local/tflint/bin
+            # Setup PATH for later run steps - ONLY for Bash and not in Bash
+            #echo 'export PATH=/usr/local/tflint/bin:$PATH' >> $BASH_ENV
+            echo "Installing tflint..."
+            install tflint /usr/local/tflint/bin
+            echo "Configuring tflint..."
+            tf_ver=$(terraform version | awk 'FNR <= 1' | cut -dv -f2)
+            echo -e "\tConfig for terraform version: ${tf_ver}"
+            if [ -f '.tflint.hcl' ]; then
+              sed -i "/terraform_version =/s/\".*\"/\"${tf_ver}\"/" .tflint.hcl
+            else
+              {
+              echo -e "config {\nterraform_version = \"${tf_ver}\"\ndeep_check = true\nignore_module = {"
+              for module in $(grep -h '[^a-zA-Z]source[ =]' *.tf | sed -r 's/.*=\s+//' | sort -u); do
+                # if not ^"../
+                echo "${module} = true"
+              done
+              echo -e "}\n}\n"
+              } > .tflint.hcl
+              fi
+            echo "tflint configuration:"
+            cat .tflint.hcl
+    - run:
+        # Not supporting modules from registry ?? v0.5.4
+        # For now, must ignore in config file
+        name: "Check: tflint"
+        command: |
+            #echo "Initializing terraform..."
+            #terraform init -input=false
+            echo "Running tflint..."
+            /usr/local/tflint/bin/tflint --version
+            /usr/local/tflint/bin/tflint
+
 jobs:
-  build:
+  ###
+  ### Documentation testing: Markdown
+  ###
+  # Markdown Lint https://github.com/DavidAnson/markdownlint
+  # CLI https://github.com/igorshubovych/markdownlint-cli
+  # https://hub.docker.com/r/circleci/node/tags/
+  markdown_lint_node:
     docker:
-      - image: hashicorp/terraform:0.11.3
-        entrypoint: /bin/sh
+      - image: circleci/node:10.5.0
     steps:
       - checkout
       - run:
-          name: "Validate tf files (terraform validate)"
+          name: "Install: markdown lint (node.js)"
           command: |
-              find . -type f -name "*.tf" -exec dirname {} \;|sort -u | while read m; do (terraform validate -check-variables=false "$m" && echo "√ $m") || exit 1 ; done
+              sudo npm install -g markdownlint-cli
       - run:
-          name: "Check: Terraform formatting (terraform fmt)"
+          name: "Check: markdown lint (node.js)"
           command: |
-              if [ `terraform fmt --list=true -diff=true -write=false | tee format-issues | wc -c` -ne 0 ]; then
-                echo "Some terraform files need be formatted, run 'terraform fmt' to fix"
-                echo "Formatting issues:"
-                cat format-issues
-                exit 1
-              fi
+              #markdownlint --help
+              echo -n "markdownlint version: "
+              markdownlint --version
+              markdownlint ./
+  # Markdown Lint https://github.com/markdownlint/markdownlint
+  # https://hub.docker.com/r/circleci/ruby/tags/
+  markdown_lint_ruby:
+    docker:
+      - image: circleci/ruby:2.5.1
+    steps:
+      - checkout
+      - run:
+          name: "Install: markdown lint (ruby)"
+          command: |
+              gem install mdl
+      - run:
+          name: "Check: markdown lint (ruby)"
+          command: |
+              #mdl --help
+              echo -n "mdl version: "
+              mdl --version
+              mdl .
+  markdown_proofer:
+    docker:
+      - image: circleci/golang:1.10
+        entrypoint: /bin/sh
+    steps:
+      - checkout
       - run:
-          name: "Install: tflint"
+          name: "Install: markdown proofer"
           command: |
-              apk add jq wget
-              # Get latest version of tflint
+              # Get latest version
               pkg_arch=linux_amd64
-              dl_url=$(curl -s https://api.github.com/repos/wata727/tflint/releases/latest | jq -r ".assets[] | select(.name | test(\"${pkg_arch}\")) | .browser_download_url")
+              # Prerelease, so latest doesn't work yet
+              #dl_url=$(curl -s https://api.github.com/repos/felicianotech/md-proofer/releases/latest | jq -r ".assets[] | select(.name | test(\"${pkg_arch}\")) | .browser_download_url")
+              dl_url='https://github.com/felicianotech/md-proofer/releases/download/v0.2.0/md-proofer--v0.2.0--linux-amd64.tar.gz'
               wget ${dl_url}
-              unzip tflint_linux_amd64.zip
-              mkdir -p /usr/local/tflint/bin
-              # Setup PATH for later run steps - ONLY for Bash and not in Bash
-              #echo 'export PATH=/usr/local/tflint/bin:$PATH' >> $BASH_ENV
-              echo "Installing tflint..."
-              install tflint /usr/local/tflint/bin
-              echo "Configuring tflint..."
-              tf_ver=$(terraform version | awk 'FNR <= 1' | cut -dv -f2)
-              echo -e "\tConfig for terraform version: ${tf_ver}"
-              if [ -f '.tflint.hcl' ]; then
-                sed -i "/terraform_version =/s/\".*\"/\"${tf_ver}\"/" .tflint.hcl
-              else
-                {
-                echo -e "config {\nterraform_version = \"${tf_ver}\"\ndeep_check = true\nignore_module = {"
-                for module in $(grep -h '[^a-zA-Z]source[ =]' *.tf | sed -r 's/.*=\s+//' | sort -u); do
-                  # if not ^"../
-                  echo "${module} = true"
-                done
-                echo "}}"
-                } > .tflint.hcl
-                fi
-              echo "tflint configuration:"
-              cat .tflint.hcl
+              tar xzf md-proofer--v0.2.0--linux-amd64.tar.gz
       - run:
-          # Not supporting modules from registry ?? v0.5.4
-          # For now, must ignore in config file
-          name: "Check: tflint"
+          name: "Check: markdown proofer"
           command: |
-              #echo "Initializing terraform..."
-              #terraform init -input=false
-              echo "Running tflint..."
-              /usr/local/tflint/bin/tflint --version
-              /usr/local/tflint/bin/tflint
+              ./md-proofer version
+              #./md-proofer lint --help
+              # Will this find all *.md in directory structure or need to run in each directory ?
+              if ./md-proofer lint ./; then
+                echo "md-proofer passed"
+              else
+                echo "md-proofer failed"
+              fi
+  ###
+  ### Terraform testing
+  ###
+  terraform_0_11_3:
+    docker:
+      - image: hashicorp/terraform:0.11.3
+        entrypoint: /bin/sh
+    <<: *steps_terraform_static_analysis
+
+  terraform_0_11_7:
+    docker:
+      - image: hashicorp/terraform:0.11.7
+        entrypoint: /bin/sh
+    <<: *steps_terraform_static_analysis
+
+  terraform_latest:
+    docker:
+      - image: hashicorp/terraform:latest
+        entrypoint: /bin/sh
+    <<: *steps_terraform_static_analysis
 
 workflows:
   version: 2
-  build:
+  terraform_testing:
     jobs:
-      - build
+      - markdown_lint_node
+      - markdown_lint_ruby
+      # Currently doesn't do anything that markdownlint node doesn't do
+      #- markdown_proofer
+      - terraform_0_11_3
+      - terraform_0_11_7
+      - terraform_latest

.gitignore

@@ -1,3 +1,4 @@
-.terraform*
-!terraform.tfstate*
-terraform.*
+*.tfstate
+*.tfstate.backup
+*.tfvars
+.terraform

.markdownlintrc

@@ -0,0 +1,4 @@
+{
+"default": true,
+"MD013": { "code_blocks": false, "tables": false },
+}

.mdlrc

@@ -0,0 +1 @@
+rules "~MD013"

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+# See http://pre-commit.com for more information
+# See http://pre-commit.com/hooks.html for more hooks
+# To update to all latest tagged versions run:
+#     pre-commit autoupdate
+# TODO: write dependencies install instructions and put in each of
+#       my pre-commit repos. Decide where to put for others
+repos:
+  - repo: https://github.com/devops-workflow/pre-commit-terraform
+    rev: v1.13.3
+    hooks:
+      - id: terraform_tools
+      #- id: terraform_template
+      #  args: [--owner=appzen-oss, --repo=terraform-template]
+      - id: terraform_fmt
+      - id: terraform_docs
+      - id: terraform_graph
+      #- id: tflint
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.4.0
+    hooks:
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-yaml
+      - id: detect-aws-credentials
+      - id: detect-private-key
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: trailing-whitespace
+        # TODO: test these
+        # check-json
+        # pretty-format-json
+  #- repo: https://github.com/jumanjihouse/pre-commit-hooks
+  #  # Requires: shellcheck, shfmt
+  #  rev: 1.8.0
+  #  hooks:
+  #    - id: shellcheck
+  #    - id: shfmt
+  #- repo: git://github.com/detailyang/pre-commit-shell
+  #  # Requires: shellcheck
+  #  rev: 1.0.2
+  #  hooks:
+  #    - id: shell-lint
+# TODO:
+# add bashate shell code style https://github.com/openstack-dev/bashate
+# gitlint https://github.com/jorisroovers/gitlint
+# Create new repo and hook for markdown linters

README.md

@@ -1,28 +1,82 @@
-[![CircleCI](https://circleci.com/gh/devops-workflow/terraform-aws-s3-buckets?style=svg)](https://circleci.com/gh/devops-workflow/terraform-aws-s3-buckets)
+# AWS S3 Buckets Terraform module
 
-AWS S3 Buckets Terraform module
-========================
+[![CircleCI](https://circleci.com/gh/appzen-oss/terraform-aws-s3-buckets.svg?style=svg)](https://circleci.com/gh/appzen-oss/terraform-aws-s3-buckets)
+[![Github release](https://img.shields.io/github/release/appzen-oss/terraform-aws-s3-buckets.svg)](https://github.com/appzen-oss/terraform-aws-s3-buckets/releases)
 
-Terraform module which creates S3 buckets on AWS.
+Terraform module which creates multiple AWS S3 buckets
 
-Terraform Registry: https://registry.terraform.io/modules/devops-workflow/s3-buckets/aws
+[Terraform Registry](https://registry.terraform.io/modules/devops-workflow/s3-buckets/aws)
 
-Usage
------
+## Usage
 
 ```hcl
 module "s3-buckets" {
-  source      = "devops-workflow/s3-buckets/aws"
-  names       = ["bucket1", "bucket2", "bucket3"]
-  environment = "dev"
-  org         = "corp"
+  source       = "devops-workflow/s3-buckets/aws"
+  names        = ["bucket1", "bucket2", "bucket3"]
+  environment  = "dev"
+  organization = "corp"
 }
 ```
 
-This would create/manage 3 S3 buckets: `corp-dev-bucket1`, `corp-dev-bucket2`, and `corp-dev-bucket3`
+This would create/manage 3 S3 buckets: `corp-dev-bucket1`, `corp-dev-bucket2`,
+and `corp-dev-bucket3`
 
 If a S3 bucket already exists, you will need to import it. Like this:
 
 ```Shell
 terraform import module.s3-buckets.aws_s3_bucket.this[0] corp-dev-bucket1
 ```
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| allow_encrypted_uploads_only | Set to `true` to prevent uploads of unencrypted objects to S3 bucket | string | `false` | no |
+| attributes | Suffix name with additional attributes (policy, role, etc.) | list | `<list>` | no |
+| block\_public\_acls | Whether Amazon S3 should block public ACLs for this bucket | string | `"true"` | no |
+| block\_public\_policy | Whether Amazon S3 should block public bucket policies for this bucket | string | `"true"` | no |
+| component | TAG: Underlying, dedicated piece of service (Cache, DB, ...) | string | `"UNDEF-S3-Buckets"` | no |
+| delimiter | Delimiter to be used between `name`, `namespaces`, `attributes`, etc. | string | `"-"` | no |
+| enabled | Set to false to prevent the module from creating anything | string | `"true"` | no |
+| environment | Environment (ex: `dev`, `qa`, `stage`, `prod`). (Second or top level namespace. Depending on namespacing options) | string | n/a | yes |
+| encryption | If encryption is true, create an S3 bucket with default encryption i.e. `AES256` | string | false | no |
+| force\_destroy | Delete all objects in bucket on destroy | string | `"false"` | no |
+| ignore\_public\_acls | Whether Amazon S3 should ignore public ACLs for this bucket | string | `"true"` | no |
+| kms_master_key_arn | The AWS KMS master key ARN used for the `SSE-KMS` encryption. This can only be used when you set the value of `encryption` as `true`. The default aws/s3 AWS KMS master key is used if this element is absent | string | `` | no |
+| monitor | TAG: Should resource be monitored | string | `"UNDEF-S3-Buckets"` | no |
+| names | List of S3 bucket names | list | n/a | yes |
+| namespace-env | Prefix name with the environment. If true, format is: <env>-<name> | string | `"true"` | no |
+| namespace-org | Prefix name with the organization. If true, format is: <org>-<env namespaced name>. If both env and org namespaces are used, format will be <org>-<env>-<name> | string | `"true"` | no |
+| organization | Organization name (Top level namespace) | string | `""` | no |
+| owner | TAG: Owner of the service | string | `"UNDEF-S3-Buckets"` | no |
+| principal | principal | string | `"*"` | no |
+| product | TAG: Company/business product | string | `"UNDEF-S3-Buckets"` | no |
+| public | Allow public read access to bucket | string | `"false"` | no |
+| restrict\_public\_buckets | Whether Amazon S3 should restrict public bucket policies for this bucket | string | `"true"` | no |
+| service | TAG: Application (microservice) name | string | `"UNDEF-S3-Buckets"` | no |
+| sse_algorithm | The server-side encryption algorithm to use. Valid values are `AES256` and `aws:kms` | string | `AES256` | no |
+| tags | A map of additional tags | map | `<map>` | no |
+| team | TAG: Department/team of people responsible for service | string | `"UNDEF-S3-Buckets"` | no |
+| versioned | Version the bucket | string | `"false"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| arns | List of AWS S3 Bucket ARNs |
+| domain\_names | List of AWS S3 Bucket Domain Names |
+| hosted\_zone\_ids | List of AWS S3 Bucket Hosted Zone IDs |
+| ids | List of AWS S3 Bucket IDs |
+| name\_bases | List of base names used to generate S3 bucket names |
+| names | List of AWS S3 Bucket Names |
+| regions | List of AWS S3 Bucket Regions |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM GRAPH HOOK -->
+
+## Resource Graph of plan
+
+![Terraform Graph](resource-plan-graph.png)
+<!-- END OF PRE-COMMIT-TERRAFORM GRAPH HOOK -->