diff --git a/.gitignore b/.gitignore index cd506c5..e4c369d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ crash.*.log # Used by OPA policies **/plan.tfplan **/plan.json +**/opa_output.json # Ignore override files as they are usually used to override resources locally and so # are not checked in diff --git a/GNUmakefile b/GNUmakefile index 3679e41..6a2b086 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -7,7 +7,6 @@ DVLINT_INCLUDE_RULES:= DVLINT_IGNORE_RULES:=dv-rule-annotations-001,dv-rule-empty-flow-001 default: devcheck - check-for-terraform: @command -v terraform >/dev/null 2>&1 || { echo >&2 "'terraform' is required but not installed. Aborting."; exit 1; } @@ -39,7 +38,6 @@ dvlint: fi; \ done - validate: check-for-terraform @echo "==> Validating Terraform code with terraform validate..." @if [ -d "./$(DEV_DIR)" ]; then \ @@ -51,6 +49,16 @@ trivy: @command -v trivy >/dev/null 2>&1 || { echo >&2 "'trivy' is required but not installed. Aborting."; exit 1; } @trivy config ./ -devcheck: fmt fmt-check validate tflint dvlint trivy +shell-files: + @echo "==> Checking and formatting shell scripts..." + @command -v shfmt >/dev/null 2>&1 || { echo >&2 "'shfmt' is required but not installed. Aborting."; exit 1; } + @command -v shellcheck >/dev/null 2>&1 || { echo >&2 "'shellcheck' is required but not installed. Aborting."; exit 1; } + @echo "==> Formatting shell scripts with shfmt..." + @shfmt -w -i 4 -sr -ci ./scripts/ + + @echo "==> Checking shell scripts with shellcheck..." + @shellcheck --exclude=SC1090,SC1091 ./scripts/*.sh + +devcheck: fmt fmt-check validate tflint dvlint trivy shell-files .PHONY: devcheck fmt fmt-check validate tflint dvlint trivy \ No newline at end of file diff --git a/README.md b/README.md index cbc80f2..0701ebf 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,16 @@ To be successful in recreating the use cases supported by this pipeline, there a - Completion of all pre-requisites and configuration steps leading to [Feature Development](https://github.com/pingidentity/pipeline-example-platform?tab=readme-ov-file#feature-development) from the example-pipeline-platform repository - [Docker](https://docs.docker.com/engine/install/) - used to deploy the UI for a sample interface - [terraform](https://developer.hashicorp.com/terraform/install) - HashiCorp Terraform (version 1.9.8 was used in this guide) -- [opa](https://www.openpolicyagent.org/docs/latest/#running-opa) - Open Policy Agent (version 0.70.0 was used in this guide) +- [opa](https://www.openpolicyagent.org/docs/latest/#running-opa) - Open Policy Agent for policy enforcement (version 0.70.0 was used in this guide) - [tflint](https://github.com/terraform-linters/tflint) - for Terraform linting (version 0.53.0 was used in this guide) - [dvlint](https://github.com/pingidentity/dvlint) - for Davinci flow linting (version 1.0.3 was used in this guide) - [trivy](https://github.com/aquasecurity/trivy) - for security scanning (version 0.56.2 was used in this guide) +- [shellcheck](https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing) - for shell script linting (version 0.10.0 was used in this guide) +- [shfmt](https://github.com/mvdan/sh) - for shell script formatting (version 3.10.0 was used in this guide) +- [jq](https://jqlang.github.io/jq/download/) - for JSON parsing (version 1.7.1 was used in this guide) > [!TIP] -> The last three tools are used by the pipeline in Github, and the pipeline will fail if these tests and configuration checks do not pass. Installing these tools locally and running `make devcheck` before committing changes should ensure that the pipeline will pass when changes are pushed. +> The last six tools are used by the pipeline in Github, and the pipeline will fail if these tests and configuration checks do not pass. To help ensure the pipeline instance of these tools passes, install these tools locally and run `make devcheck` before committing changes > [!IMPORTANT] @@ -67,15 +70,6 @@ Click the **Use this template** button at the top right of this page to create y > [!NOTE] > A pipeline will run and fail when the repository is created. This result is expected as the pipeline is attempting to deploy the application and the necessary configuration has not yet been completed. -Create a `qa` branch from the `prod` branch in the repository. This branch will be used to test the changes before they are promoted to the `prod` branch. Changes to the `qa` branch in this repository are deployed to the `qa` environment in PingOne. As with the `prod` branch, the pipeline will fail due to missing configuration. - -```bash -git checkout prod -git pull origin prod -git checkout -b qa -git push origin qa -``` - ## Development Lifecycle Diagram The use cases in this repository follow a flow similar to this diagram: @@ -211,7 +205,7 @@ source localsecrets 12. To capture the changes for inclusion in your code, export the flow. You can do so by selecting the three dots at the top right of the DaVinci flow editor UI and clicking **Download Flow JSON**. Ensure to select **Include Variable Values** when you export. -![Export Menu](./img/exportMenu.png "Export Menu") +![Export Menu](./img/pingOneEnvs.png "Export Menu") 13. For the sake of brevity, assume that testing has been done, and you are ready to proceed. After the application is "tested", the new configuration must be added to the Terraform configuration. This addition will happen in a few steps: diff --git a/scripts/lib.sh b/scripts/lib.sh index 0219d60..2b2f1fc 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,36 +1,37 @@ #!/usr/bin/env sh +# shellcheck disable=SC2154 ## this holds the common functions used by other scripts #### checkVars() { - for var in \ - "${TF_VAR_pingone_client_region_code}" \ - "${TF_VAR_pingone_client_environment_id}" \ - "${TF_VAR_pingone_client_id}" \ - "${TF_VAR_pingone_client_secret}" \ - "${TF_VAR_pingone_davinci_admin_username}" \ - "${TF_VAR_pingone_davinci_admin_password}" \ - "${TF_VAR_pingone_davinci_admin_environment_id}" \ - "${AWS_ACCESS_KEY_ID}" \ - "${AWS_SECRET_ACCESS_KEY}" \ - "${TF_VAR_tf_state_bucket}" \ - "${TF_VAR_tf_state_region}" \ - "${TF_VAR_tf_state_key_prefix}" ; do - if [ -z "${var}" ]; then - echo "Please set the required environment variables: - TF_VAR_pingone_region_code - TF_VAR_pingone_environment_id - TF_VAR_pingone_client_id - TF_VAR_pingone_client_secret - TF_VAR_pingone_davinci_admin_username - TF_VAR_pingone_davinci_admin_password - TF_VAR_pingone_davinci_admin_environment_id - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - TF_VAR_tf_state_bucket - TF_VAR_tf_state_region - TF_VAR_tf_state_key_prefix" - exit 1 - fi - done -} \ No newline at end of file + for var in \ + "${TF_VAR_pingone_client_region_code}" \ + "${TF_VAR_pingone_client_environment_id}" \ + "${TF_VAR_pingone_client_id}" \ + "${TF_VAR_pingone_client_secret}" \ + "${TF_VAR_pingone_davinci_admin_username}" \ + "${TF_VAR_pingone_davinci_admin_password}" \ + "${TF_VAR_pingone_davinci_admin_environment_id}" \ + "${AWS_ACCESS_KEY_ID}" \ + "${AWS_SECRET_ACCESS_KEY}" \ + "${TF_VAR_tf_state_bucket}" \ + "${TF_VAR_tf_state_region}" \ + "${TF_VAR_tf_state_key_prefix}"; do + if [ -z "${var}" ]; then + echo "Please set the required environment variables: + TF_VAR_pingone_region_code + TF_VAR_pingone_environment_id + TF_VAR_pingone_client_id + TF_VAR_pingone_client_secret + TF_VAR_pingone_davinci_admin_username + TF_VAR_pingone_davinci_admin_password + TF_VAR_pingone_davinci_admin_environment_id + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + TF_VAR_tf_state_bucket + TF_VAR_tf_state_region + TF_VAR_tf_state_key_prefix" + exit 1 + fi + done +} diff --git a/scripts/local_feature_deploy.sh b/scripts/local_feature_deploy.sh index 570d6ff..581f93c 100755 --- a/scripts/local_feature_deploy.sh +++ b/scripts/local_feature_deploy.sh @@ -1,16 +1,16 @@ #!/usr/bin/env sh +# shellcheck disable=SC2154 ### this script is used to run terraform apply for your local feature branch only. ### test -f scripts/lib.sh || { - echo "Please run the script from the root of the repository" - exit 1 + echo "Please run the script from the root of the repository" + exit 1 } _command="apply" -usage () -{ -cat < "${OPADIR}"/plan.json +} + +opa_eval() { + echo "" + echo "" + echo "##################################################" + echo "Running OPA policy evaluation..." + echo "" + echo "" + # We want to continue running the script even if the OPA evaluation fails (|| true) + # shellcheck disable=SC2015 + cd "${OPADIR}" && opa eval -i plan.json -d ./policies/flow_checks.rego -d ./policies/davinci_flow.rego -d ./policies/library.rego 'data.terraform.davinci.deny' > opa_output.json || true + + # Check if the deny result is non-empty + deny_result=$(jq '.result[0].expressions[0].value' opa_output.json) + + cd .. + + if [[ "$(echo "$deny_result" | jq -e 'keys | length > 0')" == "true" ]]; then + echo "Policy Failed: Deny messages found." + echo "Deny messages:" + echo "$deny_result" | jq . + exit 1 + else + echo "Policy Passed: No deny messages found." + fi +} + +run_policy_tests() { + # Check if verbose mode is enabled + if echo "$-" | grep -q x; then + echo "Running tests in verbose mode..." + cd "${OPADIR}" && opa test -v ./policies/flow_checks.rego ./policies/davinci_flow.rego ./policies/library.rego ./policies/davinci_flow_test.rego && cd .. + else + echo "Running tests..." + cd "${OPADIR}" && opa test ./policies/flow_checks.rego ./policies/davinci_flow.rego ./policies/library.rego ./policies/davinci_flow_test.rego && cd .. + fi +} + +exit_usage() { + echo "$*" + usage +} + +# Flag to track if verbose mode is enabled +verbose_mode=false + +# Flag to track if any arguments were provided +arguments_provided=false + +while ! test -z "${1}"; do + case "${1}" in + --eval-only) + arguments_provided=true + opa_eval + ;; + --plan-and-eval) + arguments_provided=true + init_all + init_s3 + terraform_init + opa_eval + ;; + --run-policy-tests) + arguments_provided=true + run_policy_tests + ;; + -v | --verbose) + verbose_mode=true + set -x + ;; + -h | --help) + exit_usage "" + ;; + *) + exit_usage "Unrecognized Option" + ;; + esac + shift +done + +# Enable verbose mode globally if the flag is set +if [ "$verbose_mode" = true ]; then + set -x +fi + +# Default to `run_tests` if no arguments were provided +if [ "$arguments_provided" = false ]; then + echo "No options provided. Defaulting to running the sample OPA policy tests..." + echo "Use --help for more options." + run_policy_tests +fi diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh new file mode 100755 index 0000000..ce44d73 --- /dev/null +++ b/scripts/shellcheck.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env sh +# +# Ping Identity DevOps - CI scripts +# +# This script finds all shell files in the current working directory (recursive) +# and runs shellcheck on them. Shellcheck warnings and number of files with warnings are printed. +# +test "${VERBOSE}" = "true" && set -x + +command_prefix="" +if ! type shellcheck; then + if test "$(uname -m)" = "x86_64" && test "$(uname -s)" = "Linux"; then + echo "INFO: Downloading latest shellcheck version for Linux" + + # Install shellcheck + shellcheck_filename="shellcheck.tar.xz" + + # Download the latest version of shellcheck for linux x86_64 from GitHub. + shellcheck_download_url=$(curl --silent https://api.github.com/repos/koalaman/shellcheck/releases/latest | jq -r '.assets[] | select(.name|test("linux.x86_64")) | .browser_download_url') + test -z "${shellcheck_download_url}" && echo "Error: Failed to retrieve shellcheck download URL" && exit 1 + curl --location --silent --output "${shellcheck_filename}" "${shellcheck_download_url}" + test $? -ne 0 && echo "Error: Failed to retrieve shellcheck tar file from GitHub" && exit 1 + + # Extract the binary + tar -xf "${shellcheck_filename}" --exclude 'README.txt' --exclude 'LICENSE.txt' --strip-components=1 + test $? -ne 0 && echo "Error: Failed to extract shellcheck binary" && exit 1 + + # Give execute permissions to shellcheck + chmod +x shellcheck + test $? -ne 0 && echo "Error: Failed to exit execute permissions on shellcheck binary" && exit 1 + + command_prefix="./" + else + echo "Missing shellcheck" + exit 99 + fi +fi + +# For each file in the project, if it has a .sh extension, add it to a tmp file +find "$(pwd)" -type f -not -path '*/\.*' -name "*.sh" > tmp + +test "${VERBOSE}" = "true" && echo "Files in use:" && cat tmp + +num_files_fail_shellcheck=0 +# Scan each file in tmp with shellcheck +while IFS= read -r shell_file; do + #Exclude SC1090 and SC1091 as files are not being found, despite shellcheck source definitions. + "${command_prefix}shellcheck" --exclude=SC1090,SC1091 "${shell_file}" + test $? -ne 0 && num_files_fail_shellcheck=$((num_files_fail_shellcheck + 1)) +done < tmp +rm tmp + +test -n "${command_prefix}" && rm shellcheck + +echo "Number of Files with Shellcheck Warnings: ${num_files_fail_shellcheck}" + +if test ${num_files_fail_shellcheck} -ne 0; then + exit 1 +fi +exit 0 diff --git a/scripts/shfmt.sh b/scripts/shfmt.sh new file mode 100755 index 0000000..7d52e53 --- /dev/null +++ b/scripts/shfmt.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env sh +# +# Ping Identity DevOps - CI scripts +# +# This script finds all shell files in the current working directory (recursive) +# which contain formatting that differs from shfmt's configuration. +# In --diff mode, diffs of these occurrences are printed +# In --write mode, files are updated to use the configured format for the project. +# +test "${VERBOSE}" = "true" && set -x + +# Usage printing function +usage() { + test -n "${*}" && echo "${*}" + cat << END_USAGE +Usage: ${0} {options} + where {options} include: + -d, --diff + Finds all shell files in the current working directory (recursive) + which contain formatting that differs from shfmt's configuration. + If found, diffs of these occurrences are printed. This is the default. + -w, --write + Overwrites all shell files in the current working directory (recursive) + which contain formatting that differs from shfmt's configuration. + If found, files are updated to use the configured format for the project. + -f, --file + target a specific file rather than the default behavior of analyzing + all the eligible files in the project + --help + Display general usage information +END_USAGE + exit 99 +} + +format_file() { + #Checks for space indents of size 4 + #Checks for spaces following redirects + #Checks for indented switch case statements + #Prints diff of actual file vs shfmt format + "${command_prefix}shfmt" -i 4 -sr -ci "${mode}" "${1}" + return ${?} +} + +while ! test -z "${1}"; do + case "${1}" in + -d | --diff) + mode="-d" + ;; + -w | --write) + mode="-w" + ;; + -f | --file) + shift + file="${1}" + ;; + --help) + usage + ;; + *) + usage "Unrecognized option" + ;; + esac + shift +done + +#Default mode to --diff +if test -z "${mode}"; then + mode="-d" +fi + +command_prefix="" +if ! type shfmt; then + if test "$(uname -m)" = "x86_64" && test "$(uname -s)" = "Linux"; then + echo "INFO: Downloading latest shfmt version for Linux" + + # Install shfmt + shfmt_filename="shfmt" + + # Download the latest version of shfmt for linux x86_64 from GitHub. + # For github actions + shfmt_download_url=$(curl --silent https://api.github.com/repos/mvdan/sh/releases/latest | jq -r '.assets[] | select(.name|test("linux_amd64")) | .browser_download_url') + test -z "${shfmt_download_url}" && echo "Error: Failed to retrieve shfmt download URL" && exit 1 + curl --location --silent --output "${shfmt_filename}" "${shfmt_download_url}" + test $? -ne 0 && echo "Error: Failed to retrieve shfmt binary from GitHub" && exit 1 + + # Give execute permissions to shfmt + chmod +x shfmt + test $? -ne 0 && echo "Error: Failed to exit execute permissions on shfmt binary" && exit 1 + + command_prefix="./" + else + echo "Missing shfmt" + exit 99 + fi +fi + +num_files_fail_shfmt=0 +if test -n "${file}"; then + echo "Checking format of ${file}" + format_file "${file}" + test $? -ne 0 && num_files_fail_shfmt=$((num_files_fail_shfmt + 1)) +else + # For each file in the project, if it starts with a shebang, add it to a list in tmp. + find "$(pwd)" -type f -not -path '*/\.*' -name "*.sh" > tmp + + test "${VERBOSE}" = "true" && echo "Files in use:" && cat tmp + + # Scan each file in tmp with shfmt + while IFS= read -r shell_file; do + format_file "${shell_file}" + test $? -ne 0 && num_files_fail_shfmt=$((num_files_fail_shfmt + 1)) + done < tmp + rm tmp +fi +test -n "${command_prefix}" && rm shfmt + +test "${mode}" = "-d" && echo "Number of Files that do not match shfmt format: ${num_files_fail_shfmt}" +if test ${num_files_fail_shfmt} -ne 0; then + exit 1 +fi +exit 0 diff --git a/terraform/vars.tf b/terraform/vars.tf index 51bb3d1..df2e877 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -24,5 +24,5 @@ variable "pingone_davinci_admin_region" { } variable "pingone_target_environment_id" { type = string - description = "The target environment id to deploy the application to" + description = "The target environment id to which to deploy the application" } \ No newline at end of file