diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9e5a489 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,106 @@ +name: ci + +# Static checks, native variable validation tests, the Python pre-flight check, +# and the plan-based pytest suite. Runs on every PR with relevant path changes +# and on pushes to master / release/v* / the in-flight v26 chore branches. + +on: + pull_request: + paths: + - "**/*.tf" + - "**/*.tftest.hcl" + - "templates/TEMPLATE_terraform.tfvars" + - "scripts/installer/**" + - "tests/**" + - "Makefile" + - ".github/workflows/ci.yml" + - ".tflint.hcl" + push: + branches: + - master + - "release/v*" + workflow_dispatch: + +# A push to a branch with an open PR fires both the push and pull_request +# triggers, producing duplicate runs. Group by ref + event so the second one +# cancels the first. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + ci: + name: ci + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.14.8 + terraform_wrapper: false + + - name: terraform fmt + run: terraform fmt -check -recursive -diff + + - name: Setup tflint + uses: terraform-linters/setup-tflint@v4 + with: + tflint_version: v0.61.0 + + - name: tflint init + run: tflint --init + + - name: tflint + # TFLINT_CONFIG_FILE is required so child modules walked by --recursive + # use the root .tflint.hcl — they don't auto-discover it. + env: + TFLINT_CONFIG_FILE: ${{ github.workspace }}/.tflint.hcl + run: tflint --recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: | + pip install -r scripts/installer/requirements.txt + pip install -r tests/requirements.txt + + - name: Terraform init (modules only) + run: terraform init -backend=false + + - name: Generate test fixtures + env: + CX_SKIP_SSM: "true" + run: | + mkdir -p tests/logs + make generate_test_data + + - name: Run terraform test + run: make terraform_test + + - name: extractors.py smoke + # Confirms the Python tfvars parser reads the fixture end-to-end without a + # Docker daemon. Only became possible once extractors.py stopped shelling + # out to hcl2json. The `installer` package lives under scripts/, so we + # have to add it to PYTHONPATH for the inline invocation. + env: + PYTHONPATH: scripts + run: | + cp tests/datafiles/terraform.tfvars terraform.tfvars + trap "rm -f terraform.tfvars" EXIT + python3 -c "from installer.utils.extractors import tf_vars_json_payload; assert tf_vars_json_payload['tower_container_version'].startswith('v'); print(f'parsed {len(tf_vars_json_payload)} keys')" + + # Follow-up: wire the @pytest.mark.local plan-based suite in here. Now that + # extractors.py runs Docker-free, the conftest session_setup also runs + # Docker-free (hcl2 swap + tfvars-backup gating + CX_SKIP_AWS_CHECK), and + # tests/requirements.txt is fixed, the remaining blocker is the bare + # `terraform plan` invocations in the pytest executor — those need the + # AWS provider to initialize, and the fixture's `aws_profile` + lack of + # AWS_* env creds in CI need a coherent solution. Tackle separately. diff --git a/.github/workflows/terraform-test.yml b/.github/workflows/terraform-test.yml deleted file mode 100644 index ba85a3f..0000000 --- a/.github/workflows/terraform-test.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: terraform test - -# Runs the variable-validation suite under tests/terraform/. -# See tests/terraform/README.md for what's covered and how to run locally. - -on: - pull_request: - paths: - - "**/*.tf" - - "**/*.tftest.hcl" - - "templates/TEMPLATE_terraform.tfvars" - - "scripts/installer/validation/**" - - "scripts/installer/data_external/**" - - "scripts/installer/utils/**" - - "tests/datafiles/**" - - "tests/terraform/**" - - "Makefile" - - ".github/workflows/terraform-test.yml" - push: - branches: - - master - - "release/v*" - - "chore/v26-**" # Temporary: keep visibility while the stacked v26 work is in flight. - workflow_dispatch: - -jobs: - terraform-test: - name: terraform test - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.14.8 - terraform_wrapper: false - - - name: terraform fmt - run: terraform fmt -check -recursive -diff - - - name: Setup tflint - uses: terraform-linters/setup-tflint@v4 - with: - tflint_version: v0.61.0 - - - name: tflint init - run: tflint --init - - - name: tflint - # TFLINT_CONFIG_FILE is required so child modules walked by --recursive - # use the root .tflint.hcl — they don't auto-discover it. - env: - TFLINT_CONFIG_FILE: ${{ github.workspace }}/.tflint.hcl - run: tflint --recursive - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - # tests/requirements.txt is the pytest-suite fixture; this job only needs - # stdlib Python (generate_testing_secrets.py uses no third-party imports). - - - name: Terraform init (modules only) - run: terraform init -backend=false - - - name: Generate test fixtures - env: - CX_SKIP_SSM: "true" - run: | - mkdir -p tests/logs - make generate_test_data - - - name: Run terraform test - run: make terraform_test diff --git a/005_parameter_store.tf b/005_parameter_store.tf index 619bc87..6df1d0f 100644 --- a/005_parameter_store.tf +++ b/005_parameter_store.tf @@ -26,22 +26,20 @@ data "aws_ssm_parameter" "wave_lite_secrets" { # Generate individual SSM Parameters # ------------------------------------------------ resource "aws_ssm_parameter" "client_supplied_secrets_tower" { - for_each = local.tower_secret_keys - name = nonsensitive(local.tower_secrets[each.key]["ssm_key"]) - value = local.tower_secrets[each.key]["value"] - type = "SecureString" - overwrite = var.flag_overwrite_ssm_keys + for_each = local.tower_secret_keys + name = nonsensitive(local.tower_secrets[each.key]["ssm_key"]) + value = local.tower_secrets[each.key]["value"] + type = "SecureString" } resource "aws_ssm_parameter" "client_supplied_secrets_seqerakit" { # for_each = local.seqerakit_secret_keys - for_each = var.flag_run_seqerakit == true ? local.seqerakit_secret_keys : [] - name = nonsensitive(local.seqerakit_secrets[each.key]["ssm_key"]) - value = local.seqerakit_secrets[each.key]["value"] - type = "SecureString" - overwrite = var.flag_overwrite_ssm_keys + for_each = var.flag_run_seqerakit == true ? local.seqerakit_secret_keys : [] + name = nonsensitive(local.seqerakit_secrets[each.key]["ssm_key"]) + value = local.seqerakit_secrets[each.key]["value"] + type = "SecureString" } @@ -50,19 +48,17 @@ resource "aws_ssm_parameter" "client_supplied_secrets_groundswell" { # count = var.flag_enable_groundswell == true ? 1 : 0 # for_each = local.groundswell_secret_keys - for_each = var.flag_enable_groundswell == true ? local.groundswell_secret_keys : [] - name = nonsensitive(local.groundswell_secrets[each.key]["ssm_key"]) - value = local.groundswell_secrets[each.key]["value"] - type = "SecureString" - overwrite = var.flag_overwrite_ssm_keys + for_each = var.flag_enable_groundswell == true ? local.groundswell_secret_keys : [] + name = nonsensitive(local.groundswell_secrets[each.key]["ssm_key"]) + value = local.groundswell_secrets[each.key]["value"] + type = "SecureString" } resource "aws_ssm_parameter" "client_supplied_secrets_wave_lite" { - for_each = var.flag_use_wave_lite == true ? local.wave_lite_secret_keys : [] - name = nonsensitive(local.wave_lite_secrets[each.key]["ssm_key"]) - value = local.wave_lite_secrets[each.key]["value"] - type = "SecureString" - overwrite = var.flag_overwrite_ssm_keys + for_each = var.flag_use_wave_lite == true ? local.wave_lite_secret_keys : [] + name = nonsensitive(local.wave_lite_secrets[each.key]["ssm_key"]) + value = local.wave_lite_secrets[each.key]["value"] + type = "SecureString" } diff --git a/scripts/installer/requirements.txt b/scripts/installer/requirements.txt new file mode 100644 index 0000000..1dca5ae --- /dev/null +++ b/scripts/installer/requirements.txt @@ -0,0 +1 @@ +python-hcl2==8.1.2 diff --git a/scripts/installer/utils/extractors.py b/scripts/installer/utils/extractors.py index 815d0a7..06271de 100644 --- a/scripts/installer/utils/extractors.py +++ b/scripts/installer/utils/extractors.py @@ -1,96 +1,57 @@ -from datetime import datetime -import json -import logging import os -from pathlib import Path -import platform -import subprocess import sys -import tempfile +from pathlib import Path + +import hcl2 base_import_dir = Path(__file__).resolve().parents[2] if base_import_dir not in sys.path: sys.path.append(str(base_import_dir)) -from installer.utils.logger import logger ## ------------------------------------------------------------------------------------ -## Convert terraform.tfvars to JSON -## Notes: -## 1. As of May 16, 2025, the bespoke parser to convert terraform.tfvars into json is replaced with a new container-based -## solution from tmccombs/hcl2json. The home-rolled parser was originally created to avoid introducing packages from -## the internet. Due to more complicated parsing needs, the new parser solution has been implemented. -## To refer to the previous parser, please refer to previous Git histroy. +## Convert terraform.tfvars to a Python dict. ## -## 2. The new parser relies on the containerized solution provided here: https://github.com/tmccombs/hcl2json. -## Seqera will vendor their own copy of the image within Harbor. +## Earlier revisions of this file shelled out to the `tmccombs/hcl2json` Docker image +## (see git history). That worked but required a Docker daemon to be running anywhere +## the installer was exercised — including CI and `terraform test` runs that touch the +## connection-strings external data source. Switching to `python-hcl2` removes the +## daemon dependency in exchange for one in-process Python library. ## -## WARNING / REMINDER: DONT ADD ANY stdout emissions in this logic or you'll break the TF `external` mechanism!! +## WARNING: do not emit anything to stdout from this module — the data.external block in +## modules/connection_strings/v1.0.0/main.tf relies on a single JSON line on stdout from +## generate_db_connection_string.py, and any extra output corrupts the protocol. ## ------------------------------------------------------------------------------------ -def get_tfvars_as_json(): - """ - Uses the `tmccombs/hcl2json` Docker image to convert `terraform.tfvars` into JSON format and return it as - a Python dictionary. +def _unwrap_hcl_strings(value): + """python-hcl2 returns string scalars with their surrounding double quotes preserved + (so `foo = "bar"` becomes the Python string `'"bar"'`). The downstream consumers + expect plain Python strings. Walk the parsed structure and strip those. """ + if isinstance(value, str): + if len(value) >= 2 and value[0] == '"' and value[-1] == '"': + return value[1:-1] + return value + if isinstance(value, list): + return [_unwrap_hcl_strings(item) for item in value] + if isinstance(value, dict): + return {key: _unwrap_hcl_strings(val) for key, val in value.items()} + return value + - # Check for tfvars - tfvars_original_path = os.path.abspath("terraform.tfvars") - if not os.path.exists(tfvars_original_path): +def get_tfvars_as_json(): + """Parse the project-root `terraform.tfvars` into a dict.""" + tfvars_path = os.path.abspath("terraform.tfvars") + if not os.path.exists(tfvars_path): raise FileNotFoundError( - f"terraform.tfvars file not found in path: {tfvars_original_path}." + f"terraform.tfvars file not found in path: {tfvars_path}." ) - # Because this is a 3rd party container, we are locking down as much as possible for security reasons: - # Single local file mounted as read-only, non-root user, disabled network capabilities, and - # use of immutable container hash fingerprint over mutable tag. - cmd = [ - "docker", - "run", - "--platform", - "linux/amd64", - "-i", - "--rm", - "-v", - f"{os.getcwd()}/terraform.tfvars:/tmp/terraform.tfvars:ro", - "--user", - "1000:1000", - "--network", - "none", - # "tmccombs/hcl2json@sha256:312ac54d3418b87b2ad64f82027483cb8b7750db6458b7b9ebe42ec278696e96", - "ghcr.io/seqeralabs/cx-field-tools-installer/hcl2json@sha256:48af2029d19d824ba1bd1662c008e691c04d5adcb055464e07b2dc04980dcbf5", - "/tmp/terraform.tfvars", - ] - - # Assign result to variable (to then be returned directly or written to file for debugging) - result = subprocess.run(cmd, capture_output=True, text=True) - - # Capture Docker runtime failures - if result.returncode != 0: - raise RuntimeError(f"Docker command failed:\nSTDERR: {result.stderr.strip()}") - - # Capture Docker stdout - payload = result.stdout.strip() - - # Redirect output to temporary JSON file (debugging only) - if logging.getLevelName(logger.getEffectiveLevel()) == "DEBUG": - timestamp = datetime.now().strftime("%Y_%b%d_%I-%M%p") - with tempfile.NamedTemporaryFile( - delete=False, - prefix=f"terraform_tfvars_{timestamp}_", - suffix=".json", - mode="w+", - dir="/tmp", - ) as temp_output: - temp_output.write(payload) + with open(tfvars_path) as fp: + parsed = hcl2.load(fp) - # Capture invalid json errors - try: - # logger.info(payload) - return json.loads(payload) # json.loads converts json string to python object. - except json.JSONDecodeError as e: - raise RuntimeError("Failed to decode Docker output as JSON.") from e + return _unwrap_hcl_strings(parsed) tf_vars_json_payload = get_tfvars_as_json() diff --git a/templates/TEMPLATE_terraform.tfvars b/templates/TEMPLATE_terraform.tfvars index 209a631..87c95a1 100644 --- a/templates/TEMPLATE_terraform.tfvars +++ b/templates/TEMPLATE_terraform.tfvars @@ -43,24 +43,6 @@ aws_profile = "REPLACE_ME" tower_container_version = "v25.3.0" -/* -## ------------------------------------------------------------------------------------ -## SSM -## ------------------------------------------------------------------------------------ -Activate this setting to allow n+1 deployments to overwrite SSM keys if they were not -generated by your own Terraform project (can be useful to set this to true if running 2+ -instances off of the same configuration, otherwise you will get errors at the end of the -`apply` cycle. - -Note! - - 1) Terraform will show a deprecation warning -- there is no better option than this - however, so we continue to use this option for now). - - 2) Setting this value to `false` will not prevent overrwrites if SSM entries are - tracked within your own project state. -*/ -flag_overwrite_ssm_keys = true - - /* ## ------------------------------------------------------------------------------------ ## Tags -- Default diff --git a/tests/conftest.py b/tests/conftest.py index 7b3ab70..f826550 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,10 @@ +import json import os import subprocess import time from pathlib import Path +import hcl2 import pytest # Single-sourced in top-level conftest.py. @@ -49,8 +51,14 @@ def session_setup(): # Create a fresh copy of the base testing terraform.tfvars file. subprocess.run("make generate_test_data", shell=True, check=True) - print("\nBacking up terraform.tfvars.") - FileHelper.move_file(FP.TFVARS_BASE, FP.TFVARS_BACKUP) + # Back up the project-root terraform.tfvars so a tester's local config is restored + # post-session. CI runs don't have a pre-existing tfvars at the root, so skip when + # there's nothing to back up. + if Path(FP.TFVARS_BASE).exists(): + print("\nBacking up terraform.tfvars.") + FileHelper.move_file(FP.TFVARS_BASE, FP.TFVARS_BACKUP) + else: + print("\nNo project-root terraform.tfvars to back up; continuing.") # Swap in test tfvars (base and base-override), and testing-specific outputs (e.g. locals). print("\nLoading test tfvars and output artefacts.") @@ -61,13 +69,13 @@ def session_setup(): # Prepare plan cache directory os.makedirs(FP.CACHE_PLAN_DIR, exist_ok=True) - # Prepare JSONified 009 (via hcl2json container) - # CLI_command = "./hcl2json 009_define_file_templates.tf > 009_define_file_templates.json" - command = ( - f"docker run --rm -v {FP.ROOT}:/tmp ghcr.io/seqeralabs/cx-field-tools-installer/hcl2json:vendored" - f" /tmp/009_define_file_templates.tf > {FP.ROOT}/009_define_file_templates.json" - ) - result = execute_subprocess(command) + # Prepare JSONified 009 — read by tests/utils/terraform/template_generator.py. + # Switched from a docker hcl2json call to in-process python-hcl2 so the suite + # can run without a Docker daemon. + with open(f"{FP.ROOT}/009_define_file_templates.tf") as fp: + parsed = hcl2.load(fp) + with open(f"{FP.ROOT}/009_define_file_templates.json", "w") as fp: + json.dump(parsed, fp) yield @@ -91,8 +99,9 @@ def session_setup(): delete_pycache_folders(FP.ROOT) - # Restore original tfvars - FileHelper.move_file(FP.TFVARS_BACKUP, FP.TFVARS_BASE) + # Restore original tfvars (only if we backed one up at session start). + if Path(FP.TFVARS_BACKUP).exists(): + FileHelper.move_file(FP.TFVARS_BACKUP, FP.TFVARS_BASE) @pytest.fixture(scope="session") # function diff --git a/tests/requirements.txt b/tests/requirements.txt index 7d96145..f2b71f7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,9 @@ pytest +pyyaml yamlpath testcontainers testcontainers[postgres] cryptography # To suppress pytest warnings -# Note: Requests 2.32.1+ causes problems with `docker-py` and Testcontainer (e.g. "Not supported URL scheme http+docker"). -# Resolve by downgrading requests -pip install requests=2.31.0 +# Requests 2.32.1+ causes "Not supported URL scheme http+docker" with docker-py / testcontainers. +requests==2.31.0 diff --git a/tests/terraform/cross_variable_validation.tftest.hcl b/tests/terraform/cross_variable_validation.tftest.hcl index 92a755f..300d871 100644 --- a/tests/terraform/cross_variable_validation.tftest.hcl +++ b/tests/terraform/cross_variable_validation.tftest.hcl @@ -34,16 +34,6 @@ override_data { } } -override_data { - target = module.connection_strings.data.external.generate_db_connection_string - values = { - result = { - status = "0" - value = "?permitMysqlScheme=true" - } - } -} - run "rejects_two_vpc_sources_true" { command = plan diff --git a/tests/terraform/string_shape_validation.tftest.hcl b/tests/terraform/string_shape_validation.tftest.hcl index 1817b84..e7b4987 100644 --- a/tests/terraform/string_shape_validation.tftest.hcl +++ b/tests/terraform/string_shape_validation.tftest.hcl @@ -33,16 +33,6 @@ override_data { } } -override_data { - target = module.connection_strings.data.external.generate_db_connection_string - values = { - result = { - status = "0" - value = "?permitMysqlScheme=true" - } - } -} - run "rejects_http_prefix_on_tower_server_url" { command = plan diff --git a/tests/terraform/version_validation.tftest.hcl b/tests/terraform/version_validation.tftest.hcl index b9b9c6a..c82d826 100644 --- a/tests/terraform/version_validation.tftest.hcl +++ b/tests/terraform/version_validation.tftest.hcl @@ -34,16 +34,6 @@ override_data { } } -override_data { - target = module.connection_strings.data.external.generate_db_connection_string - values = { - result = { - status = "0" - value = "?permitMysqlScheme=true" - } - } -} - run "rejects_v25_below_floor" { command = plan diff --git a/tests/terraform/workspace_id_validation.tftest.hcl b/tests/terraform/workspace_id_validation.tftest.hcl index b2d1bbf..2669d48 100644 --- a/tests/terraform/workspace_id_validation.tftest.hcl +++ b/tests/terraform/workspace_id_validation.tftest.hcl @@ -33,16 +33,6 @@ override_data { } } -override_data { - target = module.connection_strings.data.external.generate_db_connection_string - values = { - result = { - status = "0" - value = "?permitMysqlScheme=true" - } - } -} - run "rejects_non_numeric_data_studio_eligible_workspaces" { command = plan diff --git a/tests/utils/preflight/preflight.py b/tests/utils/preflight/preflight.py index 81392d7..e5c7a3c 100644 --- a/tests/utils/preflight/preflight.py +++ b/tests/utils/preflight/preflight.py @@ -1,4 +1,5 @@ import json +import os import subprocess import sys @@ -7,8 +8,14 @@ def check_aws_sso_token(): """ Terraform plan will fail if valid AWS SSO token not present. Invoke AWS CLI via subprocess call (STS get-caller-identity). - Raises RuntimeError if token is expired or invalid + Raises RuntimeError if token is expired or invalid. + + Set CX_SKIP_AWS_CHECK=true to bypass — used by CI runs of the plan-based + suite, where the fixture sets `use_mocks = true` so no real AWS calls fire. """ + if os.environ.get("CX_SKIP_AWS_CHECK", "").lower() in ("1", "true", "yes"): + return True + try: result = subprocess.run( ["aws", "sts", "get-caller-identity"], diff --git a/variables.tf b/variables.tf index 5dc25bd..9280367 100644 --- a/variables.tf +++ b/variables.tf @@ -64,17 +64,6 @@ variable "tower_container_version" { } -# ------------------------------------------------------------------------------------ -# SSM -# ------------------------------------------------------------------------------------ - -variable "flag_overwrite_ssm_keys" { - type = bool - description = "Not to be used in PROD but helpful when sharing same instance in DEV." - default = false -} - - # ------------------------------------------------------------------------------------ # Tags -- Default # ------------------------------------------------------------------------------------