diff --git a/.gitignore b/.gitignore index 8d108886..90749d19 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ tests/logs/* tests/datafiles/*.tfvars tests/datafiles/secrets/*.json tests/.templatefile_cache + +# Generated files from modules +**/generated/ diff --git a/modules/seqerakit/README.md b/modules/seqerakit/README.md new file mode 100644 index 00000000..f6163da6 --- /dev/null +++ b/modules/seqerakit/README.md @@ -0,0 +1,195 @@ +# Seqerakit Module + +Generates and manages Seqerakit configuration files and scripts for Tower/Seqera Platform integration. + +## Features + +- Generates Seqerakit setup YAML configuration +- Creates AWS Batch compute environment configurations +- Manages secrets loading from AWS Systems Manager (SSM) +- Supports both Fusion v2 and traditional compute environments +- Handles multiple credential types (AWS, GitHub, Docker, CodeCommit) +- Generates CodeCommit workspace ID scripts + +## Usage + +```hcl +module "seqerakit" { + source = "./modules/seqerakit/v1.0.0" + + # Required Configuration + aws_account = var.aws_account + aws_region = var.aws_region + aws_profile = var.aws_profile + default_tags = var.default_tags + app_name = var.app_name + + # Seqerakit Settings + flag_run_seqerakit = true + seqerakit_org_name = "my-org" + seqerakit_org_fullname = "My Organization" + seqerakit_org_url = "https://my-org.seqera.io" + seqerakit_team_name = "my-team" + seqerakit_team_members = "user1@example.com,user2@example.com" + seqerakit_workspace_name = "my-workspace" + seqerakit_workspace_fullname = "My Workspace" + seqerakit_compute_env_name = "my-compute-env" + seqerakit_compute_env_region = "us-west-2" + seqerakit_root_bucket = "my-s3-bucket" + seqerakit_workdir = "/tmp/work" + seqerakit_outdir = "/tmp/output" + + # AWS Configuration + seqerakit_aws_use_fusion_v2 = true + seqerakit_aws_use_forge = true + seqerakit_aws_use_batch = true + seqerakit_aws_fusion_instances = "m5.large,m5.xlarge" + seqerakit_aws_normal_instances = "t3.medium,t3.large" + seqerakit_aws_manual_head_queue = "head-queue" + seqerakit_aws_manual_compute_queue = "compute-queue" + + # Credentials Configuration + seqerakit_flag_credential_create_aws = true + seqerakit_flag_credential_create_github = true + seqerakit_flag_credential_create_docker = true + seqerakit_flag_credential_create_codecommit = true + seqerakit_flag_credential_use_aws_role = false + + # Infrastructure (optional) + vpc_id = var.vpc_id + subnet_ids = var.subnet_ids + security_group_ids = var.security_group_ids + ec2_key_pair_name = var.ec2_key_pair_name + + # Secrets + secrets_bootstrap_seqerakit = "/myapp/seqerakit/secrets" +} +``` + +## Secrets Management + +### Environment Variable Approach +The module uses environment variables instead of hardcoded secrets for security: + +```bash +# Load secrets from SSM into environment variables +source helpers/load_seqerakit_secrets.sh + +# Run terraform to generate config files +terraform apply +``` + +### SSM Parameter Structure +Store secrets in SSM as a JSON object: + +```json +{ + "TOWER_AWS_USER": {"value": "AKIA..."}, + "TOWER_AWS_PASSWORD": {"value": "secret..."}, + "TOWER_AWS_ROLE": {"value": "arn:aws:iam::..."}, + "TOWER_GITHUB_USER": {"value": "username"}, + "TOWER_GITHUB_TOKEN": {"value": "ghp_..."}, + "TOWER_DOCKER_USER": {"value": "username"}, + "TOWER_DOCKER_TOKEN": {"value": "token..."}, + "TOWER_CODECOMMIT_USER": {"value": "username"}, + "TOWER_CODECOMMIT_PASSWORD": {"value": "password..."}, + "TOWER_CODECOMMIT_REGION": {"value": "us-west-2"} +} +``` + +### Loading Scripts +```bash +# Auto-detect AWS profile from terraform.tfvars +source helpers/load_seqerakit_secrets.sh + +# Specify bootstrap path +source helpers/load_seqerakit_secrets.sh /myapp/seqerakit/config + +# Specify both path and profile +source helpers/load_seqerakit_secrets.sh /myapp/seqerakit/config playground +``` + +## Generated Files + +| File | Description | +|------|-------------| +| `setup.yml` | Main Seqerakit configuration | +| `aws_batch_manual.yml` | AWS Batch compute environment | +| `codecommit_set_workspace_id.sh` | CodeCommit workspace setup | +| `load_seqerakit_secrets.sh` | Secrets loading script | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| `aws_account` | AWS account ID | `string` | n/a | yes | +| `aws_region` | AWS region | `string` | n/a | yes | +| `aws_profile` | AWS profile to use | `string` | n/a | yes | +| `default_tags` | Default tags to apply to all resources | `map(string)` | n/a | yes | +| `app_name` | Application name | `string` | n/a | yes | +| `flag_run_seqerakit` | Whether to run Seqerakit setup | `bool` | n/a | yes | +| `seqerakit_org_name` | Seqerakit organization name | `string` | n/a | yes | +| `seqerakit_org_fullname` | Seqerakit organization full name | `string` | n/a | yes | +| `seqerakit_org_url` | Seqerakit organization URL | `string` | n/a | yes | +| `seqerakit_team_name` | Seqerakit team name | `string` | n/a | yes | +| `seqerakit_team_members` | Seqerakit team members (comma-separated) | `string` | n/a | yes | +| `seqerakit_workspace_name` | Seqerakit workspace name | `string` | n/a | yes | +| `seqerakit_workspace_fullname` | Seqerakit workspace full name | `string` | n/a | yes | +| `seqerakit_compute_env_name` | Seqerakit compute environment name | `string` | n/a | yes | +| `seqerakit_compute_env_region` | Seqerakit compute environment region | `string` | n/a | yes | +| `seqerakit_root_bucket` | Seqerakit root S3 bucket | `string` | n/a | yes | +| `seqerakit_workdir` | Seqerakit working directory | `string` | n/a | yes | +| `seqerakit_outdir` | Seqerakit output directory | `string` | n/a | yes | +| `seqerakit_aws_use_fusion_v2` | Whether to use Fusion v2 for Seqerakit | `bool` | n/a | yes | +| `seqerakit_aws_use_forge` | Whether to use Forge for Seqerakit | `bool` | n/a | yes | +| `seqerakit_aws_use_batch` | Whether to use AWS Batch for Seqerakit | `bool` | n/a | yes | +| `seqerakit_aws_fusion_instances` | Fusion instance types for Seqerakit | `string` | n/a | yes | +| `seqerakit_aws_normal_instances` | Normal instance types for Seqerakit | `string` | n/a | yes | +| `seqerakit_aws_manual_head_queue` | Manual head queue for Seqerakit | `string` | n/a | yes | +| `seqerakit_aws_manual_compute_queue` | Manual compute queue for Seqerakit | `string` | n/a | yes | +| `seqerakit_flag_credential_create_aws` | Whether to create AWS credentials for Seqerakit | `bool` | n/a | yes | +| `seqerakit_flag_credential_create_github` | Whether to create GitHub credentials for Seqerakit | `bool` | n/a | yes | +| `seqerakit_flag_credential_create_docker` | Whether to create Docker credentials for Seqerakit | `bool` | n/a | yes | +| `seqerakit_flag_credential_create_codecommit` | Whether to create CodeCommit credentials for Seqerakit | `bool` | n/a | yes | +| `seqerakit_flag_credential_use_aws_role` | Whether to use AWS role for Seqerakit credentials | `bool` | n/a | yes | +| `vpc_id` | VPC ID for compute environment | `string` | `""` | no | +| `subnet_ids` | Subnet IDs for compute environment | `list(string)` | `[]` | no | +| `security_group_ids` | Security group IDs for compute environment | `list(string)` | `[]` | no | +| `ec2_key_pair_name` | EC2 Key pair name for compute environment | `string` | `""` | no | +| `secrets_bootstrap_seqerakit` | SSM SecureString parameter name for Seqerakit secrets | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| `seqerakit_yml` | Generated seqerakit setup YAML content | +| `aws_batch_manual` | Generated AWS Batch manual compute environment YAML | +| `codecommit_script` | Generated CodeCommit workspace ID script | +| `load_secrets_script` | Script to load Seqerakit secrets from SSM into environment variables | +| `generated_files` | Map of generated file paths | +| `seqerakit_secrets` | Seqerakit secrets from SSM (sensitive) | + +## Troubleshooting + +### Missing Environment Variables +```bash +# Check if secrets are loaded +echo $TOWER_AWS_USER +echo $TOWER_GITHUB_TOKEN + +# Verify SSM parameter exists +aws ssm get-parameter --name "/myapp/seqerakit/secrets" --with-decryption +``` + +### Empty Values +Empty values are normal if certain credentials aren't configured in your bootstrap parameter. + +## Requirements + +- Terraform >= 1.0 +- AWS Provider >= 5.0 +- Local Provider >= 2.0 + +## Version History + +- **v1.0.0** - Initial release with consolidated module structure and environment variable secrets \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/compute-envs/aws_batch_manual.yml.tpl b/modules/seqerakit/v1.0.0/compute-envs/aws_batch_manual.yml.tpl new file mode 100644 index 00000000..373c6b37 --- /dev/null +++ b/modules/seqerakit/v1.0.0/compute-envs/aws_batch_manual.yml.tpl @@ -0,0 +1,14 @@ +compute-envs: + - type: aws-batch + config-mode: manual + name: "${seqerakit_compute_env_name}" + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + credentials: 'aws_credentials' + work-dir: ${seqerakit_workdir} + region: ${aws_region} + head-queue: ${seqerakit_aws_manual_head_queue} + compute-queue: ${seqerakit_aws_manual_compute_queue} + fusion-v2: ${use_fusion_v2} + wave: ${use_wave} + wait: 'AVAILABLE' + overwrite: True \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/credentials/codecommit.json b/modules/seqerakit/v1.0.0/credentials/codecommit.json new file mode 100644 index 00000000..7c310427 --- /dev/null +++ b/modules/seqerakit/v1.0.0/credentials/codecommit.json @@ -0,0 +1,13 @@ +{ + "credentials": { + "name": "myCodeCommit", + "description": "myCodeCommit", + "provider": "codecommit", + "baseUrl": "WILL_BE_SUBSTITUTED_BY_ANSIBLE", + "category": "dunno", + "keys": { + "username": "WILL_BE_SUBSTITUTED_BY_ANSIBLE", + "password": "WILL_BE_SUBSTITUTED_BY_ANSIBLE" + } + } +} \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/helpers/codecommit_set_workspace_id.sh.tpl b/modules/seqerakit/v1.0.0/helpers/codecommit_set_workspace_id.sh.tpl new file mode 100644 index 00000000..37f247b8 --- /dev/null +++ b/modules/seqerakit/v1.0.0/helpers/codecommit_set_workspace_id.sh.tpl @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +# Add TF vars here and export to keep segregated. +export CC_ORG_NAME=${seqerakit_org_name} +export CC_WORKSPACE_NAME=${seqerakit_workspace_name} \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/helpers/load_seqerakit_secrets.sh b/modules/seqerakit/v1.0.0/helpers/load_seqerakit_secrets.sh new file mode 100755 index 00000000..83a9e054 --- /dev/null +++ b/modules/seqerakit/v1.0.0/helpers/load_seqerakit_secrets.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# ------------------------------------------------------------------------------------ +# Load Seqerakit Secrets from SSM into Environment Variables +# ------------------------------------------------------------------------------------ +# Usage: source load_seqerakit_secrets.sh [bootstrap_ssm_path] [aws_profile] [aws_region] +# Example: source load_seqerakit_secrets.sh /scidev/seqerakit/config playground us-east-1 +# ------------------------------------------------------------------------------------ + +# set -e + +# Get parameters +BOOTSTRAP_SSM_PATH=${1:-"/scidev/seqerakit/config"} +AWS_PROFILE_PARAM=${2:-"$AWS_PROFILE"} +AWS_REGION_PARAM=${3:-"$AWS_DEFAULT_REGION"} + +echo "Loading secrets from: $BOOTSTRAP_SSM_PATH" +if [ -n "$AWS_PROFILE_PARAM" ]; then + echo "Using AWS profile: $AWS_PROFILE_PARAM" + export AWS_PROFILE="$AWS_PROFILE_PARAM" +fi +if [ -n "$AWS_REGION_PARAM" ]; then + echo "Using AWS region: $AWS_REGION_PARAM" + export AWS_DEFAULT_REGION="$AWS_REGION_PARAM" + export AWS_REGION="$AWS_REGION_PARAM" +fi + +# Check dependencies +if ! command -v jq &> /dev/null; then + echo "❌ Error: jq is required. Install with: brew install jq" + return 1 +fi + +if ! command -v aws &> /dev/null; then + echo "❌ Error: AWS CLI is required" + return 1 +fi + +# Test AWS CLI +if ! aws sts get-caller-identity &>/dev/null; then + echo "❌ Error: AWS CLI not configured. Run 'aws configure' or 'aws sso login'" + return 1 +fi + +# Get the bootstrap parameter +BOOTSTRAP_JSON=$(aws ssm get-parameters \ + --name "$BOOTSTRAP_SSM_PATH" \ + --with-decryption \ + --query "Parameters[*].{Value:Value}" \ + --output text) + +if [ -z "$BOOTSTRAP_JSON" ] || [ "$BOOTSTRAP_JSON" = "None" ]; then + echo "❌ Error: Parameter not found: $BOOTSTRAP_SSM_PATH" + return 1 +fi + +# Validate JSON +if ! echo "$BOOTSTRAP_JSON" | jq empty 2>/dev/null; then + echo "❌ Error: Invalid JSON in parameter" + return 1 +fi + +# Extract and set environment variables +export TOWER_AWS_USER=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_AWS_USER"]["value"] // empty') +export TOWER_AWS_PASSWORD=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_AWS_PASSWORD"]["value"] // empty') +export TOWER_AWS_ROLE=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_AWS_ROLE"]["value"] // empty') +export TOWER_GITHUB_USER=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_GITHUB_USER"]["value"] // empty') +export TOWER_GITHUB_TOKEN=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_GITHUB_TOKEN"]["value"] // empty') +export TOWER_DOCKER_USER=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_DOCKER_USER"]["value"] // empty') +export TOWER_DOCKER_TOKEN=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_DOCKER_TOKEN"]["value"] // empty') +export TOWER_CODECOMMIT_USER=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_CODECOMMIT_USER"]["value"] // empty') +export TOWER_CODECOMMIT_PASSWORD=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_CODECOMMIT_PASSWORD"]["value"] // empty') +export TOWER_CODECOMMIT_REGION=$(echo "$BOOTSTRAP_JSON" | jq -r '.["TOWER_CODECOMMIT_REGION"]["value"] // empty') + +echo "✓ Secrets loaded successfully" diff --git a/modules/seqerakit/v1.0.0/helpers/split_for_codecommit.py b/modules/seqerakit/v1.0.0/helpers/split_for_codecommit.py new file mode 100644 index 00000000..6237388f --- /dev/null +++ b/modules/seqerakit/v1.0.0/helpers/split_for_codecommit.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import yaml + +yaml.sort_base_mapping_type_on_output = False + +original_setup = open('../setup.yml', 'r') + +cc_first_keys = ["organizations","workspaces","teams","participants","credentials"] +cc_second_keys = ["pipelines","launch","compute-envs"] + +cc_first_dict = {} +cc_second_dict = {} + +original_yaml = yaml.safe_load(original_setup) #, Loader=Loader) + + +for key in cc_first_keys: + cc_first_dict[key] = original_yaml[key] + +for key in cc_second_keys: + cc_second_dict[key] = original_yaml[key] + + +with open('../cc_first.yaml', 'w') as yamlfile: + yaml.safe_dump(cc_first_dict, yamlfile, sort_keys=False) + +with open('../cc_second.yaml', 'w') as yamlfile: + yaml.safe_dump(cc_second_dict, yamlfile, sort_keys=False) + diff --git a/modules/seqerakit/v1.0.0/main.tf b/modules/seqerakit/v1.0.0/main.tf new file mode 100644 index 00000000..7da9819e --- /dev/null +++ b/modules/seqerakit/v1.0.0/main.tf @@ -0,0 +1,143 @@ +## ------------------------------------------------------------------------------------ +## Seqerakit Module - Main Configuration +## ------------------------------------------------------------------------------------ + +## ------------------------------------------------------------------------------------ +## Data Sources +## ------------------------------------------------------------------------------------ +data "aws_ssm_parameter" "seqerakit_secrets" { + count = var.secrets_bootstrap_seqerakit != "" ? 1 : 0 + name = var.secrets_bootstrap_seqerakit +} + +## ------------------------------------------------------------------------------------ +## Locals +## ------------------------------------------------------------------------------------ +locals { + # Use SSM secrets if available, otherwise use empty values + seqerakit_secrets = var.secrets_bootstrap_seqerakit != "" ? jsondecode(nonsensitive(data.aws_ssm_parameter.seqerakit_secrets[0].value)) : { + "TOWER_AWS_USER" = { "value" = "CHANGE_ME" } + "TOWER_AWS_PASSWORD" = { "value" = "CHANGE_ME" } + "TOWER_AWS_ROLE" = { "value" = "CHANGE_ME" } + "TOWER_GITHUB_USER" = { "value" = "CHANGE_ME" } + "TOWER_GITHUB_TOKEN" = { "value" = "CHANGE_ME" } + "TOWER_DOCKER_USER" = { "value" = "CHANGE_ME" } + "TOWER_DOCKER_TOKEN" = { "value" = "CHANGE_ME" } + "TOWER_CODECOMMIT_USER" = { "value" = "CHANGE_ME" } + "TOWER_CODECOMMIT_PASSWORD" = { "value" = "CHANGE_ME" } + "TOWER_CODECOMMIT_REGION" = { "value" = "CHANGE_ME" } + } + + # Seqerakit - Main Configuration Template + seqerakit_yml = templatefile("${path.module}/setup.yml.tpl", + { + seqerakit_org_name = var.seqerakit_org_name, + seqerakit_org_fullname = var.seqerakit_org_fullname, + seqerakit_org_url = var.seqerakit_org_url, + + seqerakit_team_name = var.seqerakit_team_name, + seqerakit_team_members = replace(var.seqerakit_team_members, "/\\s+/", ""), + + seqerakit_workspace_name = var.seqerakit_workspace_name, + seqerakit_workspace_fullname = var.seqerakit_workspace_fullname, + + seqerakit_workdir = var.seqerakit_workdir, + seqerakit_outdir = var.seqerakit_outdir, + seqerakit_compute_env_name = var.seqerakit_compute_env_name, + + seqerakit_flag_credential_create_aws = var.seqerakit_flag_credential_create_aws, + seqerakit_flag_credential_create_github = var.seqerakit_flag_credential_create_github, + seqerakit_flag_credential_create_docker = var.seqerakit_flag_credential_create_docker, + seqerakit_flag_credential_create_codecommit = var.seqerakit_flag_credential_create_codecommit, + + seqerakit_flag_credential_use_aws_role = var.seqerakit_flag_credential_use_aws_role + + # Environment variable names for secrets (values sourced from SSM) + aws_access_key_env_var = "$TOWER_AWS_USER" + aws_secret_key_env_var = "$TOWER_AWS_PASSWORD" + aws_role_arn_env_var = "$TOWER_AWS_ROLE" + github_username_env_var = "$TOWER_GITHUB_USER" + github_token_env_var = "$TOWER_GITHUB_TOKEN" + docker_username_env_var = "$TOWER_DOCKER_USER" + docker_password_env_var = "$TOWER_DOCKER_TOKEN" + + # Compute environment variables + seqerakit_aws_use_forge = var.seqerakit_aws_use_forge, + aws_region = var.seqerakit_compute_env_region, + vpc_id = var.vpc_id, + subnets = length(var.subnet_ids) > 0 ? join(",", var.subnet_ids) : "", + securityGroups = length(var.security_group_ids) > 0 ? join(",", var.security_group_ids) : "", + ec2KeyPair = var.ec2_key_pair_name, + + use_fusion_v2 = var.seqerakit_aws_use_fusion_v2 == true ? "True" : "False", + use_wave = var.seqerakit_aws_use_fusion_v2 == true ? "True" : "False", + use_fast_storage = var.seqerakit_aws_use_fusion_v2 == true ? "True" : "False", + + instance_types = ( + var.seqerakit_aws_use_fusion_v2 == true ? + replace(var.seqerakit_aws_fusion_instances, "/\\s+/", "") : + replace(var.seqerakit_aws_normal_instances, "/\\s+/", "") + ) + } + ) + + # Seqerakit - AWS Batch Manual Compute Environment Template + aws_batch_manual = templatefile("${path.module}/compute-envs/aws_batch_manual.yml.tpl", + { + aws_region = var.seqerakit_compute_env_region, + + seqerakit_org_name = var.seqerakit_org_name, + seqerakit_workspace_name = var.seqerakit_workspace_name, + seqerakit_workdir = var.seqerakit_workdir, + seqerakit_compute_env_name = var.seqerakit_compute_env_name, + + seqerakit_aws_manual_head_queue = var.seqerakit_aws_manual_head_queue, + seqerakit_aws_manual_compute_queue = var.seqerakit_aws_manual_compute_queue, + + use_fusion_v2 = var.seqerakit_aws_use_fusion_v2 == true ? "True" : "False", + use_wave = var.seqerakit_aws_use_fusion_v2 == true ? "True" : "False", + use_fast_storage = var.seqerakit_aws_use_fusion_v2 == true ? "True" : "False" + } + ) + + # Seqerakit - CodeCommit Workspace ID Script Template + codecommit_seqerakit = templatefile("${path.module}/helpers/codecommit_set_workspace_id.sh.tpl", + { + seqerakit_org_name = var.seqerakit_org_name, + seqerakit_workspace_name = var.seqerakit_workspace_name + } + ) +} + +## ------------------------------------------------------------------------------------ +## Resources +## ------------------------------------------------------------------------------------ +resource "local_file" "seqerakit_yml" { + content = local.seqerakit_yml + filename = "${path.module}/generated/setup.yml" +} + +resource "local_file" "aws_batch_manual_yml" { + content = local.aws_batch_manual + filename = "${path.module}/generated/aws_batch_manual.yml" +} + +resource "local_file" "codecommit_script" { + content = local.codecommit_seqerakit + filename = "${path.module}/generated/codecommit_set_workspace_id.sh" +} + +# Copy the secrets loading script +resource "local_file" "load_secrets_script" { + source = "${path.module}/helpers/load_seqerakit_secrets.sh" + filename = "${path.module}/generated/load_seqerakit_secrets.sh" +} + +# Make scripts executable +resource "null_resource" "make_scripts_executable" { + depends_on = [local_file.codecommit_script, local_file.load_secrets_script] + + provisioner "local-exec" { + command = "chmod +x ${path.module}/generated/codecommit_set_workspace_id.sh ${path.module}/generated/load_seqerakit_secrets.sh" + } +} diff --git a/modules/seqerakit/v1.0.0/outputs.tf b/modules/seqerakit/v1.0.0/outputs.tf new file mode 100644 index 00000000..c9acd820 --- /dev/null +++ b/modules/seqerakit/v1.0.0/outputs.tf @@ -0,0 +1,44 @@ +## ------------------------------------------------------------------------------------ +## Seqerakit Configuration Outputs +## ------------------------------------------------------------------------------------ +output "seqerakit_yml" { + description = "Generated seqerakit setup YAML content" + value = local.seqerakit_yml +} + +output "aws_batch_manual" { + description = "Generated AWS Batch manual compute environment YAML" + value = local.aws_batch_manual +} + +output "codecommit_script" { + description = "Generated CodeCommit workspace ID script" + value = local.codecommit_seqerakit +} + +output "load_secrets_script" { + description = "Script to load Seqerakit secrets from SSM into environment variables" + value = "${path.module}/generated/load_seqerakit_secrets.sh" +} + +## ------------------------------------------------------------------------------------ +## Generated Files Outputs +## ------------------------------------------------------------------------------------ +output "generated_files" { + description = "Map of generated file paths" + value = { + setup_yml = local_file.seqerakit_yml.filename + aws_batch_manual_yml = local_file.aws_batch_manual_yml.filename + codecommit_script = local_file.codecommit_script.filename + load_secrets_script = local_file.load_secrets_script.filename + } +} + +## ------------------------------------------------------------------------------------ +## Secrets Outputs +## ------------------------------------------------------------------------------------ +output "seqerakit_secrets" { + description = "Seqerakit secrets from SSM (sensitive)" + value = local.seqerakit_secrets + sensitive = true +} diff --git a/modules/seqerakit/v1.0.0/pipelines/nextflow.config b/modules/seqerakit/v1.0.0/pipelines/nextflow.config new file mode 100755 index 00000000..612d322a --- /dev/null +++ b/modules/seqerakit/v1.0.0/pipelines/nextflow.config @@ -0,0 +1 @@ +process.maxRetries = 1 \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/pipelines/post_run.txt b/modules/seqerakit/v1.0.0/pipelines/post_run.txt new file mode 100644 index 00000000..b1d97ec0 --- /dev/null +++ b/modules/seqerakit/v1.0.0/pipelines/post_run.txt @@ -0,0 +1 @@ +echo "hello from post run!" \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/pipelines/pre_run.txt b/modules/seqerakit/v1.0.0/pipelines/pre_run.txt new file mode 100755 index 00000000..e0747cf6 --- /dev/null +++ b/modules/seqerakit/v1.0.0/pipelines/pre_run.txt @@ -0,0 +1 @@ +export NXF_VER=24.04.2 \ No newline at end of file diff --git a/modules/seqerakit/v1.0.0/setup.yml.tpl b/modules/seqerakit/v1.0.0/setup.yml.tpl new file mode 100644 index 00000000..17ab1eae --- /dev/null +++ b/modules/seqerakit/v1.0.0/setup.yml.tpl @@ -0,0 +1,156 @@ +organizations: + - name: "${seqerakit_org_name}" + full-name: "${seqerakit_org_fullname}" + description: "Description for ${seqerakit_org_name}" + location: 'Global' + website: "${seqerakit_org_url}" + overwrite: False + +workspaces: + - name: "${seqerakit_workspace_name}" + full-name: "${seqerakit_workspace_fullname}" + organization: "${seqerakit_org_name}" + description: 'Workspace for ${seqerakit_workspace_name}' + visibility: 'PRIVATE' + overwrite: False + +teams: + - name: "${seqerakit_team_name}" + organization: "${seqerakit_org_name}" + description: "Description for ${seqerakit_team_name}" + members: [ ${seqerakit_team_members } ] + overwrite: False + +participants: + - name: "${seqerakit_team_name}" + type: 'TEAM' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + role: 'ADMIN' + overwrite: False + +credentials: +%{~ if seqerakit_flag_credential_create_aws == true } + - type: 'aws' + name: 'aws_credentials' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + access-key: ${aws_access_key_env_var} + secret-key: ${aws_secret_key_env_var} + %{~ if seqerakit_flag_credential_use_aws_role == true ~} + assume-role-arn: ${aws_role_arn_env_var} + %{~ endif ~} + overwrite: False +%{ endif ~} + +%{~ if seqerakit_flag_credential_create_docker == true } + - type: 'container-reg' + name: 'dockerhub_credentials' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + username: ${docker_username_env_var} + password: ${docker_password_env_var} + registry: 'docker.io' + overwrite: False +%{ endif ~} + +%{~ if seqerakit_flag_credential_create_github == true } + - type: 'github' + name: 'github_credentials' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + username: ${github_username_env_var} + password: ${github_token_env_var} + overwrite: False +%{ endif ~} + +compute-envs: +%{~ if seqerakit_aws_use_forge == true } + - type: aws-batch + config-mode: forge + name: "${seqerakit_compute_env_name}" + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + credentials: 'aws_credentials' + region: "${aws_region}" + work-dir: "${seqerakit_workdir}" + provisioning-model: SPOT + fusion-v2: ${use_fusion_v2} + wave: ${use_wave} + fargate: False + fast-storage: ${use_fast_storage} + instance-types: "${instance_types}" + no-ebs-auto-scale: True + max-cpus: 500 + wait: AVAILABLE +%{~ if vpc_id != "" ~} + vpc-id: "${vpc_id}" +%{~ endif ~} +%{~ if subnets != "" ~} + subnets: "${subnets}" +%{~ endif ~} +%{~ if securityGroups != "" ~} + security-groups: "${securityGroups}" +%{~ endif ~} +%{~ if ec2KeyPair != "" ~} + key-pair: "${ec2KeyPair}" +%{~ endif ~} +%{ endif ~} + +pipelines: + - name: 'hello-world' + url: 'https://github.com/nextflow-io/hello' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + description: 'Tiny hello world pipeline composed of four tasks.' + compute-env: "${seqerakit_compute_env_name}" + work-dir: "${seqerakit_workdir}" + revision: 'master' + config: '../pipelines/nextflow.config' + pre-run: '../pipelines/pre_run.txt' + post-run: '../pipelines/post_run.txt' + overwrite: False + + - name: 'nf-core-rnaseq-test' + url: 'https://github.com/nf-core/rnaseq' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + description: 'RNA sequencing analysis pipeline using STAR, RSEM, HISAT2 or Salmon with gene/isoform counts and extensive quality control.' + compute-env: "${seqerakit_compute_env_name}" + work-dir: "${seqerakit_workdir}" + profile: 'test' + labels: 'profile_test' + revision: '3.12.0' + params: + outdir: "${seqerakit_outdir}" + config: '../pipelines/nextflow.config' + pre-run: '../pipelines/pre_run.txt' + overwrite: False + + - name: 'nf-core-rnaseq-test-full' + url: 'https://github.com/nf-core/rnaseq' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + description: 'RNA sequencing analysis pipeline using STAR, RSEM, HISAT2 or Salmon with gene/isoform counts and extensive quality control.' + compute-env: "${seqerakit_compute_env_name}" + work-dir: "${seqerakit_workdir}" + profile: 'test_full' + labels: 'profile_test_full' + revision: '3.12.0' + params: + outdir: "${seqerakit_outdir}" + config: '../pipelines/nextflow.config' + pre-run: '../pipelines/pre_run.txt' + overwrite: False + +%{~ if seqerakit_flag_credential_create_codecommit == true } + # CodeCommitTest + - name: 'cc-test' + url: 'https://git-codecommit.us-east-1.amazonaws.com/v1/repos/grahamhello' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + description: 'CodeCommit testing pipeline' + compute-env: "${seqerakit_compute_env_name}" + work-dir: "${seqerakit_workdir}" + revision: 'main' + overwrite: False +%{ endif ~} + + + +launch: + - name: 'hello-world-launchpad' + workspace: "${seqerakit_org_name}/${seqerakit_workspace_name}" + pipeline: 'hello-world' + diff --git a/modules/seqerakit/v1.0.0/variables.tf b/modules/seqerakit/v1.0.0/variables.tf new file mode 100644 index 00000000..f821f041 --- /dev/null +++ b/modules/seqerakit/v1.0.0/variables.tf @@ -0,0 +1,202 @@ +## ------------------------------------------------------------------------------------ +## AWS Configuration +## ------------------------------------------------------------------------------------ +variable "aws_account" { + description = "AWS account ID" + type = string +} + +variable "aws_region" { + description = "AWS region" + type = string +} + +variable "aws_profile" { + description = "AWS profile to use" + type = string +} + +## ------------------------------------------------------------------------------------ +## Tags +## ------------------------------------------------------------------------------------ +variable "default_tags" { + description = "Default tags to apply to all resources" + type = map(string) +} + +## ------------------------------------------------------------------------------------ +## App Configuration +## ------------------------------------------------------------------------------------ +variable "app_name" { + description = "Application name" + type = string +} + +## ------------------------------------------------------------------------------------ +## Seqerakit Configuration +## ------------------------------------------------------------------------------------ +variable "flag_run_seqerakit" { + description = "Whether to run Seqerakit setup" + type = bool +} + +variable "seqerakit_org_name" { + description = "Seqerakit organization name" + type = string +} + +variable "seqerakit_org_fullname" { + description = "Seqerakit organization full name" + type = string +} + +variable "seqerakit_org_url" { + description = "Seqerakit organization URL" + type = string +} + +variable "seqerakit_team_name" { + description = "Seqerakit team name" + type = string +} + +variable "seqerakit_team_members" { + description = "Seqerakit team members (comma-separated)" + type = string +} + +variable "seqerakit_workspace_name" { + description = "Seqerakit workspace name" + type = string +} + +variable "seqerakit_workspace_fullname" { + description = "Seqerakit workspace full name" + type = string +} + +variable "seqerakit_compute_env_name" { + description = "Seqerakit compute environment name" + type = string +} + +variable "seqerakit_compute_env_region" { + description = "Seqerakit compute environment region" + type = string +} + +variable "seqerakit_root_bucket" { + description = "Seqerakit root S3 bucket" + type = string +} + +variable "seqerakit_workdir" { + description = "Seqerakit working directory" + type = string +} + +variable "seqerakit_outdir" { + description = "Seqerakit output directory" + type = string +} + +## ------------------------------------------------------------------------------------ +## Seqerakit AWS Configuration +## ------------------------------------------------------------------------------------ +variable "seqerakit_aws_use_fusion_v2" { + description = "Whether to use Fusion v2 for Seqerakit" + type = bool +} + +variable "seqerakit_aws_use_forge" { + description = "Whether to use Forge for Seqerakit" + type = bool +} + +variable "seqerakit_aws_use_batch" { + description = "Whether to use AWS Batch for Seqerakit" + type = bool +} + +variable "seqerakit_aws_fusion_instances" { + description = "Fusion instance types for Seqerakit" + type = string +} + +variable "seqerakit_aws_normal_instances" { + description = "Normal instance types for Seqerakit" + type = string +} + +variable "seqerakit_aws_manual_head_queue" { + description = "Manual head queue for Seqerakit" + type = string +} + +variable "seqerakit_aws_manual_compute_queue" { + description = "Manual compute queue for Seqerakit" + type = string +} + +## ------------------------------------------------------------------------------------ +## Seqerakit Credentials Configuration +## ------------------------------------------------------------------------------------ +variable "seqerakit_flag_credential_create_aws" { + description = "Whether to create AWS credentials for Seqerakit" + type = bool +} + +variable "seqerakit_flag_credential_create_github" { + description = "Whether to create GitHub credentials for Seqerakit" + type = bool +} + +variable "seqerakit_flag_credential_create_docker" { + description = "Whether to create Docker credentials for Seqerakit" + type = bool +} + +variable "seqerakit_flag_credential_create_codecommit" { + description = "Whether to create CodeCommit credentials for Seqerakit" + type = bool +} + +variable "seqerakit_flag_credential_use_aws_role" { + description = "Whether to use AWS role for Seqerakit credentials" + type = bool +} + +## ------------------------------------------------------------------------------------ +## Infrastructure Dependencies +## ------------------------------------------------------------------------------------ +variable "vpc_id" { + description = "VPC ID for compute environment" + type = string + default = "" +} + +variable "subnet_ids" { + description = "Subnet IDs for compute environment" + type = list(string) + default = [] +} + +variable "security_group_ids" { + description = "Security group IDs for compute environment" + type = list(string) + default = [] +} + +variable "ec2_key_pair_name" { + description = "EC2 Key pair name for compute environment" + type = string + default = "" +} + +## ------------------------------------------------------------------------------------ +## Secrets Configuration +## ------------------------------------------------------------------------------------ +variable "secrets_bootstrap_seqerakit" { + description = "SSM SecureString parameter name for Seqerakit secrets" + type = string +}