diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..d212b2d --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +echo "Running ruff..." +uv run ruff check . + +echo "Running vulture..." +uv run vulture . --min-confidence 70 --exclude .venv diff --git a/.github/workflows/code-analysis.yml b/.github/workflows/code-analysis.yml deleted file mode 100644 index dfab911..0000000 --- a/.github/workflows/code-analysis.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Code Analysis - -permissions: - contents: read - -on: - push: - branches: - - '**' - -jobs: - CodeAnalysis: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Display Python version - run: | - uv run python --version - - - name: Install dependencies - run: | - uv sync --frozen - - - name: Check for outdated dependencies - if: env.skip-job != 'true' - run: | - uv tree --outdated - - - name: Run tests with coverage in latest stable 3.x - run: | - uv run pytest --cov=app --cov-report=term-missing - continue-on-error: false - - - name: Run Bandit for security analysis - run: | - uv run bandit -r app - continue-on-error: false - - - name: Run Ruff for linting - run: | - uv run ruff check . - continue-on-error: false - - - name: Run Vulture for dead code analysis - run: | - uv run vulture . --min-confidence 70 --exclude .venv - continue-on-error: false - diff --git a/Dockerfile b/Dockerfile index 2a72d6e..270ef0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,4 +61,7 @@ COPY --chown=app:app . . # Switch to the unprivileged user before starting the process USER app +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD ["uv", "run", "python", "scripts/healthcheck.py"] + CMD ["uv", "run", "python", "-m", "gunicorn", "-c", "gunicorn_config.py"] \ No newline at end of file diff --git a/README.md b/README.md index 51cf3e1..2710092 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # zephir_api2 - -![CI](https://github.com/cdlib/zephir-api2/actions/workflows/ci.yml/badge.svg) The Zephir Item API provides item-level MARC metadata records for all items submitted to HathiTrust. + See [API.md](API.md) for full endpoint documentation. +## Workflow + +This repository uses [GitHub flow](https://docs.github.com/en/get-started/using-github/github-flow). The `main` branch is always assumed to be tested and ready for deployment to production. Ensure all changes are adequately tested _before_ merging into `main`. + ## Getting Started ### Prerequisites @@ -17,23 +20,33 @@ brew install uv ### Local Development -**Install dependencies:** +1. Install dependencies -```sh -uv sync -``` + ```sh + uv sync + ``` -**Set required environment variables** (see [Database configuration](#database-configuration) below), then **run the app:** +2. Install git hooks -Copy `env.template` to `.env` and update the `DATABASE_URI` variable. + ```sh + git config core.hooksPath .githooks + ``` -```sh -uv run python -m gunicorn -c gunicorn_config.py -``` + This enables the pre-commit hook that runs ruff and vulture before each commit. + +3. Set required environment variables + + Copy `env.template` to `.env` and update the `DATABASE_URI` variable. See [Database configuration](#database-configuration) below. -The API will be available at `http://localhost:8000/api/`. +4. Run the app -**Run tests:** + ```sh + uv run python -m gunicorn -c gunicorn_config.py + ``` + + The API will be available at `http://localhost:8000/api/`. + +### Running tests ```sh uv run pytest tests @@ -41,7 +54,7 @@ uv run pytest tests Tests use a bundled SQLite fixture (`tests/test_zephir_api/`) and do not require a live database connection. -**Run linting and analysis tools:** +### Linting, etc. ```sh uv run ruff check . @@ -94,7 +107,19 @@ The app resolves the database connection from environment variables in priority | `DATABASE_CREDENTIALS_SECRET_NAME` + AWS vars | Pull credentials from AWS Secrets Manager | | `DB_USERNAME` + `DB_PASSWORD` + `DB_HOST` + `DB_DATABASE` | Individual credential env vars | ---- +## CI (Continuous Integration) + +CI runs on AWS CodeBuild in the `cdl-d2d-dev` account. Every pull request triggers a build that runs tests, linting, and security checks, and reports a check status with a link to the build log back to GitHub. + +The job configuration lives in [`buildspec.yml`](buildspec.yml). + +### First-time setup + +The CodeBuild project is managed by Sceptre. Note that the GitHub connection is created and activated manually before deploying. + +### Test results + +Build status is automatically reported to GitHub on every PR. Test results are published to the CodeBuild **Test reports** panel (JUnit XML). ## Deploying to AWS diff --git a/buildspec.yml b/buildspec.yml new file mode 100644 index 0000000..45e91dc --- /dev/null +++ b/buildspec.yml @@ -0,0 +1,25 @@ +version: 0.2 + +phases: + install: + commands: + - pip install uv + + pre_build: + commands: + - uv sync --frozen + + build: + commands: + - uv run python --version + - uv tree --outdated + - uv run pytest --cov=app --cov-report=term-missing --junitxml=pytest-report.xml + - uv run bandit -r app + - uv run ruff check . + - uv run vulture . --min-confidence 70 --exclude .venv + +reports: + pytest-reports: + files: + - pytest-report.xml + file-format: JUNITXML diff --git a/deployment/config/config.yaml b/deployment/config/config.yaml index 8d02154..eb48473 100644 --- a/deployment/config/config.yaml +++ b/deployment/config/config.yaml @@ -1,2 +1,18 @@ project_code: d2d-zephir-api -region: us-west-2 \ No newline at end of file +region: us-west-2 + +sceptre_user_data_inheritance: merge +stack_tags_inheritance: merge + +sceptre_user_data: + program: d2d + service: zephir + subservice: api + repository: https://github.com/cdlib/zephir-api2.git + port: "8000" + +stack_tags: + program: !stack_attr sceptre_user_data.program + service: !stack_attr sceptre_user_data.service + subservice: !stack_attr sceptre_user_data.subservice + repository: !stack_attr sceptre_user_data.repository diff --git a/deployment/config/dev/autoscaling.yaml b/deployment/config/dev/autoscaling.yaml index 070c406..77322a4 100644 --- a/deployment/config/dev/autoscaling.yaml +++ b/deployment/config/dev/autoscaling.yaml @@ -3,14 +3,14 @@ template: type: file dependencies: - - dev/ecs.yaml + - "{{ sceptre_user_data.environment }}/ecs.yaml" parameters: - # ECSAutoScalingPlanName: "{{ base_name }}-{{ environment }}-ecs-auto-scaling-plan" - ECSClusterName: !stack_output dev/ecs.yaml::ECSClusterName - ECSServiceName: !stack_output dev/ecs.yaml::ECSServiceName + # ECSAutoScalingPlanName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-auto-scaling-plan" + ECSClusterName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSClusterName" + ECSServiceName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSServiceName" - ECSCloudFormationStackARN: !stack_output dev/ecs.yaml::ECSCloudFormationStackARN + ECSCloudFormationStackARN: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSCloudFormationStackARN" MinCapacity: "1" MaxCapacity: "3" \ No newline at end of file diff --git a/deployment/config/dev/codebuild.yaml b/deployment/config/dev/codebuild.yaml new file mode 100644 index 0000000..61c967e --- /dev/null +++ b/deployment/config/dev/codebuild.yaml @@ -0,0 +1,10 @@ +template: + path: codebuild.yaml.j2 + type: file + +# No dependencies on other stacks; CodeBuild is independent of ECS/network/etc. +# +# GitHubConnectionArn must be created and activated manually before deploying. +# See the "CI" section in README.md for instructions. +parameters: + GitHubConnectionArn: arn:aws:codeconnections:us-west-2:445017934155:connection/6af1f14e-4157-4e78-a1bd-73f160a9c4c5 diff --git a/deployment/config/dev/config.yaml b/deployment/config/dev/config.yaml index 10bb1d3..1c10c1f 100644 --- a/deployment/config/dev/config.yaml +++ b/deployment/config/dev/config.yaml @@ -1,9 +1,11 @@ -# profile: cdl-d2d-dev -base_name: {{ project_code }} -environment: dev +profile: cdl-d2d-dev + +sceptre_user_data: + environment: dev + resource_name_prefix: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-dev" + url_authority: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-dev.d2ddev.cdlib.net" stack_tags: - Program: "d2d" - Service: "zephir" - Subservice: "api" - Environment: "dev" + # Use !stack_attr instead of jinja syntax because "environment" is defined in same template. + # The resolver !stack_attr executes after jinja. + Environment: !stack_attr sceptre_user_data.environment diff --git a/deployment/config/dev/ecr.yaml b/deployment/config/dev/ecr.yaml index f81eed3..bda5fd9 100644 --- a/deployment/config/dev/ecr.yaml +++ b/deployment/config/dev/ecr.yaml @@ -1,6 +1,6 @@ template: - path: ecr.yaml + path: ecr.yaml.j2 type: file -parameters: - RepositoryName: "{{ base_name }}-{{ environment }}" \ No newline at end of file +# Comment out to create once, ignore afterwards: +ignore: True diff --git a/deployment/config/dev/ecs.yaml b/deployment/config/dev/ecs.yaml index 6c4bdf6..fc7347a 100644 --- a/deployment/config/dev/ecs.yaml +++ b/deployment/config/dev/ecs.yaml @@ -1,43 +1,37 @@ template: - path: ecs.yaml + path: ecs.yaml.j2 type: file dependencies: - - dev/ecr.yaml - - dev/logging.yaml - - dev/network.yaml + - "{{ sceptre_user_data.environment }}/ecr.yaml" + - "{{ sceptre_user_data.environment }}/logging.yaml" + - "{{ sceptre_user_data.environment }}/network.yaml" parameters: - ECSServiceSecurityGroupName: "{{ base_name }}-{{ environment }}-ecs-service-security-group" - ECSClusterName: "{{ base_name }}-{{ environment }}-ecs-cluster" - ContainerName: "{{ base_name }}-{{ environment }}-container" - TaskDefinitionFamilyName: "{{ base_name }}-{{ environment }}-ecs-task-definition" - ECSTaskRoleName: "{{ base_name }}-{{ environment }}-ecs-task-role" - ECSTaskExecutionRoleName: "{{ base_name }}-{{ environment }}-ecs-task-execution-role" - ECSServiceName: "{{ base_name }}-{{ environment }}-ecs-service" + ContainerName: "{{ sceptre_user_data.resource_name_prefix }}-container" {% if var.image_tag | default(False) %} ImageTag: "{{ var.image_tag }}" {% else %} ImageTag: latest {% endif %} - + VPCID: !stack_output_external cdl-d2d-dev-vpc-stack::vpc SubnetIDs: - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2c - LoadBalancerSecurityGroupID: !stack_output dev/network.yaml::LoadBalancerSecurityGroupID - TargetGroupARN: !stack_output dev/network.yaml::TargetGroupARN + LoadBalancerSecurityGroupID: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerSecurityGroupID" + TargetGroupARN: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::TargetGroupARN" + + RepositoryURI: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryURI" + RepositoryARN: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryARN" + + LogGroupID: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupID" + LogGroupArn: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupArn" - RepositoryURI: !stack_output dev/ecr.yaml::RepositoryURI - RepositoryARN: !stack_output dev/ecr.yaml::RepositoryARN - - LogGroupID: !stack_output dev/logging.yaml::LogGroupID - LogGroupArn: !stack_output dev/logging.yaml::LogGroupArn + ApplicationPort: "{{ sceptre_user_data.port }}" - ApplicationPort: "8000" - FlaskEnvironment: development LogLevel: DEBUG diff --git a/deployment/config/dev/logging.yaml b/deployment/config/dev/logging.yaml index 856b31d..d488df2 100644 --- a/deployment/config/dev/logging.yaml +++ b/deployment/config/dev/logging.yaml @@ -1,6 +1,3 @@ template: - path: logging.yaml + path: logging.yaml.j2 type: file - -parameters: - LogGroupName: "{{ base_name }}-{{ environment }}-log-group" \ No newline at end of file diff --git a/deployment/config/dev/network.yaml b/deployment/config/dev/network.yaml index 65ea710..e24900d 100644 --- a/deployment/config/dev/network.yaml +++ b/deployment/config/dev/network.yaml @@ -2,23 +2,19 @@ template: path: network.yaml.j2 type: file -parameters: - LoadBalancerSecurityGroupName: "{{ base_name }}-{{ environment }}-load-balancer-security-group" - LoadBalancerName: "{{ base_name }}-{{ environment }}-load-balancer" - TargetGroupName: "{{ base_name }}-{{ environment }}-target-group" +sceptre_user_data: + load_balancer_security_group_sources: + - !ssm /d2d/dev/ucop-vpn-prefix-list-id + - !ssm /d2d/dev/cdl-eip-prefix-list-id +parameters: VPCID: !stack_output_external cdl-d2d-dev-vpc-stack::vpc SubnetIDs: - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2c LoadBalancerSecurityGroupSources: !stack_attr sceptre_user_data.load_balancer_security_group_sources - - Route53ZoneID: !ssm /d2d/dev/route53-hosted-zone-id - Domain: d2d-zephir-api-dev.d2ddev.cdlib.net - ApplicationPort: "8000" -sceptre_user_data: - load_balancer_security_group_sources: - - !ssm /d2d/dev/ucop-vpn-prefix-list-id - - !ssm /d2d/dev/cdl-eip-prefix-list-id \ No newline at end of file + Route53ZoneID: !ssm /d2d/dev/route53-hosted-zone-id + Domain: "{{ sceptre_user_data.url_authority }}" + ApplicationPort: "{{ sceptre_user_data.port }}" diff --git a/deployment/config/dev/waf.yaml b/deployment/config/dev/waf.yaml index 5292525..b3ac5e9 100644 --- a/deployment/config/dev/waf.yaml +++ b/deployment/config/dev/waf.yaml @@ -1,11 +1,9 @@ template: - path: waf.yaml + path: waf.yaml.j2 type: file dependencies: - - dev/network.yaml + - "{{ sceptre_user_data.environment }}/network.yaml" parameters: - WebACLName: "{{ base_name }}-{{ environment }}-waf-webacl" - WebACLMetricName: "{{ base_name }}-{{ environment }}-waf-webacl-metric" - LoadBalancerArn: !stack_output dev/network.yaml::LoadBalancerARN + LoadBalancerArn: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerARN" diff --git a/deployment/config/prd/autoscaling.yaml b/deployment/config/prd/autoscaling.yaml index 3f1d675..77322a4 100644 --- a/deployment/config/prd/autoscaling.yaml +++ b/deployment/config/prd/autoscaling.yaml @@ -3,14 +3,14 @@ template: type: file dependencies: - - prd/ecs.yaml + - "{{ sceptre_user_data.environment }}/ecs.yaml" parameters: - # ECSAutoScalingPlanName: "{{ base_name }}-{{ environment }}-ecs-auto-scaling-plan" - ECSClusterName: !stack_output prd/ecs.yaml::ECSClusterName - ECSServiceName: !stack_output prd/ecs.yaml::ECSServiceName + # ECSAutoScalingPlanName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-auto-scaling-plan" + ECSClusterName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSClusterName" + ECSServiceName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSServiceName" - ECSCloudFormationStackARN: !stack_output prd/ecs.yaml::ECSCloudFormationStackARN + ECSCloudFormationStackARN: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSCloudFormationStackARN" MinCapacity: "1" MaxCapacity: "3" \ No newline at end of file diff --git a/deployment/config/prd/config.yaml b/deployment/config/prd/config.yaml index 972d5af..758a467 100644 --- a/deployment/config/prd/config.yaml +++ b/deployment/config/prd/config.yaml @@ -1,9 +1,9 @@ -# profile: cdl-d2d-prd -base_name: {{ project_code }} -environment: prd +profile: cdl-d2d-prd + +sceptre_user_data: + environment: prd + resource_name_prefix: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-prd" + url_authority: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}.d2dprd.cdlib.net" stack_tags: - Program: "d2d" - Service: "zephir" - Subservice: "api" - Environment: "prd" + Environment: !stack_attr sceptre_user_data.environment diff --git a/deployment/config/prd/ecr.yaml b/deployment/config/prd/ecr.yaml index f81eed3..bda5fd9 100644 --- a/deployment/config/prd/ecr.yaml +++ b/deployment/config/prd/ecr.yaml @@ -1,6 +1,6 @@ template: - path: ecr.yaml + path: ecr.yaml.j2 type: file -parameters: - RepositoryName: "{{ base_name }}-{{ environment }}" \ No newline at end of file +# Comment out to create once, ignore afterwards: +ignore: True diff --git a/deployment/config/prd/ecs.yaml b/deployment/config/prd/ecs.yaml index 0d3aded..225a0f7 100644 --- a/deployment/config/prd/ecs.yaml +++ b/deployment/config/prd/ecs.yaml @@ -1,43 +1,37 @@ template: - path: ecs.yaml + path: ecs.yaml.j2 type: file dependencies: - - prd/ecr.yaml - - prd/logging.yaml - - prd/network.yaml + - "{{ sceptre_user_data.environment }}/ecr.yaml" + - "{{ sceptre_user_data.environment }}/logging.yaml" + - "{{ sceptre_user_data.environment }}/network.yaml" parameters: - ECSServiceSecurityGroupName: "{{ base_name }}-{{ environment }}-ecs-service-security-group" - ECSClusterName: "{{ base_name }}-{{ environment }}-ecs-cluster" - ContainerName: "{{ base_name }}-{{ environment }}-container" - TaskDefinitionFamilyName: "{{ base_name }}-{{ environment }}-ecs-task-definition" - ECSTaskRoleName: "{{ base_name }}-{{ environment }}-ecs-task-role" - ECSTaskExecutionRoleName: "{{ base_name }}-{{ environment }}-ecs-task-execution-role" - ECSServiceName: "{{ base_name }}-{{ environment }}-ecs-service" - {% if var.image_tag | default(False) %} + ContainerName: "{{ sceptre_user_data.resource_name_prefix }}-container" + {% if var.image_tag | default(False) -%} ImageTag: "{{ var.image_tag }}" - {% else %} + {% else -%} ImageTag: latest {% endif %} - + VPCID: !stack_output_external cdl-d2d-prd-vpc-stack::vpc SubnetIDs: - !stack_output_external cdl-d2d-prd-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-d2d-prd-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-d2d-prd-defaultsubnet-stack::defaultsubnet2c - LoadBalancerSecurityGroupID: !stack_output prd/network.yaml::LoadBalancerSecurityGroupID - TargetGroupARN: !stack_output prd/network.yaml::TargetGroupARN + LoadBalancerSecurityGroupID: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerSecurityGroupID" + TargetGroupARN: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::TargetGroupARN" + + RepositoryURI: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryURI" + RepositoryARN: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryARN" + + LogGroupID: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupID" + LogGroupArn: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupArn" - RepositoryURI: !stack_output prd/ecr.yaml::RepositoryURI - RepositoryARN: !stack_output prd/ecr.yaml::RepositoryARN - - LogGroupID: !stack_output prd/logging.yaml::LogGroupID - LogGroupArn: !stack_output prd/logging.yaml::LogGroupArn + ApplicationPort: "{{ sceptre_user_data.port }}" - ApplicationPort: "8000" - FlaskEnvironment: production LogLevel: INFO diff --git a/deployment/config/prd/logging.yaml b/deployment/config/prd/logging.yaml index 856b31d..d488df2 100644 --- a/deployment/config/prd/logging.yaml +++ b/deployment/config/prd/logging.yaml @@ -1,6 +1,3 @@ template: - path: logging.yaml + path: logging.yaml.j2 type: file - -parameters: - LogGroupName: "{{ base_name }}-{{ environment }}-log-group" \ No newline at end of file diff --git a/deployment/config/prd/network.yaml b/deployment/config/prd/network.yaml index e2c9414..fbe9814 100644 --- a/deployment/config/prd/network.yaml +++ b/deployment/config/prd/network.yaml @@ -2,24 +2,20 @@ template: path: network.yaml.j2 type: file -parameters: - LoadBalancerSecurityGroupName: "{{ base_name }}-{{ environment }}-load-balancer-security-group" - LoadBalancerName: "{{ base_name }}-{{ environment }}-load-balancer" - TargetGroupName: "{{ base_name }}-{{ environment }}-target-group" +sceptre_user_data: + load_balancer_security_group_sources: + - 0.0.0.0/0 +parameters: VPCID: !stack_output_external cdl-d2d-prd-vpc-stack::vpc SubnetIDs: - !stack_output_external cdl-d2d-prd-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-d2d-prd-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-d2d-prd-defaultsubnet-stack::defaultsubnet2c LoadBalancerSecurityGroupSources: !stack_attr sceptre_user_data.load_balancer_security_group_sources - + Route53ZoneID: !ssm /d2d/prd/route53-hosted-zone-id - Domain: d2d-zephir-api.d2dprd.cdlib.net + Domain: "{{ sceptre_user_data.url_authority }}" SubjectAlternativeDomains: # If the domain ends in .org, IAS should've validated it before. - - zephir.cdlib.org - ApplicationPort: "8000" - -sceptre_user_data: - load_balancer_security_group_sources: - - 0.0.0.0/0 + - zephir.cdlib.org + ApplicationPort: "{{ sceptre_user_data.port }}" diff --git a/deployment/config/prd/waf.yaml b/deployment/config/prd/waf.yaml index 49b1a13..b3ac5e9 100644 --- a/deployment/config/prd/waf.yaml +++ b/deployment/config/prd/waf.yaml @@ -1,11 +1,9 @@ template: - path: waf.yaml + path: waf.yaml.j2 type: file dependencies: - - prd/network.yaml + - "{{ sceptre_user_data.environment }}/network.yaml" parameters: - WebACLName: "{{ base_name }}-{{ environment }}-waf-webacl" - WebACLMetricName: "{{ base_name }}-{{ environment }}-waf-webacl-metric" - LoadBalancerArn: !stack_output prd/network.yaml::LoadBalancerARN + LoadBalancerArn: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerARN" diff --git a/deployment/config/stg/autoscaling.yaml b/deployment/config/stg/autoscaling.yaml index da77e21..77322a4 100644 --- a/deployment/config/stg/autoscaling.yaml +++ b/deployment/config/stg/autoscaling.yaml @@ -3,14 +3,14 @@ template: type: file dependencies: - - stg/ecs.yaml + - "{{ sceptre_user_data.environment }}/ecs.yaml" parameters: - # ECSAutoScalingPlanName: "{{ base_name }}-{{ environment }}-ecs-auto-scaling-plan" - ECSClusterName: !stack_output stg/ecs.yaml::ECSClusterName - ECSServiceName: !stack_output stg/ecs.yaml::ECSServiceName + # ECSAutoScalingPlanName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-auto-scaling-plan" + ECSClusterName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSClusterName" + ECSServiceName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSServiceName" - ECSCloudFormationStackARN: !stack_output stg/ecs.yaml::ECSCloudFormationStackARN + ECSCloudFormationStackARN: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSCloudFormationStackARN" MinCapacity: "1" MaxCapacity: "3" \ No newline at end of file diff --git a/deployment/config/stg/config.yaml b/deployment/config/stg/config.yaml index 6a8dd6b..060653d 100644 --- a/deployment/config/stg/config.yaml +++ b/deployment/config/stg/config.yaml @@ -1,12 +1,12 @@ +profile: cdl-d2d-dev + sceptre_role: arn:aws:iam::445017934155:role/sceptre/d2d-dev-sceptre-role cloudformation_service_role: arn:aws:iam::445017934155:role/sceptre/d2d-zephir-api-stg-cloudformation-service-role -# Custom parameters -base_name: {{ project_code }} -environment: stg +sceptre_user_data: + environment: stg + resource_name_prefix: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-stg" + url_authority: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-stg.d2ddev.cdlib.net" stack_tags: - Program: "d2d" - Service: "zephir" - Subservice: "api" - Environment: "stg" + Environment: !stack_attr sceptre_user_data.environment diff --git a/deployment/config/stg/ecr.yaml b/deployment/config/stg/ecr.yaml index 100baaa..bda5fd9 100644 --- a/deployment/config/stg/ecr.yaml +++ b/deployment/config/stg/ecr.yaml @@ -1,9 +1,6 @@ template: - path: ecr.yaml + path: ecr.yaml.j2 type: file -parameters: - RepositoryName: "{{ base_name }}-{{ environment }}" - # Comment out to create once, ignore afterwards: ignore: True diff --git a/deployment/config/stg/ecs.yaml b/deployment/config/stg/ecs.yaml index 9f12938..4f81194 100644 --- a/deployment/config/stg/ecs.yaml +++ b/deployment/config/stg/ecs.yaml @@ -1,43 +1,37 @@ template: - path: ecs.yaml + path: ecs.yaml.j2 type: file dependencies: - - stg/ecr.yaml - - stg/logging.yaml - - stg/network.yaml + - "{{ sceptre_user_data.environment }}/ecr.yaml" + - "{{ sceptre_user_data.environment }}/logging.yaml" + - "{{ sceptre_user_data.environment }}/network.yaml" parameters: - ECSServiceSecurityGroupName: "{{ base_name }}-{{ environment }}-ecs-service-security-group" - ECSClusterName: "{{ base_name }}-{{ environment }}-ecs-cluster" - ContainerName: "{{ base_name }}-{{ environment }}-container" - TaskDefinitionFamilyName: "{{ base_name }}-{{ environment }}-ecs-task-definition" - ECSTaskRoleName: "{{ base_name }}-{{ environment }}-ecs-task-role" - ECSTaskExecutionRoleName: "{{ base_name }}-{{ environment }}-ecs-task-execution-role" - ECSServiceName: "{{ base_name }}-{{ environment }}-ecs-service" - {% if var.image_tag | default(False) %} + ContainerName: "{{ sceptre_user_data.resource_name_prefix }}-container" + {% if var.image_tag | default(False) -%} ImageTag: "{{ var.image_tag }}" - {% else %} + {% else -%} ImageTag: latest {% endif %} - + VPCID: !stack_output_external cdl-d2d-dev-vpc-stack::vpc SubnetIDs: - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2c - LoadBalancerSecurityGroupID: !stack_output stg/network.yaml::LoadBalancerSecurityGroupID - TargetGroupARN: !stack_output stg/network.yaml::TargetGroupARN + LoadBalancerSecurityGroupID: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerSecurityGroupID" + TargetGroupARN: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::TargetGroupARN" + + RepositoryURI: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryURI" + RepositoryARN: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryARN" + + LogGroupID: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupID" + LogGroupArn: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupArn" - RepositoryURI: !stack_output stg/ecr.yaml::RepositoryURI - RepositoryARN: !stack_output stg/ecr.yaml::RepositoryARN - - LogGroupID: !stack_output stg/logging.yaml::LogGroupID - LogGroupArn: !stack_output stg/logging.yaml::LogGroupArn + ApplicationPort: "{{ sceptre_user_data.port }}" - ApplicationPort: "8000" - FlaskEnvironment: development LogLevel: DEBUG diff --git a/deployment/config/stg/logging.yaml b/deployment/config/stg/logging.yaml index 856b31d..d488df2 100644 --- a/deployment/config/stg/logging.yaml +++ b/deployment/config/stg/logging.yaml @@ -1,6 +1,3 @@ template: - path: logging.yaml + path: logging.yaml.j2 type: file - -parameters: - LogGroupName: "{{ base_name }}-{{ environment }}-log-group" \ No newline at end of file diff --git a/deployment/config/stg/network.yaml b/deployment/config/stg/network.yaml index 3bcbfc8..d91056f 100644 --- a/deployment/config/stg/network.yaml +++ b/deployment/config/stg/network.yaml @@ -2,23 +2,21 @@ template: path: network.yaml.j2 type: file -parameters: - LoadBalancerSecurityGroupName: "{{ base_name }}-{{ environment }}-load-balancer-security-group" - LoadBalancerName: "{{ base_name }}-{{ environment }}-load-balancer" - TargetGroupName: "{{ base_name }}-{{ environment }}-target-group" +# stg shares the dev AWS account and VPC. All resource references below intentionally +# point to dev-account infrastructure and SSM parameters. +sceptre_user_data: + load_balancer_security_group_sources: + - !ssm /d2d/dev/ucop-vpn-prefix-list-id + - !ssm /d2d/dev/cdl-eip-prefix-list-id +parameters: VPCID: !stack_output_external cdl-d2d-dev-vpc-stack::vpc SubnetIDs: - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2c LoadBalancerSecurityGroupSources: !stack_attr sceptre_user_data.load_balancer_security_group_sources - - Route53ZoneID: !ssm /d2d/dev/route53-hosted-zone-id - Domain: d2d-zephir-api-stg.d2ddev.cdlib.net - ApplicationPort: "8000" -sceptre_user_data: - load_balancer_security_group_sources: - - !ssm /d2d/dev/ucop-vpn-prefix-list-id - - !ssm /d2d/dev/cdl-eip-prefix-list-id + Route53ZoneID: !ssm /d2d/dev/route53-hosted-zone-id + Domain: "{{ sceptre_user_data.url_authority }}" + ApplicationPort: "{{ sceptre_user_data.port }}" diff --git a/deployment/config/stg/waf.yaml b/deployment/config/stg/waf.yaml index 5b49f63..b3ac5e9 100644 --- a/deployment/config/stg/waf.yaml +++ b/deployment/config/stg/waf.yaml @@ -1,11 +1,9 @@ template: - path: waf.yaml + path: waf.yaml.j2 type: file dependencies: - - stg/network.yaml + - "{{ sceptre_user_data.environment }}/network.yaml" parameters: - WebACLName: "{{ base_name }}-{{ environment }}-waf-webacl" - WebACLMetricName: "{{ base_name }}-{{ environment }}-waf-webacl-metric" - LoadBalancerArn: !stack_output stg/network.yaml::LoadBalancerARN + LoadBalancerArn: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerARN" diff --git a/deployment/templates/codebuild.yaml.j2 b/deployment/templates/codebuild.yaml.j2 new file mode 100644 index 0000000..1b977c7 --- /dev/null +++ b/deployment/templates/codebuild.yaml.j2 @@ -0,0 +1,85 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: CodeBuild CI project for Zephir API + +Parameters: + GitHubConnectionArn: + Type: String + Description: ARN of the manually-created and activated CodeStar connection to GitHub + +Resources: + CodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + RoleName: "{{ sceptre_user_data.resource_name_prefix }}-codebuild-service-role" + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: CloudWatch-Logs + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: + - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/{{ sceptre_user_data.resource_name_prefix }}-ci" + - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/{{ sceptre_user_data.resource_name_prefix }}-ci:*" + - PolicyName: CodeBuild-Reports + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - codebuild:CreateReportGroup + - codebuild:CreateReport + - codebuild:UpdateReport + - codebuild:BatchPutTestCases + Resource: !Sub "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:report-group/{{ sceptre_user_data.resource_name_prefix }}-ci-*" + - PolicyName: GitHub-Connection + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - codeconnections:GetConnectionToken + - codeconnections:GetConnection + Resource: !Ref GitHubConnectionArn + + CodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Name: "{{ sceptre_user_data.resource_name_prefix }}-ci" + Description: "CI for Zephir API" + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + Source: + Type: GITHUB + Location: "{{ sceptre_user_data.repository }}" + BuildSpec: buildspec.yml + GitCloneDepth: 1 + ReportBuildStatus: true + Auth: + Type: OAUTH + Resource: !Ref GitHubConnectionArn + Environment: + Type: LINUX_CONTAINER + Image: aws/codebuild/standard:7.0 + ComputeType: BUILD_GENERAL1_SMALL + Triggers: + Webhook: true + FilterGroups: + - - Type: EVENT + Pattern: PULL_REQUEST_CREATED,PULL_REQUEST_UPDATED,PULL_REQUEST_REOPENED + Artifacts: + Type: NO_ARTIFACTS + +Outputs: + ProjectName: + Value: !Ref CodeBuildProject + ProjectArn: + Value: !GetAtt CodeBuildProject.Arn diff --git a/deployment/templates/ecr.yaml b/deployment/templates/ecr.yaml.j2 similarity index 78% rename from deployment/templates/ecr.yaml rename to deployment/templates/ecr.yaml.j2 index 90de146..8cda830 100644 --- a/deployment/templates/ecr.yaml +++ b/deployment/templates/ecr.yaml.j2 @@ -1,11 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: ECR repository Zephir API - - -Parameters: - RepositoryName: - Type: String - +Description: ECR repository for Zephir API Resources: Repository: @@ -28,15 +22,12 @@ Resources: } } ]} - RepositoryName: !Ref RepositoryName - + RepositoryName: "{{ sceptre_user_data.resource_name_prefix }}" Outputs: RepositoryName: - Value: !Ref RepositoryName - + Value: !Ref Repository RepositoryURI: Value: !GetAtt Repository.RepositoryUri - RepositoryARN: - Value: !GetAtt Repository.Arn + Value: !GetAtt Repository.Arn diff --git a/deployment/templates/ecs.yaml b/deployment/templates/ecs.yaml.j2 similarity index 83% rename from deployment/templates/ecs.yaml rename to deployment/templates/ecs.yaml.j2 index 6ceacfb..0f12410 100644 --- a/deployment/templates/ecs.yaml +++ b/deployment/templates/ecs.yaml.j2 @@ -1,25 +1,12 @@ AWSTemplateFormatVersion: 2010-09-09 Description: Service Stack for Zephir API - Parameters: - ECSServiceSecurityGroupName: - Type: String - ECSClusterName: - Type: String ContainerName: Type: String - TaskDefinitionFamilyName: - Type: String - ECSTaskRoleName: - Type: String - ECSTaskExecutionRoleName: - Type: String - ECSServiceName: - Type: String ImageTag: Type: String - + VPCID: Type: AWS::EC2::VPC::Id SubnetIDs: @@ -57,7 +44,7 @@ Resources: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Security group for ECS tasks - GroupName: !Ref ECSServiceSecurityGroupName + GroupName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-service-security-group" SecurityGroupIngress: - SourceSecurityGroupId: !Ref LoadBalancerSecurityGroupID IpProtocol: tcp @@ -68,7 +55,7 @@ Resources: ECSCluster: Type: AWS::ECS::Cluster Properties: - ClusterName: !Ref ECSClusterName + ClusterName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-cluster" TaskDefinition: Type: AWS::ECS::TaskDefinition @@ -83,6 +70,17 @@ Resources: awslogs-group: !Ref LogGroupID awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Ref ContainerName + HealthCheck: + Command: + - "CMD" + - "uv" + - "run" + - "python" + - "scripts/healthcheck.py" + Interval: 30 + Timeout: 5 + StartPeriod: 10 + Retries: 3 PortMappings: - AppProtocol: http ContainerPort: !Ref ApplicationPort @@ -97,7 +95,7 @@ Resources: Value: !Ref ApplicationPort - Name: DATABASE_CREDENTIALS_SECRET_NAME Value: !Ref DatabaseCredentialsSecretName - Command: + Command: - "uv" - "run" - "python" @@ -108,16 +106,16 @@ Resources: Cpu: "256" TaskRoleArn: !Ref ECSTaskRole ExecutionRoleArn: !Ref ECSTaskExecutionRole - Family: !Ref TaskDefinitionFamilyName + Family: "{{ sceptre_user_data.resource_name_prefix }}-ecs-task-definition" Memory: "512" NetworkMode: awsvpc - RequiresCompatibilities: + RequiresCompatibilities: - FARGATE ECSTaskRole: Type: AWS::IAM::Role Properties: - RoleName: !Ref ECSTaskRoleName + RoleName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-task-role" AssumeRolePolicyDocument: Statement: - Effect: Allow @@ -138,15 +136,15 @@ Resources: ECSTaskExecutionRole: Type: AWS::IAM::Role Properties: - RoleName: !Ref ECSTaskExecutionRoleName + RoleName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-task-execution-role" AssumeRolePolicyDocument: Statement: - Effect: Allow - Principal: + Principal: Service: ecs-tasks.amazonaws.com Action: sts:AssumeRole Policies: - - PolicyName: ECR-Pull + - PolicyName: "{{ sceptre_user_data.resource_name_prefix }}-execution-role-policy" PolicyDocument: Version: 2012-10-17 Statement: @@ -159,10 +157,6 @@ Resources: - Effect: Allow Action: ecr:GetAuthorizationToken Resource: '*' - - PolicyName: LogGroup-Log - PolicyDocument: - Version: 2012-10-17 - Statement: - Effect: Allow Action: - logs:CreateLogStream @@ -181,11 +175,11 @@ Resources: NetworkConfiguration: AwsvpcConfiguration: AssignPublicIp: ENABLED - SecurityGroups: + SecurityGroups: - !GetAtt ECSServiceSecurityGroup.GroupId - !Ref DatabaseSecurityGroupID Subnets: !Ref SubnetIDs - ServiceName: !Ref ECSServiceName + ServiceName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-service" TaskDefinition: !Ref TaskDefinition LoadBalancers: - ContainerName: !Ref ContainerName diff --git a/deployment/templates/logging.yaml b/deployment/templates/logging.yaml.j2 similarity index 69% rename from deployment/templates/logging.yaml rename to deployment/templates/logging.yaml.j2 index 160fec8..742aaca 100644 --- a/deployment/templates/logging.yaml +++ b/deployment/templates/logging.yaml.j2 @@ -1,24 +1,16 @@ AWSTemplateFormatVersion: 2010-09-09 Description: Log Group for Zephir API - -Parameters: - LogGroupName: - Type: String - - Resources: LogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: !Ref LogGroupName + LogGroupName: "{{ sceptre_user_data.resource_name_prefix }}-log-group" RetentionInDays: 14 UpdateReplacePolicy: Retain - Outputs: LogGroupID: Value: !Ref LogGroup - LogGroupArn: - Value: !GetAtt LogGroup.Arn \ No newline at end of file + Value: !GetAtt LogGroup.Arn diff --git a/deployment/templates/network.yaml.j2 b/deployment/templates/network.yaml.j2 index 1603adf..3700568 100644 --- a/deployment/templates/network.yaml.j2 +++ b/deployment/templates/network.yaml.j2 @@ -1,15 +1,7 @@ AWSTemplateFormatVersion: 2010-09-09 Description: Network Stack for Zephir API - Parameters: - LoadBalancerSecurityGroupName: - Type: String - LoadBalancerName: - Type: String - TargetGroupName: - Type: String - VPCID: Type: AWS::EC2::VPC::Id SubnetIDs: @@ -27,12 +19,10 @@ Parameters: ApplicationPort: Type: Number - Conditions: HasSubjectAlternativeDomains: !Not [!Equals [!Join [",", !Ref SubjectAlternativeDomains], ""]] - Resources: SSLCertificate: Type: AWS::CertificateManager::Certificate @@ -46,8 +36,8 @@ Resources: Fn::If: - HasSubjectAlternativeDomains - !Ref SubjectAlternativeDomains - - !Ref AWS::NoValue - + - !Ref AWS::NoValue + MainSiteDNS: Type: AWS::Route53::RecordSet Properties: @@ -62,7 +52,7 @@ Resources: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Security group for the Load Balancer - GroupName: !Ref LoadBalancerSecurityGroupName + GroupName: "{{ sceptre_user_data.resource_name_prefix }}-load-balancer-security-group" VpcId: !Ref VPCID SecurityGroupIngress: {% for source in sceptre_user_data.load_balancer_security_group_sources %} @@ -81,7 +71,7 @@ Resources: LoadBalancer: Type: AWS::ElasticLoadBalancingV2::LoadBalancer Properties: - Name: !Ref LoadBalancerName + Name: "{{ sceptre_user_data.resource_name_prefix }}-load-balancer" Subnets: !Ref SubnetIDs SecurityGroups: - !GetAtt LoadBalancerSecurityGroup.GroupId @@ -116,27 +106,23 @@ Resources: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: HealthCheckIntervalSeconds: 30 - # will look for a 200 status code by default unless specified otherwise HealthCheckPath: "/api/ping" HealthCheckTimeoutSeconds: 5 UnhealthyThresholdCount: 2 HealthyThresholdCount: 2 - Name: !Ref TargetGroupName + Name: "{{ sceptre_user_data.resource_name_prefix }}-target-group" Port: !Ref ApplicationPort Protocol: HTTP TargetGroupAttributes: - Key: deregistration_delay.timeout_seconds - Value: 60 # default is 300 + Value: 60 TargetType: ip VpcId: !Ref VPCID - Outputs: LoadBalancerSecurityGroupID: Value: !GetAtt LoadBalancerSecurityGroup.GroupId - TargetGroupARN: Value: !GetAtt TargetGroup.TargetGroupArn - LoadBalancerARN: Value: !Ref LoadBalancer diff --git a/deployment/templates/waf.yaml b/deployment/templates/waf.yaml.j2 similarity index 96% rename from deployment/templates/waf.yaml rename to deployment/templates/waf.yaml.j2 index 756a5ed..0120920 100644 --- a/deployment/templates/waf.yaml +++ b/deployment/templates/waf.yaml.j2 @@ -1,26 +1,21 @@ AWSTemplateFormatVersion: 2010-09-09 Description: WAF for Zephir API - Parameters: - WebACLName: - Type: String - WebACLMetricName: - Type: String LoadBalancerArn: Type: String Resources: WebACL: Type: AWS::WAFv2::WebACL - Properties: - Name: !Ref WebACLName + Properties: + Name: "{{ sceptre_user_data.resource_name_prefix }}-waf-webacl" DefaultAction: Allow: {} Scope: REGIONAL VisibilityConfig: CloudWatchMetricsEnabled: true - MetricName: !Ref WebACLMetricName + MetricName: "{{ sceptre_user_data.resource_name_prefix }}-waf-webacl-metric" SampledRequestsEnabled: true Rules: - Name: AWS-AWSManagedRulesAmazonIpReputationList diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py new file mode 100644 index 0000000..114b0aa --- /dev/null +++ b/scripts/healthcheck.py @@ -0,0 +1,10 @@ +import os +import sys +import urllib.request + +port = os.environ.get("APP_PORT", "8000") + +try: + urllib.request.urlopen(f"http://localhost:{port}/api/ping", timeout=5) +except Exception: + sys.exit(1)