Migrate CI to AWS CodeBuild

.githooks/pre-commit

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+echo "Running ruff..."
+uv run ruff check .
+
+echo "Running vulture..."
+uv run vulture . --min-confidence 70 --exclude .venv

Dockerfile

@@ -61,4 +61,7 @@ COPY --chown=app:app . .
 # Switch to the unprivileged user before starting the process
 USER app
 
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+    CMD ["uv", "run", "python", "scripts/healthcheck.py"]
+
 CMD ["uv", "run", "python", "-m", "gunicorn", "-c", "gunicorn_config.py"]

README.md

@@ -1,10 +1,13 @@
 # zephir_api2
-
-![CI](https://github.com/cdlib/zephir-api2/actions/workflows/ci.yml/badge.svg)
 
 The Zephir Item API provides item-level MARC metadata records for all items submitted to HathiTrust.
+
 See [API.md](API.md) for full endpoint documentation.
 
+## Workflow
+
+This repository uses [GitHub flow](https://docs.github.com/en/get-started/using-github/github-flow). The `main` branch is always assumed to be tested and ready for deployment to production. Ensure all changes are adequately tested _before_ merging into `main`.
+
 ## Getting Started
 
 ### Prerequisites
@@ -17,31 +20,41 @@ brew install uv
 
 ### Local Development
 
-**Install dependencies:**
+1. Install dependencies
 
-```sh
-uv sync
-```
+    ```sh
+    uv sync
+    ```
 
-**Set required environment variables** (see [Database configuration](#database-configuration) below), then **run the app:**
+2. Install git hooks
 
-Copy `env.template` to `.env` and update the `DATABASE_URI` variable.
+    ```sh
+    git config core.hooksPath .githooks
+    ```
 
-```sh
-uv run python -m gunicorn -c gunicorn_config.py
-```
+    This enables the pre-commit hook that runs ruff and vulture before each commit.
+
+3. Set required environment variables
+
+    Copy `env.template` to `.env` and update the `DATABASE_URI` variable. See [Database configuration](#database-configuration) below.
 
-The API will be available at `http://localhost:8000/api/`.
+4. Run the app
 
-**Run tests:**
+    ```sh
+    uv run python -m gunicorn -c gunicorn_config.py
+    ```
+
+    The API will be available at `http://localhost:8000/api/`.
+
+### Running tests
 
 ```sh
 uv run pytest tests
 ```
 
 Tests use a bundled SQLite fixture (`tests/test_zephir_api/`) and do not require a live database connection.
 
-**Run linting and analysis tools:**
+### Linting, etc.
 
 ```sh
 uv run ruff check .
@@ -94,7 +107,19 @@ The app resolves the database connection from environment variables in priority
 | `DATABASE_CREDENTIALS_SECRET_NAME` + AWS vars | Pull credentials from AWS Secrets Manager |
 | `DB_USERNAME` + `DB_PASSWORD` + `DB_HOST` + `DB_DATABASE` | Individual credential env vars |
 
----
+## CI (Continuous Integration)
+
+CI runs on AWS CodeBuild in the `cdl-d2d-dev` account. Every pull request triggers a build that runs tests, linting, and security checks, and reports a check status with a link to the build log back to GitHub.
+
+The job configuration lives in [`buildspec.yml`](buildspec.yml).
+
+### First-time setup
+
+The CodeBuild project is managed by Sceptre. Note that the GitHub connection is created and activated manually before deploying.
+
+### Test results
+
+Build status is automatically reported to GitHub on every PR. Test results are published to the CodeBuild **Test reports** panel (JUnit XML).
 
 ## Deploying to AWS
 

buildspec.yml

@@ -0,0 +1,25 @@
+version: 0.2
+
+phases:
+  install:
+    commands:
+      - pip install uv
+
+  pre_build:
+    commands:
+      - uv sync --frozen
+
+  build:
+    commands:
+      - uv run python --version
+      - uv tree --outdated
+      - uv run pytest --cov=app --cov-report=term-missing --junitxml=pytest-report.xml
+      - uv run bandit -r app
+      - uv run ruff check .
+      - uv run vulture . --min-confidence 70 --exclude .venv
+
+reports:
+  pytest-reports:
+    files:
+      - pytest-report.xml
+    file-format: JUNITXML

deployment/config/config.yaml

@@ -1,2 +1,18 @@
 project_code: d2d-zephir-api
-region: us-west-2
+region: us-west-2
+
+sceptre_user_data_inheritance: merge
+stack_tags_inheritance: merge
+
+sceptre_user_data:
+  program: d2d
+  service: zephir
+  subservice: api
+  repository: https://github.com/cdlib/zephir-api2.git
+  port: "8000"
+
+stack_tags:
+  program: !stack_attr sceptre_user_data.program
+  service: !stack_attr sceptre_user_data.service
+  subservice: !stack_attr sceptre_user_data.subservice
+  repository: !stack_attr sceptre_user_data.repository

deployment/config/dev/autoscaling.yaml

@@ -3,14 +3,14 @@ template:
   type: file
 
 dependencies:
-  - dev/ecs.yaml
+  - "{{ sceptre_user_data.environment }}/ecs.yaml"
 
 parameters:
-  # ECSAutoScalingPlanName: "{{ base_name }}-{{ environment }}-ecs-auto-scaling-plan"
-  ECSClusterName: !stack_output dev/ecs.yaml::ECSClusterName
-  ECSServiceName: !stack_output dev/ecs.yaml::ECSServiceName
+  # ECSAutoScalingPlanName: "{{ sceptre_user_data.resource_name_prefix }}-ecs-auto-scaling-plan"
+  ECSClusterName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSClusterName"
+  ECSServiceName: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSServiceName"
 
-  ECSCloudFormationStackARN: !stack_output dev/ecs.yaml::ECSCloudFormationStackARN
+  ECSCloudFormationStackARN: !stack_output "{{ sceptre_user_data.environment }}/ecs.yaml::ECSCloudFormationStackARN"
 
   MinCapacity: "1"
   MaxCapacity: "3"

deployment/config/dev/codebuild.yaml

@@ -0,0 +1,10 @@
+template:
+  path: codebuild.yaml.j2
+  type: file
+
+# No dependencies on other stacks; CodeBuild is independent of ECS/network/etc.
+#
+# GitHubConnectionArn must be created and activated manually before deploying.
+# See the "CI" section in README.md for instructions.
+parameters:
+  GitHubConnectionArn: arn:aws:codeconnections:us-west-2:445017934155:connection/6af1f14e-4157-4e78-a1bd-73f160a9c4c5

deployment/config/dev/config.yaml

@@ -1,9 +1,11 @@
-# profile: cdl-d2d-dev
-base_name: {{ project_code }}
-environment: dev
+profile: cdl-d2d-dev
+
+sceptre_user_data:
+  environment: dev
+  resource_name_prefix: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-dev"
+  url_authority: "{{ sceptre_user_data.program }}-{{ sceptre_user_data.service }}-{{ sceptre_user_data.subservice }}-dev.d2ddev.cdlib.net"
 
 stack_tags:
-  Program: "d2d"
-  Service: "zephir"
-  Subservice: "api"
-  Environment: "dev"
+  # Use !stack_attr instead of jinja syntax because "environment" is defined in same template.
+  # The resolver !stack_attr executes after jinja.
+  Environment: !stack_attr sceptre_user_data.environment

deployment/config/dev/ecr.yaml

@@ -1,6 +1,6 @@
 template:
-  path: ecr.yaml
+  path: ecr.yaml.j2
   type: file
 
-parameters:
-  RepositoryName: "{{ base_name }}-{{ environment }}"
+# Comment out to create once, ignore afterwards:
+ignore: True

deployment/config/dev/ecs.yaml

@@ -1,43 +1,37 @@
 template:
-  path: ecs.yaml
+  path: ecs.yaml.j2
   type: file
 
 dependencies:
-  - dev/ecr.yaml
-  - dev/logging.yaml
-  - dev/network.yaml
+  - "{{ sceptre_user_data.environment }}/ecr.yaml"
+  - "{{ sceptre_user_data.environment }}/logging.yaml"
+  - "{{ sceptre_user_data.environment }}/network.yaml"
 
 parameters:
-  ECSServiceSecurityGroupName: "{{ base_name }}-{{ environment }}-ecs-service-security-group"
-  ECSClusterName: "{{ base_name }}-{{ environment }}-ecs-cluster"
-  ContainerName: "{{ base_name }}-{{ environment }}-container"
-  TaskDefinitionFamilyName: "{{ base_name }}-{{ environment }}-ecs-task-definition"
-  ECSTaskRoleName: "{{ base_name }}-{{ environment }}-ecs-task-role"
-  ECSTaskExecutionRoleName: "{{ base_name }}-{{ environment }}-ecs-task-execution-role"
-  ECSServiceName: "{{ base_name }}-{{ environment }}-ecs-service"
+  ContainerName: "{{ sceptre_user_data.resource_name_prefix }}-container"
   {% if var.image_tag | default(False) %}
   ImageTag: "{{ var.image_tag }}"
   {% else %}
   ImageTag: latest
   {% endif %}
-  
+
   VPCID: !stack_output_external cdl-d2d-dev-vpc-stack::vpc
   SubnetIDs:
     - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2a
     - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2b
     - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2c
 
-  LoadBalancerSecurityGroupID: !stack_output dev/network.yaml::LoadBalancerSecurityGroupID
-  TargetGroupARN: !stack_output dev/network.yaml::TargetGroupARN
+  LoadBalancerSecurityGroupID: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerSecurityGroupID"
+  TargetGroupARN: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::TargetGroupARN"
+
+  RepositoryURI: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryURI"
+  RepositoryARN: !stack_output "{{ sceptre_user_data.environment }}/ecr.yaml::RepositoryARN"
+
+  LogGroupID: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupID"
+  LogGroupArn: !stack_output "{{ sceptre_user_data.environment }}/logging.yaml::LogGroupArn"
 
-  RepositoryURI: !stack_output dev/ecr.yaml::RepositoryURI
-  RepositoryARN: !stack_output dev/ecr.yaml::RepositoryARN
-
-  LogGroupID: !stack_output dev/logging.yaml::LogGroupID
-  LogGroupArn: !stack_output dev/logging.yaml::LogGroupArn
+  ApplicationPort: "{{ sceptre_user_data.port }}"
 
-  ApplicationPort: "8000"
-
   FlaskEnvironment: development
   LogLevel: DEBUG
 

deployment/config/dev/logging.yaml

@@ -1,6 +1,3 @@
 template:
-  path: logging.yaml
+  path: logging.yaml.j2
   type: file
-
-parameters:
-  LogGroupName: "{{ base_name }}-{{ environment }}-log-group"

deployment/config/dev/network.yaml

@@ -2,23 +2,19 @@ template:
   path: network.yaml.j2
   type: file
 
-parameters:
-  LoadBalancerSecurityGroupName: "{{ base_name }}-{{ environment }}-load-balancer-security-group"
-  LoadBalancerName: "{{ base_name }}-{{ environment }}-load-balancer"
-  TargetGroupName: "{{ base_name }}-{{ environment }}-target-group"
+sceptre_user_data:
+  load_balancer_security_group_sources:
+    - !ssm /d2d/dev/ucop-vpn-prefix-list-id
+    - !ssm /d2d/dev/cdl-eip-prefix-list-id
 
+parameters:
   VPCID: !stack_output_external cdl-d2d-dev-vpc-stack::vpc
   SubnetIDs:
     - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2a
     - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2b
     - !stack_output_external cdl-d2d-dev-defaultsubnet-stack::defaultsubnet2c
   LoadBalancerSecurityGroupSources: !stack_attr sceptre_user_data.load_balancer_security_group_sources
-
-  Route53ZoneID: !ssm /d2d/dev/route53-hosted-zone-id
-  Domain: d2d-zephir-api-dev.d2ddev.cdlib.net
-  ApplicationPort: "8000"
 
-sceptre_user_data:
-  load_balancer_security_group_sources:
-    - !ssm /d2d/dev/ucop-vpn-prefix-list-id
-    - !ssm /d2d/dev/cdl-eip-prefix-list-id
+  Route53ZoneID: !ssm /d2d/dev/route53-hosted-zone-id
+  Domain: "{{ sceptre_user_data.url_authority }}"
+  ApplicationPort: "{{ sceptre_user_data.port }}"

deployment/config/dev/waf.yaml

@@ -1,11 +1,9 @@
 template:
-  path: waf.yaml
+  path: waf.yaml.j2
   type: file
 
 dependencies:
-  - dev/network.yaml
+  - "{{ sceptre_user_data.environment }}/network.yaml"
 
 parameters:
-  WebACLName: "{{ base_name }}-{{ environment }}-waf-webacl"
-  WebACLMetricName: "{{ base_name }}-{{ environment }}-waf-webacl-metric"
-  LoadBalancerArn: !stack_output dev/network.yaml::LoadBalancerARN
+  LoadBalancerArn: !stack_output "{{ sceptre_user_data.environment }}/network.yaml::LoadBalancerARN"