Skip to content

Commit

Permalink
ci: switch to using circle-ci for build/test/lint
Browse files Browse the repository at this point in the history
Switch to using CircleCi for the build / test / lint pipeline because
they provide faster runners for opensource than github does.
Reducing our ci runtimes from 25min to only 11min. Also allowing us to
perform more cache hits.
  • Loading branch information
TroyKomodo committed Jul 20, 2023
1 parent 2d339f7 commit 35068f0
Show file tree
Hide file tree
Showing 5 changed files with 401 additions and 520 deletions.
399 changes: 399 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,399 @@
version: 2.1

orbs:
trivy: signavio/[email protected]
docker: circleci/[email protected]
terraform: circleci/[email protected]
codecov: codecov/[email protected]

commands:
setup-docker:
steps:
- setup_remote_docker

- run:
name: Setup docker buildx
command: |
set -eux
docker context create buildx-build
docker buildx create --use buildx-build
build-scan-docker:
steps:
- run:
name: Build API Image
command: docker build -f ./docker/api.Dockerfile -t ghcr.io/scuffletv/api:$(git rev-parse HEAD) .

- run:
name: Install Dependencies
command: apk add --no-cache curl jq wget tar gzip ca-certificates

- trivy/vulnerability-scan-report:
docker-image: ghcr.io/scuffletv/api:$(git rev-parse HEAD)

- run:
name: Build Edge Image
command: docker build -f ./docker/edge.Dockerfile -t ghcr.io/scuffletv/edge:$(git rev-parse HEAD) .

- trivy/vulnerability-scan-report:
docker-image: ghcr.io/scuffletv/edge:$(git rev-parse HEAD)

- run:
name: Build Ingest Image
command: docker build -f ./docker/ingest.Dockerfile -t ghcr.io/scuffletv/ingest:$(git rev-parse HEAD) .

- trivy/vulnerability-scan-report:
docker-image: ghcr.io/scuffletv/ingest:$(git rev-parse HEAD)

- run:
name: Build Transcoder Image
command: docker build -f ./docker/transcoder.Dockerfile -t ghcr.io/scuffletv/transcoder:$(git rev-parse HEAD) .

- trivy/vulnerability-scan-report:
docker-image: ghcr.io/scuffletv/transcoder:$(git rev-parse HEAD)
severity: CRITICAL,HIGH

- run:
name: Build Website Image
command: docker build -f ./docker/website.Dockerfile -t ghcr.io/scuffletv/website:$(git rev-parse HEAD) .

- trivy/vulnerability-scan-report:
docker-image: ghcr.io/scuffletv/website:$(git rev-parse HEAD)

jobs:
lint-test:
resource_class: large

docker:
- image: ghcr.io/scuffletv/build:latest
environment:
DATABASE_URL: postgres://root@localhost:26257/scuffle
RMQ_URL: amqp://rabbitmq:rabbitmq@localhost:5672/scuffle
REDIS_URL: redis://localhost:6379
CARGO_INCREMENTAL: 1

- image: ghcr.io/scuffletv/cockroach:latest

- image: bitnami/rabbitmq:latest
environment:
RABBITMQ_USERNAME: rabbitmq
RABBITMQ_PASSWORD: rabbitmq
RABBITMQ_VHOSTS: scuffle
RABBITMQ_DISK_FREE_ABSOLUTE_LIMIT: 50MB

- image: redis:latest

steps:
- docker/install-dockerize

- terraform/install

- checkout

- run:
name: Combine Cargo.lock Files
command: |
sha256sum Cargo.lock >> locksum
sha256sum frontend/player/Cargo.lock >> locksum
- restore_cache:
name: Restore Rust Lint/Test Cache
keys:
- &lint-test-cache lint-test-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "locksum" }}
- lint-test-cache-{{ .Environment.CACHE_VERSION }}-

- run:
name: Install dependencies
command: mask bootstrap --no-db --no-docker --no-env --no-rust

- run:
name: Wait for services to start
command: |
dockerize -wait tcp://localhost:26257 -timeout 15s
dockerize -wait tcp://localhost:5672 -timeout 15s
dockerize -wait tcp://localhost:6379 -timeout 15s
- run:
name: Migrate Database
command: mask db migrate

- run:
name: Cargo Sweep Start
command: |
cargo sweep -s
cargo sweep -s frontend/player
- run:
name: Run Lint
command: mask lint

- run:
name: Run Rust Tests
command: |
cargo llvm-cov nextest \
--workspace \
--lcov \
--output-path lcov.info \
--ignore-filename-regex "(main\.rs|tests|.*\.nocov\.rs)" \
--no-fail-fast \
-E "not test(_v6)" \
--status-level all \
--profile ci \
--tests \
--config "profile.dev.debug = 0"
- codecov/upload:
file: lcov.info

- run:
name: Install Playwright Dependencies
command: pnpm --filter website exec playwright install-deps

- run:
name: Run JavaScript Tests
command: mask test --no-rust

- run:
name: Cargo Sweep Finish
command: |
cargo sweep -f
cargo sweep -f frontend/player
- save_cache:
name: Save Test Cache
key: *lint-test-cache
paths:
- target
- frontend/player/target

build:
resource_class: large

docker:
- image: ghcr.io/scuffletv/build:latest
environment:
CARGO_INCREMENTAL: 1

steps:
- checkout

- run:
name: Install dependencies
command: mask bootstrap --no-db --no-docker --no-env --no-js-tests --no-rust

- run:
name: Combine Cargo.lock Files
command: |
sha256sum Cargo.lock >> locksum
sha256sum frontend/player/Cargo.lock >> locksum
- restore_cache:
name: Restore Rust Build Cache
keys:
- &build-cache build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "locksum" }}
- build-cache-{{ .Environment.CACHE_VERSION }}-

- run:
name: Cargo Sweep Start
command: |
cargo sweep -s
cargo sweep -s frontend/player
- run:
name: Build Rust
command: mask build rust

- run:
name: Build JavaScript
command: mask build website --no-gql-prepare

- run:
name: Cargo Sweep Finish
command: |
cargo sweep -f
cargo sweep -f frontend/player
- save_cache:
name: Save Build Cache
key: *build-cache
paths:
- target
- frontend/player/target

- store_artifacts:
path: target/x86_64-unknown-linux-gnu/release/api
destination: api

- store_artifacts:
path: target/x86_64-unknown-linux-gnu/release/edge
destination: edge

- store_artifacts:
path: target/x86_64-unknown-linux-gnu/release/ingest
destination: ingest

- store_artifacts:
path: target/x86_64-unknown-linux-gnu/release/transcoder
destination: transcoder

- run:
name: Compress Website Build
command: tar -czf website.tar.gz -C frontend/website build --transform s/build/website/

- store_artifacts:
path: website.tar.gz

- run:
name: Compress Player Build
command: tar -czf player.tar.gz -C frontend/player dist --transform s/dist/player/

- store_artifacts:
path: player.tar.gz

- run:
name: Compress Player Package
command: tar -czf player-pkg.tar.gz -C frontend/player pkg --transform s/pkg/player-pkg/

- store_artifacts:
path: player-pkg.tar.gz

- run:
name: Compress Player Demo
command: tar -czf player-demo.tar.gz -C frontend/player demo-dist --transform s/demo-dist/player-demo/

- store_artifacts:
path: player-demo.tar.gz

- persist_to_workspace:
root: .
paths:
- target/x86_64-unknown-linux-gnu/release/api
- target/x86_64-unknown-linux-gnu/release/edge
- target/x86_64-unknown-linux-gnu/release/ingest
- target/x86_64-unknown-linux-gnu/release/transcoder
- frontend/website/build

docker:
resource_class: medium

docker:
- image: docker:24.0.4-cli-alpine3.18

steps:
- run:
name: Install Git
command: apk add --no-cache git

- checkout

- setup-docker

- attach_workspace:
at: .

- build-scan-docker

docker-push:
resource_class: medium

docker:
- image: docker:24.0.4-cli-alpine3.18

steps:
- run:
name: Install Git
command: apk add --no-cache git bash

- checkout

- setup-docker

- attach_workspace:
at: .

- build-scan-docker

- run:
name: Login to GitHub Container Registry
command: echo $SCUFFLE_GHCR_TOKEN | docker login ghcr.io -u scuffletv --password-stdin

- run:
name: Push Images
shell: bash -eo pipefail
command: |
if [[ -n $CIRCLE_PR_USERNAME ]]; then
TAG="$CIRCLE_PR_USERNAME-$CIRCLE_PR_NUMBER"
elif [[ $CIRCLE_BRANCH == "main" ]]; then
TAG="latest"
else
TAG="branch-$CIRCLE_BRANCH"
fi
# Convert to lowercase & replace all invalid characters
TAG=$(echo $TAG | tr '[:upper:]' '[:lower:]')
TAG=$(echo $TAG | tr '/' '-')
TAG=$(echo $TAG | tr -cd '[[:alnum:]]._-')
echo "Tagging images with $TAG"
docker tag ghcr.io/scuffletv/api:$(git rev-parse HEAD) ghcr.io/scuffletv/api:$TAG
docker tag ghcr.io/scuffletv/edge:$(git rev-parse HEAD) ghcr.io/scuffletv/edge:$TAG
docker tag ghcr.io/scuffletv/ingest:$(git rev-parse HEAD) ghcr.io/scuffletv/ingest:$TAG
docker tag ghcr.io/scuffletv/transcoder:$(git rev-parse HEAD) ghcr.io/scuffletv/transcoder:$TAG
docker tag ghcr.io/scuffletv/website:$(git rev-parse HEAD) ghcr.io/scuffletv/website:$TAG
docker push ghcr.io/scuffletv/api:$(git rev-parse HEAD)
docker push ghcr.io/scuffletv/api:$TAG
docker push ghcr.io/scuffletv/edge:$(git rev-parse HEAD)
docker push ghcr.io/scuffletv/edge:$TAG
docker push ghcr.io/scuffletv/ingest:$(git rev-parse HEAD)
docker push ghcr.io/scuffletv/ingest:$TAG
docker push ghcr.io/scuffletv/transcoder:$(git rev-parse HEAD)
docker push ghcr.io/scuffletv/transcoder:$TAG
docker push ghcr.io/scuffletv/website:$(git rev-parse HEAD)
docker push ghcr.io/scuffletv/website:$TAG
workflows:
lint-test-build:
jobs:
- hold:
type: approval

- lint-test:
requires:
- hold

- build:
requires:
- lint-test

- docker:
requires:
- build
filters:
branches:
ignore:
- main
- /^feature\/.*$/

- docker-push-approve:
type: approval
requires:
- build

- docker-push:
requires:
- docker-push-approve
filters:
branches:
only:
- main
- /^feature\/.*$/

context:
- ghcr
Loading

0 comments on commit 35068f0

Please sign in to comment.