Skip to content

Switch release to GHCR with multi-arch builds #2195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JannikSt wants to merge 1 commit into
base: main
Choose a base branch
from
+151 −41
Draft

Switch release to GHCR with multi-arch builds #2195

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 151 additions & 41 deletions .github/workflows/release.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,61 +5,171 @@ on:
branches: [main]
tags:
- "*"
workflow_dispatch:
inputs:
platforms:
description: 'Target platforms to build'
required: true
type: choice
options:
- amd64-only
- arm64-only
- both
default: 'both'

env:
REGISTRY: ghcr.io
IMAGE_NAME: primeintellect-ai/prime-rl

jobs:
cuda:
name: Push CUDA image to Docker
runs-on: image-builder
prepare:
name: Prepare build matrix
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
version: ${{ steps.vars.outputs.version }}
full_sha: ${{ steps.vars.outputs.full_sha }}
platforms: ${{ steps.resolve-platforms.outputs.platforms }}
steps:
- name: Remove unnecessary packages
- name: Resolve platforms
id: resolve-platforms
run: |
echo "=== Before pruning ==="
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
echo "=== After pruning ==="
df -h
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "platforms=${{ inputs.platforms }}" >> $GITHUB_OUTPUT
else
echo "platforms=both" >> $GITHUB_OUTPUT
fi

# Link to discussion: https://github.com/orgs/community/discussions/25678
- name: Set build matrix
id: set-matrix
run: |
PLATFORMS="${{ steps.resolve-platforms.outputs.platforms }}"
if [ "$PLATFORMS" = "arm64-only" ]; then
echo 'matrix={"include":[{"runner":"image-builder-arm-2204","platform":"linux/arm64","suffix":"arm64"}]}' >> $GITHUB_OUTPUT
elif [ "$PLATFORMS" = "both" ]; then
echo 'matrix={"include":[{"runner":"image-builder","platform":"linux/amd64","suffix":"amd64"},{"runner":"image-builder-arm-2204","platform":"linux/arm64","suffix":"arm64"}]}' >> $GITHUB_OUTPUT
else
echo 'matrix={"include":[{"runner":"image-builder","platform":"linux/amd64","suffix":"amd64"}]}' >> $GITHUB_OUTPUT
fi

- name: Checkout
uses: actions/checkout@v3
- name: Docker meta
id: meta
uses: crazy-max/ghaction-docker-meta@v2
uses: actions/checkout@v4
with:
images: |
primeintellect/prime-rl
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha,prefix=commit-
type=raw,value=latest,enable=${{ github.ref_type == 'tag' }}
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
fetch-depth: 0

- name: Get version info
id: vars
run: |
FULL_SHA=$(git rev-parse HEAD)
echo "full_sha=${FULL_SHA}" >> $GITHUB_OUTPUT

if VERSION=$(git describe --tags --exact-match 2>/dev/null); then
echo "version=${VERSION}" >> $GITHUB_OUTPUT
else
BRANCH_NAME=$(echo "${GITHUB_REF_NAME}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' | cut -c1-50)
BASE_VERSION=$(git describe --tags --always)
echo "version=${BASE_VERSION}-${BRANCH_NAME}" >> $GITHUB_OUTPUT
fi

build:
name: Build (${{ matrix.suffix }})
needs: prepare
strategy:
fail-fast: true
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
runs-on: ${{ matrix.runner }}
permissions:
contents: read
packages: write

steps:
- name: Remove unnecessary packages
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc 2>/dev/null || true
df -h

- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
- name: Checkout
uses: actions/checkout@v4

- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.cuda
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
push: true
platforms: ${{ matrix.platform }}
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.version }}-${{ matrix.suffix }}
labels: |
org.opencontainers.image.revision=${{ needs.prepare.outputs.full_sha }}
org.opencontainers.image.version=${{ needs.prepare.outputs.version }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}

- name: Output image info
run: |
echo "## Build Summary (${{ matrix.suffix }})" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
echo "| **Platform** | ${{ matrix.platform }} |" >> $GITHUB_STEP_SUMMARY
echo "| **Image** | \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.version }}-${{ matrix.suffix }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| **Digest** | \`${{ steps.docker_build.outputs.digest }}\` |" >> $GITHUB_STEP_SUMMARY

manifest:
name: Create multi-arch manifest
needs: [prepare, build]
runs-on: ubuntu-latest
permissions:
packages: write

- name: Test import
run: docker run --entrypoint python primeintellect/prime-rl:main -c "import prime_rl"
steps:
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Create and push multi-arch manifest
run: |
VERSION="${{ needs.prepare.outputs.version }}"
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
PLATFORMS="${{ needs.prepare.outputs.platforms }}"

- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
SOURCES=""
if [ "$PLATFORMS" != "arm64-only" ]; then
SOURCES="$SOURCES ${IMAGE}:${VERSION}-amd64"
fi
if [ "$PLATFORMS" != "amd64-only" ]; then
SOURCES="$SOURCES ${IMAGE}:${VERSION}-arm64"
fi

# Create version tag + latest tag for tag events
TAGS="-t ${IMAGE}:${VERSION}"
if [ "${{ github.ref_type }}" = "tag" ]; then
TAGS="$TAGS -t ${IMAGE}:latest"
fi

docker buildx imagetools create $TAGS $SOURCES

- name: Output manifest info
run: |
VERSION="${{ needs.prepare.outputs.version }}"
echo "## Manifest Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
echo "| **Version** | \`${VERSION}\` |" >> $GITHUB_STEP_SUMMARY
echo "| **Platforms** | ${{ needs.prepare.outputs.platforms }} |" >> $GITHUB_STEP_SUMMARY
echo "| **Image** | \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${VERSION}\` |" >> $GITHUB_STEP_SUMMARY
Loading