|
| 1 | +name: Upload wheels to R2 |
| 2 | +run-name: Upload wheels from ${{ github.event.workflow_run.name }} (run ${{ github.event.workflow_run.id }}) |
| 3 | + |
| 4 | +# Runs from main whenever a nightly wheel build finishes, picks up the wheels that |
| 5 | +# build saved as GitHub artifacts and uploads them to R2. The upload step lives |
| 6 | +# here, not in the build workflows, so a pull request cannot change it, and the R2 |
| 7 | +# credentials come from the pytorchbot-env environment. |
| 8 | + |
| 9 | +on: |
| 10 | + workflow_run: |
| 11 | + workflows: |
| 12 | + - Build Linux Wheels |
| 13 | + - Build Aarch64 Linux Wheels |
| 14 | + - Build Linux CUDA Wheels |
| 15 | + - Build Aarch64 Linux CUDA Wheels |
| 16 | + - Build Windows Wheels |
| 17 | + - Build Windows CUDA Wheels |
| 18 | + - Build macOS Wheels |
| 19 | + types: |
| 20 | + - completed |
| 21 | + |
| 22 | +permissions: {} |
| 23 | + |
| 24 | +concurrency: |
| 25 | + group: ${{ github.workflow }}-${{ github.event.workflow_run.id }} |
| 26 | + cancel-in-progress: false |
| 27 | + |
| 28 | +jobs: |
| 29 | + get-matrix: |
| 30 | + if: ${{ github.repository == 'pytorch/executorch' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'nightly' && github.event.workflow_run.head_repository.full_name == github.repository }} |
| 31 | + runs-on: ubuntu-22.04 |
| 32 | + permissions: |
| 33 | + actions: read |
| 34 | + outputs: |
| 35 | + matrix: ${{ steps.matrix.outputs.matrix }} |
| 36 | + architecture: ${{ steps.matrix.outputs.architecture }} |
| 37 | + steps: |
| 38 | + - name: Link the build run |
| 39 | + shell: bash |
| 40 | + env: |
| 41 | + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} |
| 42 | + RUN_URL: ${{ github.event.workflow_run.html_url }} |
| 43 | + RUN_BRANCH: ${{ github.event.workflow_run.head_branch }} |
| 44 | + run: echo "Uploading the wheels of [${WORKFLOW_NAME} (${RUN_BRANCH})](${RUN_URL})" >> "${GITHUB_STEP_SUMMARY}" |
| 45 | + |
| 46 | + - name: Download the build matrix |
| 47 | + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 |
| 48 | + with: |
| 49 | + name: build-matrix |
| 50 | + run-id: ${{ github.event.workflow_run.id }} |
| 51 | + github-token: ${{ secrets.GITHUB_TOKEN }} |
| 52 | + |
| 53 | + - name: Read the build matrix |
| 54 | + id: matrix |
| 55 | + shell: bash |
| 56 | + env: |
| 57 | + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} |
| 58 | + run: | |
| 59 | + set -euo pipefail |
| 60 | + echo "matrix=$(jq -c . build-matrix.json)" >> "${GITHUB_OUTPUT}" |
| 61 | + # The architecture in the artifact name, as the build workflows pass it |
| 62 | + # to the upload (the macOS one passes none) |
| 63 | + case "${WORKFLOW_NAME}" in |
| 64 | + "Build Aarch64 Linux Wheels" | "Build Aarch64 Linux CUDA Wheels") echo "architecture=aarch64" >> "${GITHUB_OUTPUT}" ;; |
| 65 | + "Build Windows Wheels" | "Build Windows CUDA Wheels") echo "architecture=x64" >> "${GITHUB_OUTPUT}" ;; |
| 66 | + "Build macOS Wheels") echo "architecture=" >> "${GITHUB_OUTPUT}" ;; |
| 67 | + *) echo "architecture=x86_64" >> "${GITHUB_OUTPUT}" ;; |
| 68 | + esac |
| 69 | +
|
| 70 | + upload-r2: |
| 71 | + needs: get-matrix |
| 72 | + runs-on: ubuntu-22.04 |
| 73 | + environment: pytorchbot-env |
| 74 | + timeout-minutes: 30 |
| 75 | + permissions: |
| 76 | + actions: read |
| 77 | + contents: read |
| 78 | + strategy: |
| 79 | + fail-fast: false |
| 80 | + matrix: ${{ fromJSON(needs.get-matrix.outputs.matrix) }} |
| 81 | + name: upload-r2-${{ matrix.build_name }} |
| 82 | + steps: |
| 83 | + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 |
| 84 | + with: |
| 85 | + repository: pytorch/test-infra |
| 86 | + ref: main |
| 87 | + path: test-infra |
| 88 | + |
| 89 | + - uses: ./test-infra/.github/actions/set-channel |
| 90 | + |
| 91 | + - uses: ./test-infra/.github/actions/setup-binary-upload |
| 92 | + with: |
| 93 | + repository: pytorch/executorch |
| 94 | + ref: "" |
| 95 | + python-version: ${{ matrix.python_version }} |
| 96 | + cuda-version: ${{ matrix.desired_cuda }} |
| 97 | + arch: ${{ needs.get-matrix.outputs.architecture }} |
| 98 | + upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }} |
| 99 | + |
| 100 | + - name: Download the wheel |
| 101 | + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 |
| 102 | + with: |
| 103 | + name: ${{ env.ARTIFACT_NAME }} |
| 104 | + path: pytorch/executorch/dist/ |
| 105 | + run-id: ${{ github.event.workflow_run.id }} |
| 106 | + github-token: ${{ secrets.GITHUB_TOKEN }} |
| 107 | + |
| 108 | + - name: Upload package to R2 |
| 109 | + shell: bash |
| 110 | + working-directory: pytorch/executorch |
| 111 | + env: |
| 112 | + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} |
| 113 | + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} |
| 114 | + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} |
| 115 | + AWS_DEFAULT_REGION: auto |
| 116 | + run: | |
| 117 | + set -ex |
| 118 | + if [[ -z "${R2_ACCOUNT_ID}" || -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" ]]; then |
| 119 | + echo "::error::R2 secrets are missing from the pytorchbot-env environment" |
| 120 | + exit 1 |
| 121 | + fi |
| 122 | +
|
| 123 | + # shellcheck disable=SC1090 |
| 124 | + source "${BUILD_ENV_FILE}" |
| 125 | +
|
| 126 | + pip install awscli==1.32.18 |
| 127 | +
|
| 128 | + PYTORCH_S3_BUCKET_PATH=${PYTORCH_S3_BUCKET_PATH/"s3://pytorch"/"s3://pytorch-downloads"} |
| 129 | + for pkg in dist/*; do |
| 130 | + shm_id=$(sha256sum "${pkg}" | awk '{print $1}') |
| 131 | + aws s3 cp "$pkg" "${PYTORCH_S3_BUCKET_PATH}" \ |
| 132 | + --metadata "checksum-sha256=${shm_id}" --endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" |
| 133 | + done |
0 commit comments