feat(bench): resource-monitor integration for benchmark orchestration (#159) #132
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist | |
| # | |
| # Copyright 2022-2024, axodotdev | |
| # SPDX-License-Identifier: MIT or Apache-2.0 | |
| # | |
| # CI that: | |
| # | |
| # * checks for a Git Tag that looks like a release | |
| # * builds artifacts with dist (archives, installers, hashes) | |
| # * uploads those artifacts to temporary workflow zip | |
| # * on success, uploads the artifacts to a GitHub Release | |
| # | |
| # Note that the GitHub Release will be created with a generated | |
| # title/body based on your changelogs. | |
| name: cargo dist | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ inputs.tag || github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| "contents": "read" | |
| # This task will run whenever you workflow_dispatch with a tag that looks like a version | |
| # like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION must be a Cargo-style SemVer Version (must have at least major.minor.patch). | |
| # | |
| # If PACKAGE_NAME is specified, then the announcement will be for that | |
| # package (erroring out if it doesn't have the given version or isn't dist-able). | |
| # | |
| # If PACKAGE_NAME isn't specified, then the announcement will be for all | |
| # (dist-able) packages in the workspace with that version (this mode is | |
| # intended for workspaces with only one dist-able package, or with all dist-able | |
| # packages versioned/released in lockstep). | |
| # | |
| # If you push multiple tags at once, separate instances of this workflow will | |
| # spin up, creating an independent announcement for each one. However, GitHub | |
| # will hard limit this to 3 tags per commit, as it will assume more tags is a | |
| # mistake. | |
| # | |
| # If there's a prerelease-style suffix to the version, then the release(s) | |
| # will be marked as a prerelease. | |
| on: | |
| pull_request: | |
| paths: | |
| - "crates/**" | |
| - "bindings/python/**" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "dist-workspace.toml" | |
| - ".github/workflows/cargo-dist-release.yml" | |
| - ".github/workflows/release-please.yaml" | |
| workflow_call: | |
| inputs: | |
| tag: | |
| description: Release Tag | |
| required: true | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: Release Tag | |
| required: true | |
| default: dry-run | |
| type: string | |
| jobs: | |
| # Run 'dist plan' (or host) to determine what tasks we need to do | |
| plan: | |
| name: Plan release artifacts | |
| runs-on: "ubuntu-22.04" | |
| timeout-minutes: 15 | |
| outputs: | |
| val: ${{ steps.plan.outputs.manifest }} | |
| tag: ${{ (inputs.tag != 'dry-run' && inputs.tag) || '' }} | |
| tag-flag: ${{ inputs.tag && inputs.tag != 'dry-run' && format('--tag={0}', | |
| inputs.tag) || '' }} | |
| publishing: ${{ inputs.tag && inputs.tag != 'dry-run' }} | |
| permissions: | |
| "contents": "write" # Required because `dist host --steps=create` may create or update release metadata. | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: Install Rust | |
| run: rustup update "1.85" --no-self-update && rustup default "1.85" | |
| - name: Install dist | |
| # we specify bash to get pipefail; it guards against the `curl` command | |
| # failing. otherwise `sh` won't catch that `curl` returned non-0 | |
| shell: bash | |
| run: "curl --proto '=https' --tlsv1.2 -LsSf | |
| https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | |
| | sh" | |
| - name: Cache dist | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: cargo-dist-cache | |
| path: ~/.cargo/bin/dist | |
| # sure would be cool if github gave us proper conditionals... | |
| # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible | |
| # functionality based on whether this is a pull_request, and whether it's from a fork. | |
| # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* | |
| # but also really annoying to build CI around when it needs secrets to work right.) | |
| - id: plan | |
| env: | |
| RELEASE_TAG: ${{ inputs.tag }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${RELEASE_TAG}" && "${RELEASE_TAG}" != "dry-run" ]]; then | |
| if [[ ! "${RELEASE_TAG}" =~ ^[A-Za-z0-9._/-]+$ ]]; then | |
| echo "::error::Invalid release tag '${RELEASE_TAG}'" | |
| exit 1 | |
| fi | |
| dist host --allow-dirty --steps=create --tag="${RELEASE_TAG}" --output-format=json > plan-dist-manifest.json | |
| else | |
| dist plan --allow-dirty --output-format=json > plan-dist-manifest.json | |
| fi | |
| echo "dist ran successfully" | |
| cat plan-dist-manifest.json | |
| echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" | |
| - name: "Upload dist-manifest.json" | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: artifacts-plan-dist-manifest | |
| path: plan-dist-manifest.json | |
| # Build and packages all the platform-specific things | |
| build-local-artifacts: | |
| name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) | |
| timeout-minutes: 45 | |
| # Let the initial task tell us to not run (currently very blunt) | |
| needs: | |
| - plan | |
| if: | |
| ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != | |
| null && (needs.plan.outputs.publishing == 'true' || | |
| fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') || | |
| inputs.tag == 'dry-run' }} | |
| strategy: | |
| fail-fast: false | |
| # Target platforms/runners are computed by dist in create-release. | |
| # Each member of the matrix has the following arguments: | |
| # | |
| # - runner: the github runner | |
| # - dist-args: cli flags to pass to dist | |
| # - install-dist: expression to run to install dist on the runner | |
| # | |
| # Typically there will be: | |
| # - 1 "global" task that builds universal installers | |
| # - N "local" tasks that build each platform's binaries and platform-specific installers | |
| matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} | |
| runs-on: ${{ matrix.runner }} | |
| container: ${{ matrix.container && matrix.container.image || null }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json | |
| steps: | |
| - name: enable windows longpaths | |
| run: | | |
| git config --global core.longpaths true | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: Install Rust non-interactively if not already installed | |
| if: ${{ matrix.container }} | |
| run: | | |
| if ! command -v cargo > /dev/null 2>&1; then | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| fi | |
| - name: Use rustup to set correct Rust version | |
| run: rustup update "1.85" --no-self-update && rustup default "1.85" | |
| - uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| key: ${{ join(matrix.targets, '-') }} | |
| cache-provider: ${{ matrix.cache_provider }} | |
| - name: Install dist | |
| shell: bash | |
| env: | |
| MATRIX_INSTALL_DIST_RUN: ${{ matrix.install_dist.run }} | |
| MATRIX_INSTALL_DIST_SHELL: ${{ matrix.install_dist.shell }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "${MATRIX_INSTALL_DIST_SHELL}" == "pwsh" ]]; then | |
| pwsh -NoLogo -NoProfile -Command "${MATRIX_INSTALL_DIST_RUN}" | |
| else | |
| sh -c "${MATRIX_INSTALL_DIST_RUN}" | |
| fi | |
| # Get the dist-manifest | |
| - name: Fetch local artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: artifacts-* | |
| path: target/distrib/ | |
| merge-multiple: true | |
| - name: Install dependencies | |
| shell: bash | |
| env: | |
| MATRIX_PACKAGES_INSTALL: ${{ matrix.packages_install }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${MATRIX_PACKAGES_INSTALL}" ]]; then | |
| bash -euo pipefail -c "${MATRIX_PACKAGES_INSTALL}" | |
| fi | |
| # MODIFIED: musl-tools only ships a C wrapper (musl-gcc); highs-sys needs | |
| # a C++ compiler that links against musl's libstdc++, not glibc's. | |
| # For container-based builds (messense/rust-musl-cross), the toolchain is pre-installed. | |
| # For non-container builds, we download from musl.cc (may timeout - consider using container). | |
| # Re-apply after running `cargo dist init`. | |
| - name: Install musl cross-toolchain | |
| if: ${{ contains(join(matrix.targets, ','), 'musl') && !matrix.container }} | |
| shell: bash | |
| run: | | |
| curl -fsSL https://musl.cc/x86_64-linux-musl-cross.tgz | sudo tar xz -C /opt | |
| sudo ln -sf /opt/x86_64-linux-musl-cross/bin/x86_64-linux-musl-g++ /usr/local/bin/x86_64-linux-musl-g++ | |
| sudo ln -sf /opt/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc /usr/local/bin/x86_64-linux-musl-gcc | |
| # MODIFIED: Point cc/cargo at the musl cross-toolchain so that C/C++ | |
| # code (highs-sys) compiles and links against musl, not glibc. | |
| # For container builds, the x86_64-unknown-linux-musl-gcc/g++ are in PATH. | |
| # For non-container builds, we use the /usr/local/bin symlinks. | |
| # Re-apply after running `cargo dist init`. | |
| - name: Configure musl cross-compilation env | |
| if: ${{ contains(join(matrix.targets, ','), 'musl') }} | |
| shell: bash | |
| env: | |
| MATRIX_CONTAINER_IMAGE: ${{ matrix.container && matrix.container.image || '' }} | |
| run: | | |
| if [[ -n "${MATRIX_CONTAINER_IMAGE}" ]]; then | |
| # Container has musl toolchain in PATH | |
| { | |
| echo "CC_x86_64_unknown_linux_musl=x86_64-unknown-linux-musl-gcc" | |
| echo "CXX_x86_64_unknown_linux_musl=x86_64-unknown-linux-musl-g++" | |
| echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-unknown-linux-musl-gcc" | |
| } >> "$GITHUB_ENV" | |
| else | |
| # Non-container: use the downloaded toolchain | |
| { | |
| echo "CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc" | |
| echo "CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++" | |
| echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-linux-musl-gcc" | |
| } >> "$GITHUB_ENV" | |
| fi | |
| - name: Build artifacts | |
| shell: bash | |
| env: | |
| PLAN_TAG_FLAG: ${{ needs.plan.outputs.tag-flag }} | |
| DIST_ARGS: ${{ matrix.dist_args }} | |
| run: | | |
| set -euo pipefail | |
| read -r -a dist_args_array <<< "${DIST_ARGS}" | |
| # Actually do builds and make zips and whatnot | |
| # shellcheck disable=SC2086 | |
| dist build --allow-dirty ${PLAN_TAG_FLAG} --print=linkage --output-format=json "${dist_args_array[@]}" > dist-manifest.json | |
| echo "dist ran successfully" | |
| - id: cargo-dist | |
| name: Post-build | |
| # We force bash here just because github makes it really hard to get values up | |
| # to "real" actions without writing to env-vars, and writing to env-vars has | |
| # inconsistent syntax between shell and powershell. | |
| shell: bash | |
| run: | | |
| # Parse out what we just built and upload it to scratch storage | |
| { | |
| echo "paths<<EOF" | |
| dist print-upload-files-from-manifest --allow-dirty --manifest dist-manifest.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| cp dist-manifest.json "$BUILD_MANIFEST_NAME" | |
| - name: "Upload artifacts" | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: artifacts-build-local-${{ join(matrix.targets, '_') }} | |
| path: | | |
| ${{ steps.cargo-dist.outputs.paths }} | |
| ${{ env.BUILD_MANIFEST_NAME }} | |
| # Build and package all the platform-agnostic(ish) things | |
| build-global-artifacts: | |
| name: Build global artifacts | |
| needs: | |
| - plan | |
| - build-local-artifacts | |
| runs-on: "ubuntu-22.04" | |
| timeout-minutes: 20 | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: Install Rust | |
| run: rustup update "1.85" --no-self-update && rustup default "1.85" | |
| - name: Install cached dist | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: cargo-dist-cache | |
| path: ~/.cargo/bin/ | |
| - run: chmod +x ~/.cargo/bin/dist | |
| # Get all the local artifacts for the global tasks to use (for e.g. checksums) | |
| - name: Fetch local artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: artifacts-* | |
| path: target/distrib/ | |
| merge-multiple: true | |
| - id: cargo-dist | |
| shell: bash | |
| run: | | |
| # shellcheck disable=SC2086 | |
| dist build --allow-dirty ${NEEDS_PLAN_OUTPUTS_TAG_FLAG} --output-format=json "--artifacts=global" > dist-manifest.json | |
| echo "dist ran successfully" | |
| # Parse out what we just built and upload it to scratch storage | |
| { | |
| echo "paths<<EOF" | |
| jq --raw-output ".upload_files[]" dist-manifest.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| cp dist-manifest.json "$BUILD_MANIFEST_NAME" | |
| env: | |
| NEEDS_PLAN_OUTPUTS_TAG_FLAG: ${{ needs.plan.outputs.tag-flag }} | |
| - name: "Upload artifacts" | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: artifacts-build-global | |
| path: | | |
| ${{ steps.cargo-dist.outputs.paths }} | |
| ${{ env.BUILD_MANIFEST_NAME }} | |
| # Determines if we should publish/announce | |
| host: | |
| name: Publish release artifacts | |
| timeout-minutes: 15 | |
| needs: | |
| - plan | |
| - build-local-artifacts | |
| - build-global-artifacts | |
| # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) | |
| if: ${{ always() && needs.plan.result == 'success' && | |
| needs.plan.outputs.publishing == 'true' && | |
| (needs.build-global-artifacts.result == 'skipped' || | |
| needs.build-global-artifacts.result == 'success') && | |
| (needs.build-local-artifacts.result == 'skipped' || | |
| needs.build-local-artifacts.result == 'success') }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| "contents": "write" # Required to upload artifacts to the GitHub Release and publish it. | |
| runs-on: "ubuntu-22.04" | |
| outputs: | |
| val: ${{ steps.host.outputs.manifest }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: Install Rust | |
| run: rustup update "1.85" --no-self-update && rustup default "1.85" | |
| - name: Install cached dist | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: cargo-dist-cache | |
| path: ~/.cargo/bin/ | |
| - run: chmod +x ~/.cargo/bin/dist | |
| # Fetch artifacts from scratch-storage | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: artifacts-* | |
| path: target/distrib/ | |
| merge-multiple: true | |
| - id: host | |
| shell: bash | |
| run: | | |
| # shellcheck disable=SC2086 | |
| dist host --allow-dirty ${NEEDS_PLAN_OUTPUTS_TAG_FLAG} --steps=upload --steps=release --output-format=json > dist-manifest.json | |
| echo "artifacts uploaded and released successfully" | |
| cat dist-manifest.json | |
| echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" | |
| env: | |
| NEEDS_PLAN_OUTPUTS_TAG_FLAG: ${{ needs.plan.outputs.tag-flag }} | |
| - name: "Upload dist-manifest.json" | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| # Overwrite the previous copy | |
| name: artifacts-dist-manifest | |
| path: dist-manifest.json | |
| # Create a GitHub Release while uploading all files to it | |
| - name: "Download GitHub Artifacts" | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: artifacts-* | |
| path: artifacts | |
| merge-multiple: true | |
| - name: Cleanup | |
| run: | | |
| # Remove the granular manifests | |
| rm -f artifacts/*-dist-manifest.json | |
| # MODIFIED: upload to existing draft release created by release-please | |
| # instead of creating a new release. Re-apply after running `cargo dist init`. | |
| - name: Upload to GitHub Release and publish | |
| env: | |
| PRERELEASE_FLAG: "${{ | |
| fromJson(steps.host.outputs.manifest).announcement_is_prerelease && | |
| '--prerelease' || '' }}" | |
| NEEDS_PLAN_OUTPUTS_TAG: ${{ needs.plan.outputs.tag }} | |
| run: | | |
| # Upload artifacts to the existing draft release | |
| gh release upload "${NEEDS_PLAN_OUTPUTS_TAG}" artifacts/* --clobber | |
| # Publish the release (undraft) | |
| # shellcheck disable=SC2086 | |
| gh release edit "${NEEDS_PLAN_OUTPUTS_TAG}" --draft=false ${PRERELEASE_FLAG} | |
| announce: | |
| name: Announce published release | |
| timeout-minutes: 5 | |
| needs: | |
| - plan | |
| - host | |
| # use "always() && ..." to allow us to wait for all publish jobs while | |
| # still allowing individual publish jobs to skip themselves (for prereleases). | |
| # "host" however must run to completion, no skipping allowed! | |
| if: ${{ always() && needs.host.result == 'success' }} | |
| runs-on: "ubuntu-22.04" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive |