diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 575a6f56..0641f119 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,14 @@ jobs: runs-on: ubuntu-latest env: UV_NO_EDITABLE: true + VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }} + # only required until R2 supports CRC64NVME checksums + # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11 + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required steps: - uses: actions/checkout@v4 name: Checkout @@ -40,20 +48,12 @@ jobs: # Setup C++ dependencies - run: ./cpp/setup_pipelines.sh name: Clone pipelines & vcpkg - - uses: TAServers/vcpkg-cache@v3 - name: Setup vcpkg cache - id: vcpkg-cache - with: - token: ${{ secrets.GITHUB_TOKEN }} # Setup Python dependencies - name: Install uv uses: astral-sh/setup-uv@v5 - name: Install the project - run: uv sync --dev --verbose --no-editable - env: - VCPKG_FEATURE_FLAGS: "binarycaching" # Possibly redundant, but explicitly sets the binary caching feature flag - VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite" + run: uv sync --dev --verbose # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 @@ -107,6 +107,15 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-13, macos-14] + env: + VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }} + # only required until R2 supports CRC64NVME checksums + # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11 + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required steps: - uses: actions/checkout@v4 name: Checkout @@ -115,25 +124,11 @@ jobs: - run: ./cpp/setup_pipelines.sh name: Clone pipelines & vcpkg - # Restore vcpkg cache - - uses: TAServers/vcpkg-cache@v3 - name: Setup vcpkg cache - id: vcpkg-cache - with: - token: ${{ secrets.GITHUB_TOKEN }} - - uses: pypa/cibuildwheel@v2.23.0 name: Build wheel env: CIBW_BUILD: "cp311-manylinux_x86_64 cp311-macosx_x86_64 cp311-macosx_arm64" - # for vcpkg caches - CIBW_ENVIRONMENT_MACOS: > - MACOSX_DEPLOYMENT_TARGET="11" - VCPKG_FEATURE_FLAGS="binarycaching" - VCPKG_BINARY_SOURCES="clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite" - CIBW_ENVIRONMENT_LINUX: > - VCPKG_FEATURE_FLAGS="binarycaching" - VCPKG_BINARY_SOURCES="clear;files,/host${{ steps.vcpkg-cache.outputs.path }},readwrite" + CIBW_ENVIRONMENT_PASS_LINUX: VCPKG_BINARY_SOURCES AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REQUEST_CHECKSUM_CALCULATION AWS_RESPONSE_CHECKSUM_VALIDATION # save artifacts for testing - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d81d1310..d1d9e8d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,14 @@ jobs: build_wheels: runs-on: ${{ matrix.os }} env: - VCPKG_DEFAULT_BINARY_CACHE: "${{ github.workspace }}/.vcpkg_cache" + VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }} + # only required until R2 supports CRC64NVME checksums + # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11 + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required strategy: matrix: os: [ubuntu-latest, macos-13, macos-14] @@ -18,41 +25,13 @@ jobs: - uses: actions/checkout@v4 name: Checkout - # Let vcpkg store caches in github actions - - name: Restore vcpkg cache - id: cache - uses: actions/cache/restore@v4 - with: - path: "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" - key: vcpkg-${{ matrix.os }} - - name: Ensure vcpkg cache directory exists - run: mkdir -p "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" - # Do all the building - run: ./cpp/setup_pipelines.sh name: Clone pipelines & vcpkg - uses: pypa/cibuildwheel@v2.23.0 name: Build wheel env: - # for vcpkg caches - CIBW_ENVIRONMENT_LINUX : VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" - - # Delete the old cache on hit to emulate a cache update. See - # https://github.com/actions/cache/issues/342. - - name: Delete old cache - env: - GH_TOKEN: ${{ github.token }} - if: steps.cache.outputs.cache-hit - # Using `--repo` makes it so that this step doesn't require checking out the repo first. - run: gh cache delete --repo ${{ github.repository }} ${{ steps.cache.outputs.cache-primary-key }} - - # cache vcpkg - - name: Save vcpkg cache - if: always() - uses: actions/cache/save@v4 - with: - path: "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" - key: vcpkg-${{ matrix.os }} + CIBW_ENVIRONMENT_PASS_LINUX: VCPKG_BINARY_SOURCES AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REQUEST_CHECKSUM_CALCULATION AWS_RESPONSE_CHECKSUM_VALIDATION # upload to use in other jobs - uses: actions/upload-artifact@v4 @@ -102,6 +81,14 @@ jobs: runs-on: ubuntu-latest env: UV_NO_EDITABLE: true + VCPKG_BINARY_SOURCES: ${{ vars.VCPKG_BINARY_SOURCES }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }} + # only required until R2 supports CRC64NVME checksums + # https://community.cloudflare.com/t/an-error-occurred-internalerror-when-calling-the-putobject-operation/764905/11 + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required steps: - uses: actions/checkout@v4 name: Checkout @@ -111,20 +98,12 @@ jobs: # Setup C++ dependencies - run: ./cpp/setup_pipelines.sh name: Clone pipelines & vcpkg - - uses: TAServers/vcpkg-cache@v3 - name: Setup vcpkg cache - id: vcpkg-cache - with: - token: ${{ secrets.GITHUB_TOKEN }} # Setup Python dependencies - name: Install uv uses: astral-sh/setup-uv@v5 - name: Install the project run: uv sync --dev --verbose - env: - VCPKG_FEATURE_FLAGS: "binarycaching" # Possibly redundant, but explicitly sets the binary caching feature flag - VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite" # Setup git - name: Configure Git user @@ -134,6 +113,6 @@ jobs: # Finally deploy docs - name: Get version - run: echo "EVALIO_VERSION=$(python -c 'import evalio; print(evalio.__version__)')" >> $GITHUB_ENV + run: echo "EVALIO_VERSION=$(python -c 'import evalio; print(evalio.__version__)')" >> "$GITHUB_ENV" - name: Deploy docs run: uv run mike deploy --push --update-aliases $EVALIO_VERSION stable \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 90c907b5..09e7a1fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,12 @@ test-command = "evalio ls pipelines && pytest {project}/tests" build-verbosity = 1 [tool.cibuildwheel.linux] -before-all = "yum -y install curl zip unzip tar" +before-all = """ + yum -y install curl zip unzip tar + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + ./aws/install +""" # tbb requires at least 10.13, pcl 10.15. We bump to 11 for simplicity. [tool.cibuildwheel.macos.environment] diff --git a/uv.lock b/uv.lock index df4c5d62..5683d6a3 100644 --- a/uv.lock +++ b/uv.lock @@ -266,7 +266,7 @@ wheels = [ [[package]] name = "evalio" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "argcomplete" },