Skip to content

Commit 1ff19ef

Browse files
dudoslavihnorton
dudoslav
and
ihnorton
authored
Scikit-build-core build system rework (#1988)
Rework build system using `scikit-build-core` Rework all CI/CD logic to use new build system Fix CI/CD numpy compatibility tests --------- Co-authored-by: Isaiah Norton <[email protected]>
1 parent 9643fff commit 1ff19ef

33 files changed

+578
-1559
lines changed

‎.github/disabled-workflows/release.yml

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎.github/workflows/build-wheels.yml

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
name: Build and test wheels
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- release-*
8+
- "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
9+
tags:
10+
- "*"
11+
pull_request:
12+
branches:
13+
- "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
14+
15+
jobs:
16+
build_wheels:
17+
name: Wheel ${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
18+
runs-on: ${{ matrix.buildplat[0] }}
19+
strategy:
20+
matrix:
21+
buildplat:
22+
- [ubuntu-22.04, manylinux_x86_64]
23+
- [macos-13, macosx_x86_64]
24+
- [macos-14, macosx_arm64]
25+
- [windows-2022, win_amd64]
26+
python: ["cp38", "cp39", "cp310", "cp311", "cp312"]
27+
28+
steps:
29+
- uses: actions/checkout@v4
30+
31+
- name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce
32+
if: ${{ startsWith(matrix.os, 'macos-') == true }}
33+
run: |
34+
set -e pipefail
35+
brew update
36+
brew install automake pkg-config ninja llvm
37+
38+
- name: Build wheels
39+
uses: pypa/[email protected]
40+
env:
41+
CIBW_BUILD_VERBOSITY: 3
42+
CIBW_ENVIRONMENT_MACOS: >
43+
CC=clang
44+
CXX=clang++
45+
MACOSX_DEPLOYMENT_TARGET: "11.0"
46+
CIBW_ARCHS: all
47+
CIBW_PRERELEASE_PYTHONS: True
48+
CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
49+
# __init__.py interferes with the tests and is included as local file instead of
50+
# used from wheels. To be honest, tests should not be in the source folder at all.
51+
CIBW_BEFORE_TEST: rm {project}/tiledb/__init__.py
52+
with:
53+
output-dir: wheelhouse
54+
55+
- uses: actions/upload-artifact@v4
56+
with:
57+
name: cibw-wheels-${{ matrix.buildplat[1] }}-${{ strategy.job-index }}
58+
path: "./wheelhouse/*.whl"
59+
60+
build_sdist:
61+
name: Build source distribution
62+
runs-on: ubuntu-latest
63+
outputs:
64+
sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }}
65+
steps:
66+
- uses: actions/checkout@v4
67+
68+
- name: Build sdist
69+
run: pipx run build --sdist
70+
71+
- name: Get sdist package name
72+
id: get_sdist_name
73+
run: |
74+
echo "sdist_name=$(ls dist/ | head -n 1)" >> "$GITHUB_OUTPUT"
75+
76+
- uses: actions/upload-artifact@v4
77+
with:
78+
name: sdist
79+
path: dist/*.tar.gz
80+
81+
test_sdist:
82+
name: Test source distribution package
83+
needs: [build_sdist]
84+
strategy:
85+
matrix:
86+
os:
87+
- macos-13
88+
- macos-14
89+
- windows-2022
90+
- ubuntu-22.04
91+
python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
92+
runs-on: ${{ matrix.os }}
93+
steps:
94+
- name: Set up Python ${{ matrix.python }}
95+
uses: actions/setup-python@v5
96+
with:
97+
python-version: ${{ matrix.python }}
98+
99+
- name: Download sdist artifact
100+
uses: actions/download-artifact@v4
101+
with:
102+
name: sdist
103+
path: dist
104+
105+
- name: Install sdist artifact
106+
run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }}
107+
108+
- uses: actions/checkout@v4
109+
110+
- name: Install test dependencies
111+
run: pip install pytest hypothesis psutil pyarrow
112+
113+
- name: Run tests
114+
shell: bash
115+
run: |
116+
PROJECT_CWD=$PWD
117+
rm tiledb/__init__.py
118+
cd ..
119+
pytest -vv --showlocals $PROJECT_CWD
120+
121+
upload_pypi:
122+
needs: [build_wheels, test_sdist]
123+
runs-on: ubuntu-latest
124+
environment: pypi
125+
permissions:
126+
id-token: write
127+
outputs:
128+
package_version: ${{ steps.get_package_version.outputs.package_version }}
129+
steps:
130+
- uses: actions/download-artifact@v4
131+
with:
132+
path: dist
133+
merge-multiple: true
134+
135+
- id: get_package_version
136+
run: |
137+
echo "package_version=$(ls dist/ | head -n 1 | cut -d - -f 2)" >> "$GITHUB_OUTPUT"
138+
139+
- name: Upload to test-pypi
140+
uses: pypa/gh-action-pypi-publish@release/v1
141+
with:
142+
repository-url: https://test.pypi.org/legacy/
143+
144+
- name: Upload to pypi
145+
if: startsWith(github.ref, 'refs/tags/')
146+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/ci.yml

Lines changed: 31 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,29 @@ jobs:
1414
shell: bash
1515
strategy:
1616
matrix:
17-
os: [ubuntu-latest, macos-12, windows-latest]
17+
os:
18+
- ubuntu-latest
19+
- macos-13
20+
# libfaketime tests fail on macos arm. Disable tests for now.
21+
# - macos-14
22+
- windows-latest
1823
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
1924
fail-fast: false
2025
env:
21-
# 11.7 necessary due to: https://github.com/actions/setup-python/issues/682#issuecomment-1604261330
22-
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-12' && matrix.python-version == '3.8' && '11.7' || '11' }}
23-
#MACOSX_DEPLOYMENT_TARGET: "10.11"
24-
# On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory
26+
MACOSX_DEPLOYMENT_TARGET: "11"
2527
steps:
2628
- name: Checkout TileDB-Py `dev`
27-
uses: actions/checkout@v3
29+
uses: actions/checkout@v4
2830

2931
- name: Setup MSVC toolset (VS 2022)
3032
uses: TheMrMilchmann/setup-msvc-dev@v3
31-
if: matrix.os == 'windows-latest'
33+
if: startsWith(matrix.os, 'windows')
3234
with:
3335
arch: x64
3436

3537
- name: Install Ninja (VS 2022)
3638
uses: seanmiddleditch/gha-setup-ninja@v4
37-
if: matrix.os == 'windows-latest'
39+
if: startsWith(matrix.os, 'windows')
3840

3941
- name: Set up Python ${{ matrix.python-version }}
4042
uses: actions/setup-python@v4
@@ -57,79 +59,29 @@ jobs:
5759
# - https://github.com/actions/runner-images/pull/7125
5860
- name: "Install homebrew dependencies"
5961
run: brew install pkg-config
60-
if: matrix.os == 'macos-12'
61-
62-
- name: "Install dependencies"
63-
run: python -m pip install --upgrade -r misc/requirements_ci.txt
64-
65-
- name: "Get TILEDB_VERSION"
66-
run: echo "LIBTILEDB_VERSION=$(python setup.py get_tiledb_version | tail -n 1)" >> $GITHUB_ENV
67-
68-
- name: "Get LIBTILEDB_SHA"
69-
run: echo "LIBTILEDB_SHA=$(git ls-remote https://github.com/TileDB-Inc/TileDB $LIBTILEDB_VERSION | cut -c1-7)" >> $GITHUB_ENV
70-
71-
- name: "Download TileDB From Zip And Build TileDB-Py (Windows)"
72-
run: |
73-
choco install wget --no-progress
74-
75-
if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip; then
76-
mkdir libtiledb
77-
unzip tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip -d libtiledb
78-
cp libtiledb/bin/tiledb.dll tiledb
79-
python setup.py develop --tiledb=libtiledb
80-
else
81-
# Build from source as fallback
82-
python setup.py build_ext --inplace
83-
python setup.py develop
84-
fi
85-
env:
86-
TILEDB_FORCE_ALL_DEPS: True
87-
CMAKE_GENERATOR: "Ninja"
88-
if: matrix.os == 'windows-latest'
89-
90-
- name: "Download TileDB From Tarball And Build TileDB-Py (macOS)"
91-
run: |
92-
set -xeo pipefail
93-
94-
if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then
95-
mkdir libtiledb
96-
sudo tar -vzxf tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C libtiledb
97-
python setup.py develop --tiledb=libtiledb
98-
else
99-
# Build from source as fallback
100-
python setup.py build_ext --inplace
101-
python setup.py develop
102-
fi
103-
if: matrix.os == 'macos-12'
104-
105-
- name: "Download TileDB From Tarball And Build TileDB-Py (Linux)"
106-
run: |
107-
set -xeo pipefail
108-
109-
if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then
110-
mkdir libtiledb
111-
sudo tar -vzxf tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C libtiledb
112-
python setup.py develop --tiledb=libtiledb
113-
else
114-
# Build from source as fallback
115-
python setup.py build_ext --inplace
116-
python setup.py develop
117-
fi
118-
if: matrix.os == 'ubuntu-latest'
119-
120-
- name: "Check build directory"
121-
run: ls -Rl
62+
if: startsWith(matrix.os, 'macos')
12263

12364
- name: "Install libfaketime (linux and macOS)"
124-
if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-12'
65+
if: ${{ ! startsWith(matrix.os, 'windows') }}
12566
run: |
12667
git clone https://github.com/wolfcw/libfaketime/
12768
cd libfaketime
12869
sudo make install
12970
cd ..
13071
72+
- name: "Build and Install TileDB-Py"
73+
# We use pipx here to produce wheel/sdist to upload as artifact in case of error
74+
run: |
75+
pipx run --python ${{ matrix.python-version }} build
76+
WHEEL_NAME=$(ls dist/*.whl)
77+
pip install --verbose ${WHEEL_NAME}[test]
78+
13179
- name: "Run tests"
132-
run: pytest -vv --showlocals
80+
run: |
81+
PROJECT_CWD=$PWD
82+
rm tiledb/__init__.py
83+
cd /tmp
84+
pytest -vv --showlocals $PROJECT_CWD
13385
13486
- name: "Print log files (failed build only)"
13587
run: |
@@ -144,3 +96,10 @@ jobs:
14496
cat $f
14597
done;
14698
if: failure()
99+
100+
- name: "Upload files for debug"
101+
if: always()
102+
uses: actions/upload-artifact@v4
103+
with:
104+
name: cibw-wheels-${{ matrix.os }}-${{ matrix.python-version }}
105+
path: "."

0 commit comments

Comments
 (0)