Scikit-build-core build system rework

.github/workflows/build-wheels.yml

@@ -0,0 +1,146 @@
+name: Build and test wheels
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - release-*
+      - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
+    tags:
+      - "*"
+  pull_request:
+    branches:
+      - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
+
+jobs:
+  build_wheels:
+    name: Wheel ${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
+    runs-on: ${{ matrix.buildplat[0] }}
+    strategy:
+      matrix:
+        buildplat:
+          - [ubuntu-22.04, manylinux_x86_64]
+          - [macos-13, macosx_x86_64]
+          - [macos-14, macosx_arm64]
+          - [windows-2022, win_amd64]
+        python: ["cp38", "cp39", "cp310", "cp311", "cp312"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce
+        if: ${{ startsWith(matrix.os, 'macos-') == true }}
+        run: |
+          set -e pipefail
+          brew update
+          brew install automake pkg-config ninja llvm
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_ENVIRONMENT_MACOS: >
+            CC=clang
+            CXX=clang++
+          MACOSX_DEPLOYMENT_TARGET: "11.0"
+          CIBW_ARCHS: all
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
+          # __init__.py interferes with the tests and is included as local file instead of
+          # used from wheels. To be honest, tests should not be in the source folder at all.
+          CIBW_BEFORE_TEST: rm {project}/tiledb/__init__.py
+        with:
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.buildplat[1] }}-${{ strategy.job-index }}
+          path: "./wheelhouse/*.whl"
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    outputs:
+      sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Get sdist package name
+        id: get_sdist_name
+        run: |
+          echo "sdist_name=$(ls dist/ | head -n 1)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  test_sdist:
+    name: Test source distribution package
+    needs: [build_sdist]
+    strategy:
+      matrix:
+        os:
+          - macos-13
+          - macos-14
+          - windows-2022
+          - ubuntu-22.04
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: Install sdist artifact
+        run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }}
+
+      - uses: actions/checkout@v4
+
+      - name: Install test dependencies
+        run: pip install pytest hypothesis psutil pyarrow
+
+      - name: Run tests
+        shell: bash
+        run: |
+          PROJECT_CWD=$PWD
+          rm tiledb/__init__.py
+          cd ..
+          pytest -vv --showlocals $PROJECT_CWD
+
+  upload_pypi:
+    needs: [build_wheels, test_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    outputs:
+      package_version: ${{ steps.get_package_version.outputs.package_version }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - id: get_package_version
+        run: |
+          echo "package_version=$(ls dist/ | head -n 1 | cut -d - -f 2)" >> "$GITHUB_OUTPUT"
+
+      - name: Upload to test-pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+      - name: Upload to pypi
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/ci.yml

@@ -14,27 +14,29 @@ jobs:
         shell: bash
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-12, windows-latest]
+        os:
+          - ubuntu-latest
+          - macos-13
+          # libfaketime tests fail on macos arm. Disable tests for now.
+          # - macos-14
+          - windows-latest
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
       fail-fast: false
     env:
-      # 11.7 necessary due to: https://github.com/actions/setup-python/issues/682#issuecomment-1604261330
-      MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-12' && matrix.python-version == '3.8' && '11.7' || '11' }}
-      #MACOSX_DEPLOYMENT_TARGET: "10.11"
-      # On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory
+      MACOSX_DEPLOYMENT_TARGET: "11"
     steps:
       - name: Checkout TileDB-Py `dev`
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup MSVC toolset (VS 2022)
         uses: TheMrMilchmann/setup-msvc-dev@v3
-        if: matrix.os == 'windows-latest'
+        if: startsWith(matrix.os, 'windows')
         with:
           arch: x64
 
       - name: Install Ninja (VS 2022)
         uses: seanmiddleditch/gha-setup-ninja@v4
-        if: matrix.os == 'windows-latest'
+        if: startsWith(matrix.os, 'windows')
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -57,79 +59,29 @@ jobs:
       # - https://github.com/actions/runner-images/pull/7125
       - name: "Install homebrew dependencies"
         run: brew install pkg-config
-        if: matrix.os == 'macos-12'
-
-      - name: "Install dependencies"
-        run: python -m pip install --upgrade -r misc/requirements_ci.txt
-
-      - name: "Get TILEDB_VERSION"
-        run: echo "LIBTILEDB_VERSION=$(python setup.py get_tiledb_version | tail -n 1)" >> $GITHUB_ENV
-
-      - name: "Get LIBTILEDB_SHA"
-        run: echo "LIBTILEDB_SHA=$(git ls-remote https://github.com/TileDB-Inc/TileDB $LIBTILEDB_VERSION | cut -c1-7)" >> $GITHUB_ENV
-
-      - name: "Download TileDB From Zip And Build TileDB-Py (Windows)"
-        run: |
-          choco install wget --no-progress
-
-          if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip; then
-            mkdir libtiledb
-            unzip tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip -d libtiledb
-            cp libtiledb/bin/tiledb.dll tiledb
-            python setup.py develop --tiledb=libtiledb
-          else
-            # Build from source as fallback
-            python setup.py build_ext --inplace
-            python setup.py develop
-          fi
-        env:
-          TILEDB_FORCE_ALL_DEPS: True
-          CMAKE_GENERATOR: "Ninja"
-        if: matrix.os == 'windows-latest'
-
-      - name: "Download TileDB From Tarball And Build TileDB-Py (macOS)"
-        run: |
-          set -xeo pipefail
-
-          if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then
-            mkdir libtiledb
-            sudo tar -vzxf tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C libtiledb
-            python setup.py develop --tiledb=libtiledb
-          else
-            # Build from source as fallback
-            python setup.py build_ext --inplace
-            python setup.py develop
-          fi
-        if: matrix.os == 'macos-12'
-
-      - name: "Download TileDB From Tarball And Build TileDB-Py (Linux)"
-        run: |
-          set -xeo pipefail
-
-          if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then
-            mkdir libtiledb
-            sudo tar -vzxf tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C libtiledb
-            python setup.py develop --tiledb=libtiledb
-          else
-            # Build from source as fallback
-            python setup.py build_ext --inplace
-            python setup.py develop
-          fi
-        if: matrix.os == 'ubuntu-latest'
-
-      - name: "Check build directory"
-        run: ls -Rl
+        if: startsWith(matrix.os, 'macos')
 
       - name: "Install libfaketime (linux and macOS)"
-        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-12'
+        if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           git clone https://github.com/wolfcw/libfaketime/
           cd libfaketime
           sudo make install
           cd ..
 
+      - name: "Build and Install TileDB-Py"
+        # We use pipx here to produce wheel/sdist to upload as artifact in case of error
+        run: |
+          pipx run --python ${{ matrix.python-version }} build
+          WHEEL_NAME=$(ls dist/*.whl)
+          pip install --verbose ${WHEEL_NAME}[test]
+
       - name: "Run tests"
-        run: pytest -vv --showlocals
+        run: |
+          PROJECT_CWD=$PWD
+          rm tiledb/__init__.py
+          cd /tmp
+          pytest -vv --showlocals $PROJECT_CWD
 
       - name: "Print log files (failed build only)"
         run: |
@@ -144,3 +96,10 @@ jobs:
                cat $f
             done;
         if: failure()
+
+      - name: "Upload files for debug"
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ matrix.python-version }}
+          path: "."