Skip to content

Commit 2be89d4

Browse files
danpfdanpf
committed
Adding Python monorepo builder
1 parent 2dcfd98 commit 2be89d4

File tree

12 files changed

+491
-236
lines changed

12 files changed

+491
-236
lines changed

Diff for: ‎.github/workflows/test_and_deploy.yml

+9-52
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,6 @@ concurrency:
1818
jobs:
1919
test-and-build-dpf-ssw-aligner:
2020
name: "Build & Test dpf-ssw-aligner-pyrs"
21-
strategy:
22-
matrix:
23-
os: [ubuntu-20.04, windows-2019, macos-11]
24-
project_folder: ["./src/python/dpf-ssw-aligner-rs"]
25-
python-version: ['cp310', 'cp311']
26-
runs-on: ${{ matrix.os }}
27-
env:
28-
CIBW_ENVIRONMENT: 'PATH="$PATH:$HOME/.cargo/bin"'
29-
CIBW_SKIP: "p*-win* *-win32 *-win_arm64 *-musllinux_*"
3021

3122
steps:
3223
- uses: actions/checkout@v3
@@ -39,65 +30,31 @@ jobs:
3930
with:
4031
platforms: all
4132

42-
- name: Install latest nightly
43-
uses: dtolnay/rust-toolchain@nightly
44-
45-
- name: Build & test wheels dpf-ssw-aligner-rs wheels
46-
uses: pypa/[email protected]
33+
- name: Set up Python 3.12
34+
uses: actions/setup-python@v4
4735
with:
48-
package-dir: src/python/dpf-ssw-aligner-rs
49-
env:
50-
CIBW_BUILD: '${{ matrix.python-version }}-*'
51-
# we build for "alt_arch_name" if it exists, else 'auto'
52-
CIBW_ARCHS: ${{ matrix.alt_arch_name || 'auto' }}
53-
CIBW_ENVIRONMENT: 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"'
54-
CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
55-
CIBW_BEFORE_ALL_WINDOWS: "rustup target add x86_64-pc-windows-msvc i686-pc-windows-msvc"
56-
CIBW_BEFORE_ALL_MACOS: "rustup target add aarch64-apple-darwin x86_64-apple-darwin"
57-
CIBW_BEFORE_BUILD: rustup show
58-
CIBW_BEFORE_BUILD_LINUX: >
59-
curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=nightly --profile=minimal -y &&
60-
rustup show
61-
# CIBW_TEST_SKIP: '*-macosx_universal2:arm64'
62-
CIBW_BUILD_VERBOSITY: 1
36+
python-version: "3.12"
6337

64-
- name: Build sdist
65-
run: pipx run build --sdist --outdir wheelhouse src/python/dpf-ssw-aligner-rs
38+
- name: Build, Test, and Upload dpf-ssw-aligner-rs
39+
run: python ./mono.py cicd_dpf_ssw_aligner
6640

6741
- uses: actions/upload-artifact@v3
6842
with:
6943
path: ./wheelhouse/dpf_ssw_aligner*.whl
7044

7145
test-and-build-dpf-mrcfile:
7246
name: "Build & Test dpf-mrcfile"
73-
strategy:
74-
matrix:
75-
os: [ubuntu-20.04, windows-2019, macos-11]
76-
project_folder: ["./src/python/dpf-mrcfile"]
77-
python-version: ['3.10', '3.11']
78-
runs-on: ${{ matrix.os }}
79-
env:
80-
CIBW_SKIP: "p*-win* *-win32 *-win_arm64 *-musllinux_*"
8147

8248
steps:
8349
- uses: actions/checkout@v3
8450

85-
- name: Set up Python ${{ matrix.python-version }}
51+
- name: Set up Python 3.12
8652
uses: actions/setup-python@v4
8753
with:
88-
python-version: ${{ matrix.python-version }}
89-
90-
- name: Install dependencies
91-
run: |
92-
python -m pip install --upgrade pip
93-
pip install pytest build src/python/dpf-mrcfile
94-
95-
- name: Test with pytest
96-
run: |
97-
pytest src/python/dpf-mrcfile/test
54+
python-version: "3.12"
9855

99-
- name: Build sdist
100-
run: python -m build -o wheelhouse src/python/dpf-mrcfile
56+
- name: Build, Test, and Upload dpf-mrcfile
57+
run: python ./mono.py cicd_dpf_ssw_aligner
10158

10259
- uses: actions/upload-artifact@v3
10360
if: runner.os == 'Linux'

Diff for: ‎mono.py

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
#!/usr/bin/env python
2+
"""
3+
For now we'll run most with shell=True so that we can move away from this quickly if necessary
4+
"""
5+
6+
import argparse
7+
import os
8+
import platform
9+
from typing import List
10+
import sys
11+
import logging
12+
import subprocess
13+
14+
LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
15+
logging.basicConfig(level=LOGLEVEL)
16+
log = logging.getLogger(__name__)
17+
18+
CIBUILDWHEEL_VERSION = "2.16.2"
19+
CIBUILDWHEEL_PLATFORM = "auto"
20+
21+
22+
def parseargs(args: list[str]) -> argparse.Namespace:
23+
parser = argparse.ArgumentParser()
24+
parser.add_argument(
25+
"--mode",
26+
choices=["install_ci_deps", "cicd_dpf_ssw_aligner_rs", "cicd_dpf_mrcfile"],
27+
help="mode to run",
28+
required=True,
29+
)
30+
parser.add_argument(
31+
"--cibw_platform",
32+
choices=["auto", "linux", "macos", "windows"],
33+
help="cibuildwheel platform to build",
34+
default=CIBUILDWHEEL_PLATFORM,
35+
)
36+
return parser.parse_args(args)
37+
38+
39+
def run_l(cmd: list[str]) -> None:
40+
str_cmd = " ".join(cmd)
41+
log.debug(f"Running: {str_cmd}")
42+
ret = subprocess.run(cmd)
43+
if ret.returncode:
44+
raise RuntimeError(f"Failure with command: {str_cmd}")
45+
46+
47+
def run_s(cmd: str, env_add: dict[str, str]) -> None:
48+
log.debug(f"Running: {cmd}")
49+
env = dict(os.environ)
50+
env.update(env_add)
51+
ret = subprocess.run(cmd, env=env, shell=True)
52+
if ret.returncode:
53+
raise RuntimeError(f"Failure with command: {cmd}")
54+
55+
56+
def install_rust_cibw_command():
57+
return (
58+
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs"
59+
" | sh -s -- --default-toolchain none -y"
60+
" && rustup toolchain install nightly --allow-downgrade --profile minimal --component clippy"
61+
" && rustup show"
62+
)
63+
64+
65+
def install_ci_deps():
66+
cmds = f"""
67+
python -m pip install cibuildwheel=={CIBUILDWHEEL_VERSION}
68+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y
69+
rustup toolchain install nightly --allow-downgrade --profile minimal --component clippy
70+
rustup show
71+
pip install pipx
72+
"""
73+
match platform.system():
74+
case "Darwin":
75+
cmds += "rustup target add aarch64-apple-darwin x86_64-apple-darwin"
76+
case "Windows":
77+
cmds += "rustup target add x86_64-pc-windows-msvc i686-pc-windows-msvc"
78+
for cmd in cmds.split("\n"):
79+
if cmd:
80+
run_s(cmd, {})
81+
82+
83+
def cicd_dpf_ssw_aligner_rs():
84+
env = {
85+
"CIBW_PLATFORM": CIBUILDWHEEL_PLATFORM,
86+
"CIBW_ARCHS": "auto",
87+
"CIBW_ENVIRONMENT": 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"',
88+
"CIBW_ENVIRONMENT_WINDOWS": 'PATH="$UserProfile\\.cargo\\bin;$PATH"',
89+
"CIBW_BEFORE_BUILD": install_rust_cibw_command(),
90+
"CIBW_BUILD_VERBOSITY": "1",
91+
}
92+
93+
cmds = """
94+
python -m cibuildwheel ./src/python/dpf-ssw-aligner-rs --output-dir wheelhouse
95+
ls wheelhouse
96+
pipx run build --sdist --outdir wheelhouse ./src/python/dpf-ssw-aligner-rs
97+
ls wheelhouse
98+
"""
99+
for cmd in cmds.split("\n"):
100+
if cmd:
101+
run_s(cmd, env)
102+
103+
104+
def cicd_dpf_mrcfile():
105+
env = {
106+
"CIBW_PLATFORM": CIBUILDWHEEL_PLATFORM,
107+
"CIBW_ARCHS": "auto",
108+
"CIBW_ENVIRONMENT": 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"',
109+
"CIBW_ENVIRONMENT_WINDOWS": 'PATH="$UserProfile\\.cargo\\bin;$PATH"',
110+
"CIBW_BEFORE_BUILD": install_rust_cibw_command(),
111+
"CIBW_BUILD_VERBOSITY": "1",
112+
}
113+
114+
cmds = """
115+
python -m cibuildwheel ./src/python/dpf-mrcfile --output-dir wheelhouse
116+
ls wheelhouse
117+
pipx run build --sdist --outdir wheelhouse ./src/python/dpf-mrcfile
118+
ls wheelhouse
119+
"""
120+
for cmd in cmds.split("\n"):
121+
if cmd:
122+
run_s(cmd, env)
123+
124+
125+
def main(_args: List[str]) -> None:
126+
args = parseargs(_args)
127+
global CIBUILDWHEEL_PLATFORM
128+
CIBUILDWHEEL_PLATFORM = args.cibw_platform
129+
match args.mode:
130+
case "install_ci_deps":
131+
install_ci_deps()
132+
case "cicd_dpf_ssw_aligner_rs":
133+
install_ci_deps()
134+
cicd_dpf_ssw_aligner_rs()
135+
case "cicd_dpf_mrcfile":
136+
install_ci_deps()
137+
cicd_dpf_mrcfile()
138+
139+
140+
if __name__ == "__main__":
141+
main(sys.argv[1:])

Diff for: ‎src/python/dpf-mrcfile/pyproject.toml

+43-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[project]
2-
name = "dpf_mrcfile"
2+
name = "dpf-mrcfile"
33
description="A library for handling scientific volume and image-stack data like from .MAP .MRC .MRCS"
44
requires-python = ">=3.10"
55
authors = [{name = "Danny Farrell", email = "[email protected]"}]
@@ -14,6 +14,48 @@ classifiers = [
1414
"Operating System :: MacOS :: MacOS X",
1515
]
1616

17+
18+
[project.optional-dependencies]
19+
dev = ["pytest>=6.0", "twine"]
20+
1721
[tool.cibuildwheel]
1822
test-requires = "pytest"
1923
test-command = "pytest {package}/test"
24+
test-skip = ["*universal2:arm64"]
25+
skip = "*-win32 *-win_arm64 pp*win* *musllinux*"
26+
27+
[tool.ruff]
28+
src = ["src/dpf-mrcfile"]
29+
30+
[tool.ruff.lint]
31+
extend-select = [
32+
"B", # flake8-bugbear
33+
"I", # isort
34+
"ARG", # flake8-unused-arguments
35+
"C4", # flake8-comprehensions
36+
"EM", # flake8-errmsg
37+
"ICN", # flake8-import-conventions
38+
"G", # flake8-logging-format
39+
"PGH", # pygrep-hooks
40+
"PIE", # flake8-pie
41+
"PL", # pylint
42+
"PTH", # flake8-use-pathlib
43+
"RET", # flake8-return
44+
"RUF", # Ruff-specific
45+
"SIM", # flake8-simplify
46+
"T20", # flake8-print
47+
"UP", # pyupgrade
48+
"YTT", # flake8-2020
49+
"EXE", # flake8-executable
50+
"NPY", # NumPy specific rules
51+
"PD", # pandas-vet
52+
]
53+
ignore = [
54+
"PLR", # Design related pylint codes
55+
"PT", # flake8-pytest-style
56+
]
57+
isort.required-imports = ["from __future__ import annotations"]
58+
59+
[tool.ruff.per-file-ignores]
60+
"src/dpf-mrcfile/test/**" = ["T20"]
61+

0 commit comments

Comments
 (0)