Skip to content

Commit

Permalink
Add Python pybind11 bindings + Upgrade ci/cd
Browse files Browse the repository at this point in the history
This PR adds python bindings as well as updates the backend build
process of the mmtf-cpp library significantly.  These improvements are
mainly from a convenience perspective and include:
- Removing all build-based submodules
- Moving to cmake fetchcontent build
- Simplify CMakeLists with better linking procedures
- Upgrade msgpack-c
- Upgrade catch2
- Move to github actions for ci/cd
- Use cibuildwheel for wheel cd

Pybind11 library:
The pybind11 library utilizes the c++ code of mmtf-cpp in order to build
an extremely fast cpp layer underneath the python interface.  You have
to keep in mind that moving between c++ and python is slow, but this is
still much faster than the previously existing python library. see this
example:

time to load a single mmtf file 1000x
cpp bare 0.29s
this library 0.44s
python og 4.34s
  • Loading branch information
danpf committed Dec 15, 2023
1 parent 3901478 commit 6b67aba
Show file tree
Hide file tree
Showing 28 changed files with 1,882 additions and 247 deletions.
62 changes: 62 additions & 0 deletions .github/workflows/cpp.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
---
name: cpp
'on':
workflow_dispatch: null
pull_request: null
push:
branches:
- master
concurrency:
group: '${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true
jobs:
build:
name: Build and test cpp
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-22.04
cc: gcc
cxx: g++
- os: ubuntu-22.04
cc: gcc
cxx: g++
env_list: EMSCRIPTEN=ON
- os: ubuntu-22.04
cc: clang
cxx: clang++
- os: ubuntu-22.04
cc: gcc
cxx: g++
cmake_args: "-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-march=native'"
- os: macos-latest
cc: clang
cxx: clang++
- os: windows-latest
cc: ''
cxx: ''
runs-on: '${{ matrix.os }}'
env:
CC: '${{ matrix.cc }}'
CXX: '${{ matrix.cxx }}'
CMAKE_ARGS: '${{ matrix.cmake_args }}'
steps:
- uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- name: Set environment list variables
run: |
env_vars="${{ matrix.env_list }}"
for var in $env_vars; do
echo "$var" >> $GITHUB_ENV
done
if: matrix.os != 'windows-latest'
- name: Setup cmake
uses: jwlawson/[email protected]
with:
cmake-version: 3.16.x
- uses: seanmiddleditch/gha-setup-ninja@master
- name: build and test
run: ./ci/build_and_run_tests.sh
48 changes: 48 additions & 0 deletions .github/workflows/emscripten.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
---
name: WASM
'on':
workflow_dispatch: null
push:
branches:
- master
concurrency:
group: '${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true
jobs:
build-wasm-emscripten:
name: Pyodide
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install pyodide-build
run: pip install pyodide-build==0.23.4
- name: Compute emsdk version
id: compute-emsdk-version
run: |
pyodide xbuildenv install --download
EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)
echo "emsdk-version=$EMSCRIPTEN_VERSION" >> $GITHUB_OUTPUT
- uses: mymindstorm/setup-emsdk@v12
with:
version: '${{ steps.compute-emsdk-version.outputs.emsdk-version }}'
actions-cache-folder: emsdk-cache
- name: Build
run: CFLAGS=-fexceptions LDFLAGS=-fexceptions pyodide build
- uses: actions/upload-artifact@v3
with:
path: dist/*.whl
- uses: actions/setup-node@v4
with:
node-version: 18
- name: Set up Pyodide virtual environment
run: |
pyodide venv .venv-pyodide
.venv-pyodide/bin/pip install $(echo -n dist/*.whl)
- name: Test
run: .venv-pyodide/bin/python -m unittest src/python/tests/tests.py
39 changes: 39 additions & 0 deletions .github/workflows/pip.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
---
name: Pip
'on':
workflow_dispatch: null
pull_request: null
push:
branches:
- master
concurrency:
group: '${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true
jobs:
build:
name: Build with Pip
runs-on: '${{ matrix.platform }}'
strategy:
fail-fast: false
matrix:
platform:
- windows-latest
- macos-latest
- ubuntu-latest
python-version:
- '3.8'
- '3.11'
- '3.12'
- pypy-3.8
steps:
- uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '${{ matrix.python-version }}'
- name: Build and install
run: pip install --verbose .
- name: Test
run: python src/python/tests/tests.py
74 changes: 74 additions & 0 deletions .github/workflows/wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
---
name: Wheels
'on':
workflow_dispatch: null
pull_request: null
push:
branches:
- master
release:
types:
- published
env:
FORCE_COLOR: 3
concurrency:
group: '${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true
jobs:
build_sdist:
name: Build SDist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Build SDist
run: pipx run build --sdist
- name: Check metadata
run: pipx run twine check dist/*
- uses: actions/upload-artifact@v3
with:
path: dist/*.tar.gz
build_wheels:
name: 'Wheels on ${{ matrix.os }}'
runs-on: '${{ matrix.os }}'
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: pypa/[email protected]
env:
CIBW_ARCHS_MACOS: universal2
CIBW_ARCHS_WINDOWS: auto ARM64
CMAKE_GENERATOR: '${{ env.CMAKE_GENERATOR }}'
- name: Verify clean directory
run: git diff --exit-code
shell: bash
- uses: actions/upload-artifact@v3
with:
path: wheelhouse/*.whl
upload_all:
name: Upload if release
needs:
- build_wheels
- build_sdist
runs-on: ubuntu-latest
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- uses: actions/setup-python@v4
with:
python-version: 3.x
- uses: actions/download-artifact@v3
with:
name: artifact
path: dist
- uses: pypa/gh-action-pypi-publish@release/v1
with:
password: '${{ secrets.pypi_password }}'
169 changes: 169 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,172 @@ build/*
docs/html/*
examples/out/*
examples/out_json_ref/*

# python eggs
src/python/*.egg-info
**/__pycache__
**/*.pyc


# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/
**/_version.py

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/

Loading

0 comments on commit 6b67aba

Please sign in to comment.