diff --git a/.github/workflows/update_bioconda_recipe.yml b/.github/workflows/update_bioconda_recipe.yml new file mode 100644 index 00000000..1fb9a3dd --- /dev/null +++ b/.github/workflows/update_bioconda_recipe.yml @@ -0,0 +1,148 @@ +name: Update Bioconda Recipe + +# Opens a version-bump pull request against bioconda/bioconda-recipes when a +# release is published. Bioconda's autobump bot usually does this on its own +# within a day or so; this workflow makes the bump deterministic instead of +# waiting on the bot. +# +# Requires a BIOCONDA_PAT secret: a token with `public_repo` scope on a fork of +# bioconda/bioconda-recipes owned by BIOCONDA_FORK_OWNER. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + version: + description: 'Version to update (e.g., 0.33.0)' + required: true + type: string + +permissions: + contents: read + +env: + BIOCONDA_FORK_OWNER: mwiewior + +jobs: + update-bioconda: + runs-on: ubuntu-latest + steps: + - name: Resolve version + id: version + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.version }}" + else + VERSION="${GITHUB_REF#refs/tags/}" + fi + # Tags are published without a leading "v", but tolerate one. + VERSION="${VERSION#v}" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Resolved version: $VERSION" + + - name: Wait for the sdist to appear on PyPI + id: sdist + run: | + set -euo pipefail + VERSION="${{ steps.version.outputs.version }}" + URL="https://pypi.org/pypi/polars-bio/${VERSION}/json" + + # Publishing to PyPI is a separate workflow, so the release event can + # win the race. Poll rather than fail on a 404. + for attempt in $(seq 1 60); do + SDIST_URL=$(curl -fsSL "$URL" 2>/dev/null \ + | jq -r '.urls[] | select(.packagetype=="sdist") | .url' || true) + if [ -n "$SDIST_URL" ]; then + echo "sdist available after ${attempt} attempt(s): $SDIST_URL" + break + fi + echo "Attempt ${attempt}: sdist for ${VERSION} not on PyPI yet; retrying in 30s" + sleep 30 + done + + if [ -z "${SDIST_URL:-}" ]; then + echo "::error::sdist for polars-bio ${VERSION} never appeared on PyPI" + exit 1 + fi + + curl -fsSL -o sdist.tar.gz "$SDIST_URL" + SHA256=$(sha256sum sdist.tar.gz | cut -d' ' -f1) + echo "sha256=$SHA256" >> "$GITHUB_OUTPUT" + echo "SHA256: $SHA256" + + - name: Checkout bioconda-recipes fork + uses: actions/checkout@v4 + with: + repository: ${{ env.BIOCONDA_FORK_OWNER }}/bioconda-recipes + token: ${{ secrets.BIOCONDA_PAT }} + path: bioconda-recipes + fetch-depth: 0 + + - name: Sync fork with upstream + working-directory: bioconda-recipes + run: | + set -euo pipefail + # Branching off a stale fork produces a PR full of unrelated commits. + git remote add upstream https://github.com/bioconda/bioconda-recipes.git + git fetch upstream master + git checkout -B "update-polars-bio-${{ steps.version.outputs.version }}" upstream/master + + - name: Update recipe + working-directory: bioconda-recipes + env: + VERSION: ${{ steps.version.outputs.version }} + SHA256: ${{ steps.sdist.outputs.sha256 }} + run: | + set -euo pipefail + RECIPE=recipes/polars-bio/meta.yaml + test -f "$RECIPE" || { echo "::error::$RECIPE not found — has the recipe been merged yet?"; exit 1; } + + # Anchored so these only ever touch the intended lines. + sed -i -E "s|^\{% set version = \".*\" %\}$|{% set version = \"${VERSION}\" %}|" "$RECIPE" + sed -i -E "s|^( sha256: ).*$|\1${SHA256}|" "$RECIPE" + sed -i -E "s|^( number: ).*$|\10|" "$RECIPE" + + grep -q "set version = \"${VERSION}\"" "$RECIPE" || { echo "::error::version not updated"; exit 1; } + grep -q "${SHA256}" "$RECIPE" || { echo "::error::sha256 not updated"; exit 1; } + + - name: Push and open pull request + working-directory: bioconda-recipes + env: + GH_TOKEN: ${{ secrets.BIOCONDA_PAT }} + VERSION: ${{ steps.version.outputs.version }} + SHA256: ${{ steps.sdist.outputs.sha256 }} + run: | + set -euo pipefail + if git diff --quiet recipes/polars-bio/meta.yaml; then + echo "No changes to push." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add recipes/polars-bio/meta.yaml + git commit -m "Update polars-bio to ${VERSION}" + git push --force origin "HEAD:update-polars-bio-${VERSION}" + + gh pr create \ + --repo bioconda/bioconda-recipes \ + --base master \ + --head "${{ env.BIOCONDA_FORK_OWNER }}:update-polars-bio-${VERSION}" \ + --title "Update polars-bio to ${VERSION}" \ + --body "$(cat <=1.37.1` | yes | +| `pyarrow` | `>=23.0.1,<25` | yes | +| `datafusion` | `>=53.0.0,<54` | yes (53.0.0) | +| `tqdm` | `>=4.67.0,<5` | yes | +| `polars-config-meta` | `>=0.3.0,<1` | **no — submit first** | + +Note that `polars-config-meta` declares `dependencies = []` in its +`pyproject.toml` and lists polars only as an optional extra, but imports polars +at module scope. The conda recipe therefore declares polars as a hard run +dependency; without it the package installs but fails on import. + +## Build notes + +Two things in the source tree need handling in `build.sh`, which is why the +recipe uses a build script rather than an inline `script:` entry: + +- `rust-toolchain.toml` pins an exact toolchain for local development. The conda + build supplies its own `rustc` and has no `rustup` to satisfy the pin with, so + the file is removed. +- Upstream CI builds PyPI wheels with `-Ctarget-cpu=skylake` / `apple-m1` and + `-Dwarnings`. Both are wrong for a redistributable package, so `RUSTFLAGS` is + cleared. + +The crate graph is large (662 crates) and six dependencies resolve from git +tags, so the build environment needs `git` and network access. `Cargo.lock` is +shipped in the sdist, so those revisions are pinned. `cargo-bundle-licenses` +records the licences of that crate graph into `THIRDPARTY.yml`, which is listed +in `license_file` alongside `LICENSE`. + +One local-only gotcha: running `conda-build` without conda-forge's pinning lets +it choose a macOS deployment target from the host SDK, which can exceed the +running OS version. maturin then tags the wheel with that version and the test +phase fails `pip check` with "not supported on this platform". Pass +`--variants "{MACOSX_DEPLOYMENT_TARGET: ['11.0']}"` to reproduce what CI does. +This does not affect bioconda, which pins the target well below the runner. + +## Local verification + +```bash +conda create -n cbuild -c conda-forge conda-build +conda activate cbuild + +# polars-config-meta (noarch) +conda-build conda-forge-recipes/polars-config-meta \ + -c conda-forge --override-channels --variants "{python_min: ['3.10']}" + +# polars-bio, picking the dependency up from the local channel +conda-build . -c local -c conda-forge --override-channels --python 3.12 +``` + +For a build that matches bioconda CI more closely: + +```bash +conda create -n bioconda -c conda-forge -c bioconda bioconda-utils +conda activate bioconda +bioconda-utils lint --packages polars-bio +bioconda-utils build --docker --mulled-test --packages polars-bio +``` + +## Platforms + +Bioconda builds `linux-64` and `osx-64` by default; `osx-arm64` and +`linux-aarch64` require an explicit `extra: additional-platforms:` entry. The +recipe currently targets the defaults only — cross-compiling a Rust tree this +large is worth adding once the package is green, not as part of the initial +submission. Bioconda does not build Windows packages at all; Windows users +continue to install from PyPI. + +## Release automation + +`.github/workflows/update_bioconda_recipe.yml` opens a version-bump pull request +against `bioconda/bioconda-recipes` when a release is published. Bioconda's own +autobump bot usually does this unprompted once the package exists; the workflow +makes the bump deterministic rather than waiting on the bot. It requires a +`BIOCONDA_PAT` secret with `public_repo` scope. + +## Maintainers + +- @mwiewior + +## References + +- [Bioconda contributor guidelines](https://bioconda.github.io/contributor/guidelines.html) +- [Building locally](https://bioconda.github.io/contributor/building-locally.html) +- [conda-forge Rust knowledge base](https://conda-forge.org/docs/maintainer/knowledge_base.html#rust) diff --git a/bioconda-recipe/build.sh b/bioconda-recipe/build.sh new file mode 100644 index 00000000..eb5ed9da --- /dev/null +++ b/bioconda-recipe/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euxo pipefail + +# The source tree pins an exact toolchain for local development. The conda build +# supplies its own rustc and has no rustup to honour the pin with, so drop it. +rm -f rust-toolchain.toml + +# Upstream CI builds wheels with `-Ctarget-cpu=skylake` / `apple-m1` and +# `-Dwarnings`. Neither is appropriate for a redistributable package: the first +# emits instructions that fault on older hardware, the second turns any new +# compiler lint into a build failure. Build for the baseline architecture. +unset RUSTFLAGS || true + +export CARGO_PROFILE_RELEASE_DEBUG=false +export CARGO_PROFILE_RELEASE_STRIP=symbols +# Several dependencies resolve from git tags; the CLI honours the build +# environment's proxy and credential configuration where cargo's built-in +# fetcher does not. +export CARGO_NET_GIT_FETCH_WITH_CLI=true +export CARGO_NET_RETRY=5 + +# Record the licences of the vendored crate graph; referenced by license_file. +cargo-bundle-licenses --format yaml --output THIRDPARTY.yml + +$PYTHON -m pip install . -vv --no-deps --no-build-isolation diff --git a/bioconda-recipe/conda-forge-recipes/polars-config-meta/meta.yaml b/bioconda-recipe/conda-forge-recipes/polars-config-meta/meta.yaml new file mode 100644 index 00000000..5b7769ec --- /dev/null +++ b/bioconda-recipe/conda-forge-recipes/polars-config-meta/meta.yaml @@ -0,0 +1,57 @@ +{% set name = "polars-config-meta" %} +{% set version = "0.3.4" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + # The PyPI sdist omits the LICENSE file, which conda-forge requires, so the + # GitHub tag archive is used instead. + url: https://github.com/lmmx/{{ name }}/archive/refs/tags/{{ version }}.tar.gz + sha256: 175f3b981eb0ed1d9f387e811628f24edc6dbdc67aedf9e5fa7e4f160523771c + +build: + noarch: python + number: 0 + script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation + +requirements: + host: + - python {{ python_min }} + - uv-build >=0.11.6,<0.12 + - pip + run: + - python >={{ python_min }} + # Upstream lists polars only as an optional extra, but + # polars_config_meta/__init__.py imports it unconditionally, so it is a + # hard runtime requirement in practice. The lower bound matches the + # `polars` extra in pyproject.toml. + - polars >=1.30 + +test: + imports: + - polars_config_meta + commands: + - pip check + requires: + - pip + - python {{ python_min }} + +about: + home: https://github.com/lmmx/polars-config-meta + summary: A Polars plugin for persistent DataFrame-level metadata + description: | + polars-config-meta is a Polars plugin that attaches persistent, + DataFrame-level metadata to Polars DataFrames and LazyFrames. Metadata + survives operations that return new frames, which plain attribute + assignment cannot do. + license: MIT + license_family: MIT + license_file: LICENSE + doc_url: https://github.com/lmmx/polars-config-meta + dev_url: https://github.com/lmmx/polars-config-meta + +extra: + recipe-maintainers: + - mwiewior diff --git a/bioconda-recipe/meta.yaml b/bioconda-recipe/meta.yaml index c9302c71..7ba7c69f 100644 --- a/bioconda-recipe/meta.yaml +++ b/bioconda-recipe/meta.yaml @@ -1,72 +1,92 @@ {% set name = "polars-bio" %} -{% set version = "0.21.0" %} +{% set version = "0.33.0" %} package: name: {{ name|lower }} version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/polars_bio-{{ version }}.tar.gz - sha256: 868509e3bdc87723bab02f7d78109eb3a450e5d5623bf36a674395346f8f0e49 + url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/polars_bio-{{ version }}.tar.gz + sha256: 39f0dcd55660d20f37c464ac889cf8977d0eea4b9b433c135aa6244afab213b9 build: number: 0 - script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation - skip: true # [py<310] + # polars-bio requires Python >=3.11,<3.15 (pyproject.toml). + skip: true # [py < 311] + run_exports: + # Still 0.x with an evolving API, so pin downstream consumers to the + # minor version rather than the major one. + - {{ pin_subpackage(name, max_pin="x.x") }} requirements: build: - {{ compiler('c') }} - {{ compiler('cxx') }} + - {{ stdlib('c') }} - {{ compiler('rust') }} - cargo-bundle-licenses + - cmake + - make + - pkg-config + # The crate tree pulls several dependencies straight from git tags, so the + # build environment needs a git client. + - git - cross-python_{{ target_platform }} # [build_platform != target_platform] host: - python - pip - maturin >=1.0,<2.0 - - polars >=1.37.0 run: - python - polars >=1.37.1 - - pyarrow >=21.0.0,<23 - - datafusion >=50.0.0,<51 + - pyarrow >=23.0.1,<25 + - datafusion >=53.0.0,<54 - tqdm >=4.67.0,<5 - - typing-extensions >=4.14.0,<5 - polars-config-meta >=0.3.0,<1 test: imports: - polars_bio commands: - - python -c "import polars_bio; print(polars_bio.__version__)" - - python -c "import polars; import polars_bio; df = polars.DataFrame({'chr': ['chr1'], 'start': [100], 'end': [200]}); print('Basic DataFrame test passed')" + - pip check + # Exercise the compiled extension, not just the import. + - python -c "import polars_bio as pb; print(pb.__version__)" + - python -m pytest -q test_overlap.py + requires: + - pip + - pytest + - polars + files: + - test_overlap.py about: home: https://github.com/biodatageeks/polars-bio license: Apache-2.0 license_family: Apache - license_file: LICENSE + license_file: + - LICENSE + - THIRDPARTY.yml summary: 'Blazing fast genomic operations on large Python dataframes' description: | - polars-bio is a Python library for genomics built on top of polars, + polars-bio is a Python library for genomics built on top of Polars, Apache Arrow and Apache DataFusion. It provides a DataFrame API for - genomics data and is designed to be blazing fast, memory efficient - and easy to use. + genomics data and is designed to be fast, memory efficient and easy + to use. - Key Features: - - Optimized for performance and memory efficiency - - Popular genomics operations with DataFrame API - - SQL-powered bioinformatic data querying - - Native parallel engine powered by Apache DataFusion - - Out-of-core/streaming processing - - Support for federated and streamed reading from cloud storages - - Zero-copy data exchange with Apache Arrow - - Bioinformatics file formats support - - Pre-built wheels for Linux, Windows, and macOS + Key features: + - Popular genomic interval operations with a DataFrame API + - SQL-powered querying of bioinformatics data + - Native parallel engine powered by Apache DataFusion + - Out-of-core / streaming processing + - Reading from cloud storage + - Zero-copy data exchange with Apache Arrow + - Support for VCF, BAM, CRAM, SAM, FASTQ, FASTA, BED, GFF, GTF and + BigWig/BigBed doc_url: https://biodatageeks.org/polars-bio/ dev_url: https://github.com/biodatageeks/polars-bio extra: recipe-maintainers: - mwiewior + identifiers: + - doi:10.1093/bioinformatics/btaf640 diff --git a/bioconda-recipe/test_overlap.py b/bioconda-recipe/test_overlap.py new file mode 100644 index 00000000..4fc4970c --- /dev/null +++ b/bioconda-recipe/test_overlap.py @@ -0,0 +1,44 @@ +"""Smoke test run by the conda package test phase. + +Exercises the compiled Rust extension end to end rather than only importing it, +so a package that imports but cannot execute a query fails the build. +""" + +import polars as pl + +import polars_bio as pb + + +def test_overlap_finds_expected_pairs(): + df1 = pl.DataFrame( + { + "chrom": ["chr1", "chr1", "chr2"], + "start": [100, 500, 100], + "end": [200, 600, 200], + } + ) + df2 = pl.DataFrame( + { + "chrom": ["chr1", "chr2"], + "start": [150, 900], + "end": [250, 1000], + } + ) + + # Setting this explicitly keeps the test independent of the global + # coordinate-system default, and confirms the polars-config-meta + # dependency is wired up. + df1.config_meta.set(coordinate_system_zero_based=False) + df2.config_meta.set(coordinate_system_zero_based=False) + + result = pb.overlap(df1, df2, output_type="polars.DataFrame") + + # chr1:100-200 overlaps chr1:150-250. chr1:500-600 has no partner, and the + # chr2 intervals are disjoint, so exactly one pair is expected. + assert result.height == 1, result + + row = result.row(0, named=True) + assert row["chrom_1"] == "chr1" + assert row["start_1"] == 100 + assert row["chrom_2"] == "chr1" + assert row["start_2"] == 150 diff --git a/docs/superpowers/specs/2026-07-27-adoption-zero-config-first-run-design.md b/docs/superpowers/specs/2026-07-27-adoption-zero-config-first-run-design.md new file mode 100644 index 00000000..da224158 --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-adoption-zero-config-first-run-design.md @@ -0,0 +1,210 @@ +# Zero-config first run: conda distribution and parallel-by-default + +**Date:** 2026-07-27 +**Status:** Design — awaiting review +**Scope:** First sub-project of a larger adoption effort (see [Follow-on sub-projects](#follow-on-sub-projects)) + +## Problem + +polars-bio is at roughly 8.6k PyPI downloads/month. Its peers: bioframe 31k, pyranges 82k, +pybedtools 97k, pysam 1.17M. The library has a published Bioinformatics paper, a fast release +cadence, benchmark-backed performance claims, and a thorough test suite. + +A survey of all 137 GitHub issues (46 from external users, 22 distinct external reporters) found +**almost no complaints about performance or about the correctness of interval operations**. The +core value proposition holds up under real use. All observed friction is at the edges: installing +the library, loading the user's own files, and understanding why it is not as fast as advertised. + +This sub-project addresses the two edges that gate a new user's first five minutes: + +1. **polars-bio cannot be installed with conda.** It is on neither bioconda nor conda-forge, while + bioframe, pyranges, pybedtools and pysam all are. In bioinformatics, conda is the dominant + install channel: nf-core modules, Snakemake wrappers and biocontainers resolve packages from + bioconda, and a bioconda recipe automatically produces a Docker/Singularity biocontainer. Being + absent excludes polars-bio from every pipeline that follows those conventions. + +2. **Parallelism is off by default.** `polars_bio/context.py:36` sets + `datafusion.execution.target_partitions` to `"1"`, documented in `docs/features/parallel.md` as + "parallel execution disabled". A new user pip-installs, runs an overlap, and receives none of the + multicore speedup the README leads with. The headline claim ("native parallel engine", "282x + with 8 CPU cores") is contradicted by the out-of-the-box experience. + +Both are cheap to fix relative to their leverage, and both are prerequisites for the rest of the +adoption work: capability and documentation improvements only reach users who got past the install. + +## Goals + +- A user on any major platform can run `conda install -c bioconda polars-bio` and get a working + library. +- A user who writes no configuration gets multicore performance on a multicore machine. +- Neither change introduces a correctness regression. In particular, enabling parallelism by + default must not silently reorder written records. + +## Non-goals + +- Switching the default coordinate system to 0-based. This is a breaking change with its own + OpenSpec proposal (`openspec/changes/switch-default-to-zero-based-coordinates`) and deserves its + own release cycle. It is the natural next sub-project. +- Any new file format, interval operation, or analysis function. +- Loosening dependency pins beyond what conda packaging requires. + +## Design + +The work splits into two independent tracks that can proceed in parallel, plus one blocking +correctness fix that gates the second track. + +### Track A — conda distribution + +polars-bio already has an unshipped, untracked recipe in `bioconda-recipe/` and a release-triggered +workflow at `.github/workflows/update_bioconda_recipe.yml`. Neither has been submitted. The work is +to finish and land them. + +**A1. Publish `polars-config-meta` to conda-forge (prerequisite).** +polars-bio depends on `polars-config-meta>=0.3.0,<1`, which is not available on conda-forge or +bioconda. Verified against the Anaconda API on 2026-07-27: + +| Dependency | conda-forge | Notes | +|---|---|---| +| `polars` | 1.43.0 | available | +| `pyarrow` | 25.0.0 | available | +| `datafusion` | 54.0.0 | available; polars-bio pins `>=53,<54`, so the 53 line must still be resolvable | +| `polars-config-meta` | **missing** | blocks the bioconda recipe | + +`polars-config-meta` is a small pure-Python package, so this is a `noarch: python` recipe submitted +to `conda-forge/staged-recipes`. It is a dependency of polars-bio rather than a bioinformatics tool, +which is why conda-forge is the correct channel for it rather than bioconda. + +**A2. Submit polars-bio to bioconda.** +polars-bio is a compiled Rust extension, so the recipe builds from the sdist with a Rust toolchain +rather than repackaging wheels. Bioconda has established precedent for Rust-backed packages. The +recipe must pin `datafusion` to the 53 line to match `pyproject.toml`, and declare the same +`requires-python = ">=3.11,<3.15"` floor. + +**A3. Wire the release automation.** +`.github/workflows/update_bioconda_recipe.yml` already computes version and sha256 on release +publish. It needs to be committed, and its target changed from a local recipe edit to opening a pull +request against `bioconda/bioconda-recipes`. Note that bioconda's own autobump bot will often do +this unprompted once the package exists; the workflow's value is making the version bump +deterministic rather than waiting on the bot. + +**Verification.** The recipe passes `bioconda-utils lint`, builds under +`bioconda-utils build --docker --mulled-test`, and a post-merge smoke test installs from bioconda in +a clean environment and runs an overlap plus one read per format. + +### Track B — parallel by default + +**B1. Fix issue #421 first. This blocks B2 and is not optional.** + +Issue #421 documents that at `target_partitions > 1`, `write_bam` / `write_cram` / `write_sam` and +their `sink_*` variants do not preserve input row order, and that the output order is +nondeterministic across runs. The alignment write plans are single-partition +(`write_exec` uses `Partitioning::UnknownPartitioning(1)`), so DataFusion inserts a +`CoalescePartitionsExec` that merges parallel read partitions in *completion* order rather than +partition-index order. + +The issue explicitly notes that "the default `target_partitions` is `1`, so the test suite and +typical usage are unaffected". **Flipping the default converts this latent bug into a default-on +data-integrity bug**: every user writing alignments on a multicore machine would silently get +records in a nondeterministic order, and downstream tools that assume coordinate-sorted BAM would +need a re-sort. For a project whose stated positioning is correctness and verified parity, shipping +that would be self-defeating. + +The fix is to make the coalesce feeding the writer order-preserving — either by declaring an output +ordering on the read exec so DataFusion selects `SortPreservingMergeExec`, or by coalescing in +partition-index order in the write path. This lives in `datafusion-bio-formats` (bam/cram +`write_exec` plus `physical_exec`), followed by a version bump here. + +VCF write must be audited for the same failure mode; `tests/test_vcf_write.py` already exercises +`tp > 1` and is the natural place to assert it. + +**B2. Change the default to auto-detected cores.** + +Remove the hardcoded `"1"` from the `datafusion_conf` dict in `polars_bio/context.py`, letting +DataFusion apply its own default. Verified in `datafusion-common-53.1.0/src/config.rs:506`, that +default is `get_available_parallelism()`, which delegates to `std::thread::available_parallelism()` +(`utils/mod.rs:927`) and is memoized in a `LazyLock`. Two details matter: + +- `Context.__init__` builds both the Rust `BioSessionContext` and a mirrored + `datafusion.context.SessionConfig(datafusion_conf)`. Both must agree on the new default, or + Python-side and Rust-side plans will disagree on partition count. +- The upstream reader-thread fold (`datafusion-bio-formats` v1.8.7, merged in `25f4d47`) is what + makes this safe. Before it, each partition spawned its own reader thread and effective core usage + was roughly 2× `target_partitions`. Post-fix, effective cores track `target_partitions` at ~1×, + verified across four FastQC datasets. Without that fold, an auto-detected default would + oversubscribe the machine. + +**B3. Audit the test suite for tp-sensitivity.** + +The suite currently runs at `tp = 1` and therefore has never exercised the multi-partition path +broadly. Flipping the default runs everything at `tp = N`, which is the point — but it will surface +latent ordering and race assumptions. Two are already known: the `test_streaming.py` flakiness +(Arrow C Stream consumed race) and the singleton-`Context` option leak between tests that caused the +PR #420 BAM/CRAM failures. Tests that genuinely depend on single-partition behavior should set the +option explicitly via a fixture that restores the prior value, rather than relying on the global +default. + +**Verification.** A determinism test writes the same input at `tp = 1` and `tp = 8` across repeated +runs and asserts byte-identical or value-identical row order for BAM, CRAM, SAM and VCF. The +existing benchmark suite is re-run to confirm the default now reproduces published multicore +numbers without configuration. + +### Documentation + +`docs/features/parallel.md` currently states "The default value is **1** (parallel execution +disabled)" and must be rewritten to describe the new default and how to *reduce* parallelism, which +becomes the less common case. The README's performance claims become true as stated for a default +install. A changelog entry should call out the behavior change explicitly, since users who tuned +`target_partitions` around the old default may see different resource usage. + +## Risks + +**Bioconda build complexity.** Compiling a Rust extension with a large dependency tree inside +bioconda's build containers may hit toolchain or build-time limits. Mitigation: build and mulled-test +locally under Docker before submitting, and treat the conda-forge `polars-config-meta` recipe as the +independent first step so its progress is not blocked by polars-bio's build. + +**Raising the default surfaces unknown tp-sensitive bugs.** #421 is the one we know about; the +suite's limited multi-partition coverage means there may be others. Mitigation: B3 exists precisely +to find them, and the tracks are ordered so the default flip lands after the suite runs green at +`tp > 1`. + +**Resource usage changes for existing users.** Anyone who relied on the implicit single-threaded +default will see polars-bio consume all available cores. This is lower risk than it first appears: +`std::thread::available_parallelism()` respects cgroup v1/v2 CPU quotas and `sched_getaffinity` on +Linux, so containerized and HPC-scheduler-bound runs get their allotted share rather than the host +core count. Mitigation is therefore a prominent changelog entry plus one integration test asserting +the detected partition count under a constrained cgroup, rather than custom quota detection. + +## Sequencing + +Track A and Track B are independent and can run concurrently. Within B, B1 strictly precedes B2. + +1. A1 — `polars-config-meta` to conda-forge (unblocks A2; long external review latency, so start first) +2. B1 — fix #421 upstream, bump, verify (blocks B2) +3. A2 — polars-bio to bioconda +4. B3 — test-suite audit at `tp > 1` +5. B2 — flip the default, update docs and changelog +6. A3 — release automation + +## Follow-on sub-projects + +Listed in recommended order, each to get its own spec: + +1. **I/O trust on real-world files.** Roughly 22% of external issues are VCF field/INFO/genotype + parsing problems, several involving *silent* data loss rather than an error (#204 loaded the wrong + row count from Ensembl VCFs; #312 truncated multi-valued INFO). Nearly every reader bug traces to + a file shape absent from the test corpus: Ensembl, ClinVar, DeepVariant, Cell Ranger, 10X + CITE-seq, nanopore tags, multi-member gzip. Proposed: a real-world file corpus as a CI gate, plus + the multisample genotype ergonomics ask in #394. This is the highest-value follow-on, because + polars-bio's real competitive surface is the I/O layer where pysam does 1.17M downloads/month, and + trust there is earned only on files you did not choose. +2. **0-based coordinates by default.** Four independent reporters have been confused by the current + 1-based default (#259, #278, #413, #356). The OpenSpec change already exists. +3. **Capability pull.** No interval operation has any strand awareness — no `strandedness` or + `ignore_strand` parameter across all eight ops, which is pyranges' single biggest differentiator + and effectively required for RNA-seq and annotation work. Also: bedtools-style `map` aggregation + over overlapping features, writers for BED/GFF/GTF/BigWig (annotation round-trips are currently + impossible), int64 coordinates (#169, currently a 2Gb chromosome cap), and cloud sinks. +4. **Onboarding and positioning.** No migration guide from bioframe/pyranges/pybedtools, one + tutorial notebook, and issue #260 ("how is this different from just using Polars?") never + answered in the docs.