From a145d722123af7f78bd67e1ebbc8120b1102526f Mon Sep 17 00:00:00 2001 From: Tom White Date: Mon, 20 Apr 2026 11:11:45 +0100 Subject: [PATCH 1/3] Fix case of no compression --- pyproject.toml | 5 ++- tests/test_normalise.py | 18 ++++++++++ uv.lock | 80 +++++++++++++++++++++++++++++++++++++++-- vczstore/normalise.py | 6 ++-- 4 files changed, 102 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 590d7d2..0553f9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ authors = [ {name = "sgkit Developers", email = "project@sgkit.dev"}, ] dependencies = [ - "bio2zarr @ git+https://github.com/sgkit-dev/bio2zarr.git", + "bio2zarr", "click>=8.2.0", "more_itertools", "numpy>=1.23.5,<2.4", @@ -113,3 +113,6 @@ known-third-party = [ "vcztools", "zarr" ] + +[tool.uv.sources] +bio2zarr = { path = "../bio2zarr", editable = true } diff --git a/tests/test_normalise.py b/tests/test_normalise.py index 6db4ba5..582a856 100644 --- a/tests/test_normalise.py +++ b/tests/test_normalise.py @@ -5,6 +5,7 @@ import zarr from numpy.testing import assert_array_equal +from vczstore.append import append from vczstore.normalise import ( index_variants, normalise, @@ -12,6 +13,8 @@ variant_alleles_are_equivalent, ) +from .utils import compare_vcf_and_vcz, convert_vcf_to_vcz + # TODO: replace this with make_vcz in vcztools/bio2zarr # see https://github.com/sgkit-dev/vczstore/issues/37 @@ -324,3 +327,18 @@ def test_normalise__other_call_fields_not_implemented(): with pytest.raises(NotImplementedError): normalise(vcz1, vcz2, vcz2_norm) + + +def test_normalise_and_append(tmp_path): + vcz0 = convert_vcf_to_vcz("sample-variants.vcf.gz", tmp_path, ploidy=2) + vcz1 = convert_vcf_to_vcz("sample-part1.vcf.gz", tmp_path) + vcz1_norm = zarr.storage.MemoryStore() + + normalise(vcz0, vcz1, vcz1_norm) + + append(vcz0, vcz1_norm) + + # check equivalence with original VCF + compare_vcf_and_vcz( + tmp_path, "view --no-version", "sample-part1.vcf.gz", "view --no-version", vcz0 + ) diff --git a/uv.lock b/uv.lock index 99d5ae3..f074f33 100644 --- a/uv.lock +++ b/uv.lock @@ -52,8 +52,7 @@ wheels = [ [[package]] name = "bio2zarr" -version = "0.2.0" -source = { git = "https://github.com/sgkit-dev/bio2zarr.git#fb26d23d1f091a6a7f85773de0ceadf80fe8a332" } +source = { editable = "../bio2zarr" } dependencies = [ { name = "click" }, { name = "coloredlogs" }, @@ -65,6 +64,81 @@ dependencies = [ { name = "zarr" }, ] +[package.metadata] +requires-dist = [ + { name = "click" }, + { name = "coloredlogs" }, + { name = "cyvcf2", marker = "extra == 'all'" }, + { name = "cyvcf2", marker = "extra == 'vcf'" }, + { name = "humanfriendly" }, + { name = "numpy", specifier = ">=2" }, + { name = "pandas" }, + { name = "tabulate" }, + { name = "tqdm" }, + { name = "tskit", marker = "extra == 'all'", specifier = ">=1" }, + { name = "tskit", marker = "extra == 'tskit'", specifier = ">=1" }, + { name = "zarr", specifier = ">=3.1" }, +] +provides-extras = ["tskit", "vcf", "all"] + +[package.metadata.requires-dev] +dev = [ + { name = "asciinema-automation" }, + { name = "bash-kernel" }, + { name = "bed-reader" }, + { name = "click", specifier = ">=8.2.0" }, + { name = "cyvcf2", marker = "sys_platform != 'win32'" }, + { name = "dask", extras = ["array"] }, + { name = "hypothesis-vcf" }, + { name = "jupyter-book", specifier = "<2" }, + { name = "msprime" }, + { name = "prek", specifier = "==0.3.3" }, + { name = "pysam", marker = "sys_platform != 'win32'" }, + { name = "pytest" }, + { name = "pytest-coverage" }, + { name = "pytest-xdist" }, + { name = "ruff", specifier = "==0.15.1" }, + { name = "sphinx-click" }, + { name = "tqdm" }, + { name = "tskit", specifier = ">=1" }, + { name = "twine" }, + { name = "validate-pyproject", extras = ["all"] }, + { name = "xarray", specifier = "<2025.3.1" }, + { name = "zizmor", specifier = "==1.24.1" }, +] +docs = [ + { name = "asciinema-automation" }, + { name = "bash-kernel" }, + { name = "cyvcf2", marker = "sys_platform != 'win32'" }, + { name = "jupyter-book", specifier = "<2" }, + { name = "sphinx-click" }, + { name = "tskit", specifier = ">=1" }, +] +lint = [ + { name = "prek", specifier = "==0.3.3" }, + { name = "ruff", specifier = "==0.15.1" }, + { name = "zizmor", specifier = "==1.24.1" }, +] +packaging = [ + { name = "twine" }, + { name = "validate-pyproject", extras = ["all"] }, +] +test = [ + { name = "bed-reader" }, + { name = "click", specifier = ">=8.2.0" }, + { name = "cyvcf2", marker = "sys_platform != 'win32'" }, + { name = "dask", extras = ["array"] }, + { name = "hypothesis-vcf" }, + { name = "msprime" }, + { name = "pysam", marker = "sys_platform != 'win32'" }, + { name = "pytest" }, + { name = "pytest-coverage" }, + { name = "pytest-xdist" }, + { name = "tqdm" }, + { name = "tskit", specifier = ">=1" }, + { name = "xarray", specifier = "<2025.3.1" }, +] + [[package]] name = "certifi" version = "2026.2.25" @@ -1591,7 +1665,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "bio2zarr", git = "https://github.com/sgkit-dev/bio2zarr.git" }, + { name = "bio2zarr", editable = "../bio2zarr" }, { name = "click", specifier = ">=8.2.0" }, { name = "hypothesis", marker = "extra == 'icechunk'" }, { name = "icechunk", marker = "extra == 'icechunk'" }, diff --git a/vczstore/normalise.py b/vczstore/normalise.py index bd5f1e5..be83ff6 100644 --- a/vczstore/normalise.py +++ b/vczstore/normalise.py @@ -1,6 +1,6 @@ import numpy as np import zarr -from bio2zarr.zarr_utils import create_empty_group_array, get_compressor +from bio2zarr.zarr_utils import create_empty_group_array, get_compressor_config from more_itertools import peekable from vcztools.constants import STR_FILL, STR_MISSING from vcztools.retrieval import VczReader @@ -51,7 +51,7 @@ def normalise(vcz1, vcz2, vcz2_norm): shape=shape, dtype=arr.dtype, chunks=chunks, - compressor=get_compressor(arr), + compressor=get_compressor_config(arr), dimension_names=array_dims(arr), ) else: @@ -66,7 +66,7 @@ def normalise(vcz1, vcz2, vcz2_norm): shape=arr.shape, dtype=arr.dtype, chunks=arr.chunks, - compressor=get_compressor(arr), + compressor=get_compressor_config(arr), dimension_names=array_dims(arr), ) From b6d2ee28f077d8167afcdb9b0978db806f15727f Mon Sep 17 00:00:00 2001 From: Tom White Date: Mon, 20 Apr 2026 15:00:53 +0100 Subject: [PATCH 2/3] Add normalise test using VCF data --- pyproject.toml | 5 +--- uv.lock | 80 ++------------------------------------------------ 2 files changed, 4 insertions(+), 81 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0553f9f..590d7d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ authors = [ {name = "sgkit Developers", email = "project@sgkit.dev"}, ] dependencies = [ - "bio2zarr", + "bio2zarr @ git+https://github.com/sgkit-dev/bio2zarr.git", "click>=8.2.0", "more_itertools", "numpy>=1.23.5,<2.4", @@ -113,6 +113,3 @@ known-third-party = [ "vcztools", "zarr" ] - -[tool.uv.sources] -bio2zarr = { path = "../bio2zarr", editable = true } diff --git a/uv.lock b/uv.lock index f074f33..99d5ae3 100644 --- a/uv.lock +++ b/uv.lock @@ -52,7 +52,8 @@ wheels = [ [[package]] name = "bio2zarr" -source = { editable = "../bio2zarr" } +version = "0.2.0" +source = { git = "https://github.com/sgkit-dev/bio2zarr.git#fb26d23d1f091a6a7f85773de0ceadf80fe8a332" } dependencies = [ { name = "click" }, { name = "coloredlogs" }, @@ -64,81 +65,6 @@ dependencies = [ { name = "zarr" }, ] -[package.metadata] -requires-dist = [ - { name = "click" }, - { name = "coloredlogs" }, - { name = "cyvcf2", marker = "extra == 'all'" }, - { name = "cyvcf2", marker = "extra == 'vcf'" }, - { name = "humanfriendly" }, - { name = "numpy", specifier = ">=2" }, - { name = "pandas" }, - { name = "tabulate" }, - { name = "tqdm" }, - { name = "tskit", marker = "extra == 'all'", specifier = ">=1" }, - { name = "tskit", marker = "extra == 'tskit'", specifier = ">=1" }, - { name = "zarr", specifier = ">=3.1" }, -] -provides-extras = ["tskit", "vcf", "all"] - -[package.metadata.requires-dev] -dev = [ - { name = "asciinema-automation" }, - { name = "bash-kernel" }, - { name = "bed-reader" }, - { name = "click", specifier = ">=8.2.0" }, - { name = "cyvcf2", marker = "sys_platform != 'win32'" }, - { name = "dask", extras = ["array"] }, - { name = "hypothesis-vcf" }, - { name = "jupyter-book", specifier = "<2" }, - { name = "msprime" }, - { name = "prek", specifier = "==0.3.3" }, - { name = "pysam", marker = "sys_platform != 'win32'" }, - { name = "pytest" }, - { name = "pytest-coverage" }, - { name = "pytest-xdist" }, - { name = "ruff", specifier = "==0.15.1" }, - { name = "sphinx-click" }, - { name = "tqdm" }, - { name = "tskit", specifier = ">=1" }, - { name = "twine" }, - { name = "validate-pyproject", extras = ["all"] }, - { name = "xarray", specifier = "<2025.3.1" }, - { name = "zizmor", specifier = "==1.24.1" }, -] -docs = [ - { name = "asciinema-automation" }, - { name = "bash-kernel" }, - { name = "cyvcf2", marker = "sys_platform != 'win32'" }, - { name = "jupyter-book", specifier = "<2" }, - { name = "sphinx-click" }, - { name = "tskit", specifier = ">=1" }, -] -lint = [ - { name = "prek", specifier = "==0.3.3" }, - { name = "ruff", specifier = "==0.15.1" }, - { name = "zizmor", specifier = "==1.24.1" }, -] -packaging = [ - { name = "twine" }, - { name = "validate-pyproject", extras = ["all"] }, -] -test = [ - { name = "bed-reader" }, - { name = "click", specifier = ">=8.2.0" }, - { name = "cyvcf2", marker = "sys_platform != 'win32'" }, - { name = "dask", extras = ["array"] }, - { name = "hypothesis-vcf" }, - { name = "msprime" }, - { name = "pysam", marker = "sys_platform != 'win32'" }, - { name = "pytest" }, - { name = "pytest-coverage" }, - { name = "pytest-xdist" }, - { name = "tqdm" }, - { name = "tskit", specifier = ">=1" }, - { name = "xarray", specifier = "<2025.3.1" }, -] - [[package]] name = "certifi" version = "2026.2.25" @@ -1665,7 +1591,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "bio2zarr", editable = "../bio2zarr" }, + { name = "bio2zarr", git = "https://github.com/sgkit-dev/bio2zarr.git" }, { name = "click", specifier = ">=8.2.0" }, { name = "hypothesis", marker = "extra == 'icechunk'" }, { name = "icechunk", marker = "extra == 'icechunk'" }, From d73d3f2bb795613041371b0fd8dd6c5a71bfe50a Mon Sep 17 00:00:00 2001 From: Tom White Date: Mon, 20 Apr 2026 15:10:54 +0100 Subject: [PATCH 3/3] Update bio2zarr --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 99d5ae3..8627741 100644 --- a/uv.lock +++ b/uv.lock @@ -52,8 +52,8 @@ wheels = [ [[package]] name = "bio2zarr" -version = "0.2.0" -source = { git = "https://github.com/sgkit-dev/bio2zarr.git#fb26d23d1f091a6a7f85773de0ceadf80fe8a332" } +version = "0.2.1.dev2" +source = { git = "https://github.com/sgkit-dev/bio2zarr.git#4b50f8803d05ac7ea0a3fa8537cc7c9a3389493f" } dependencies = [ { name = "click" }, { name = "coloredlogs" },