Skip to content

Commit df75cbb

Browse files
benjeffery-ofhbenjeffery-ofh
authored
Add preflight consistency checks (#59)
1 parent a3567df commit df75cbb

4 files changed

Lines changed: 432 additions & 11 deletions

File tree

‎tests/test_append.py‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
# tests/data/vcf/sample-variants.vcf.gz
2424

2525

26+
import numpy as np
2627
import pytest
28+
import zarr
2729

2830
from vczstore.append import append
2931

@@ -101,6 +103,252 @@ def test_append_fail_alleles_mismatch(tmp_path):
101103
append(vcz1, vcz2)
102104

103105

106+
def test_append_fails_for_misaligned_variant_chunks():
107+
store1 = zarr.storage.MemoryStore()
108+
root1 = zarr.create_group(store=store1)
109+
root1.create_array(
110+
"contig_id",
111+
data=np.array(["0"]),
112+
dimension_names=["contigs"],
113+
compressors=None,
114+
filters=None,
115+
)
116+
root1.create_array(
117+
"variant_contig",
118+
data=np.array([0, 0], dtype=np.int32),
119+
chunks=(2,),
120+
dimension_names=["variants"],
121+
compressors=None,
122+
filters=None,
123+
)
124+
root1.create_array(
125+
"variant_position",
126+
data=np.array([1, 2], dtype=np.int32),
127+
chunks=(2,),
128+
dimension_names=["variants"],
129+
compressors=None,
130+
filters=None,
131+
)
132+
root1.create_array(
133+
"variant_allele",
134+
data=np.array([["A", "T"], ["C", "G"]]),
135+
chunks=(2, 2),
136+
dimension_names=["variants", "alleles"],
137+
compressors=None,
138+
filters=None,
139+
)
140+
root1.create_array(
141+
"sample_id",
142+
data=np.array(["S1"]),
143+
dimension_names=["samples"],
144+
compressors=None,
145+
filters=None,
146+
)
147+
root1.create_array(
148+
"call_genotype",
149+
data=np.array([[[0, 1]], [[1, 1]]], dtype=np.int8),
150+
chunks=(1, 1, 2),
151+
dimension_names=["variants", "samples", "ploidy"],
152+
compressors=None,
153+
filters=None,
154+
)
155+
156+
store2 = zarr.storage.MemoryStore()
157+
root2 = zarr.create_group(store=store2)
158+
for name in ("contig_id", "variant_contig", "variant_position", "variant_allele"):
159+
root2.create_array(
160+
name,
161+
data=root1[name][:],
162+
chunks=root1[name].chunks,
163+
dimension_names=root1[name].metadata.dimension_names,
164+
compressors=None,
165+
filters=None,
166+
)
167+
root2.create_array(
168+
"sample_id",
169+
data=np.array(["S2"]),
170+
dimension_names=["samples"],
171+
compressors=None,
172+
filters=None,
173+
)
174+
root2.create_array(
175+
"call_genotype",
176+
data=np.array([[[0, 0]], [[0, 1]]], dtype=np.int8),
177+
chunks=(2, 1, 2),
178+
dimension_names=["variants", "samples", "ploidy"],
179+
compressors=None,
180+
filters=None,
181+
)
182+
183+
with pytest.raises(ValueError, match="VCZ-aligned variant chunks"):
184+
append(store1, store2)
185+
186+
root1_after = zarr.open_group(store=store1, mode="r")
187+
np.testing.assert_array_equal(root1_after["sample_id"][:], np.array(["S1"]))
188+
189+
190+
def test_append_fails_before_mutating_when_source_call_array_is_missing():
191+
store1 = zarr.storage.MemoryStore()
192+
root1 = zarr.create_group(store=store1)
193+
root1.create_array(
194+
"contig_id",
195+
data=np.array(["0"]),
196+
dimension_names=["contigs"],
197+
compressors=None,
198+
filters=None,
199+
)
200+
root1.create_array(
201+
"variant_contig",
202+
data=np.array([0, 0], dtype=np.int32),
203+
chunks=(2,),
204+
dimension_names=["variants"],
205+
compressors=None,
206+
filters=None,
207+
)
208+
root1.create_array(
209+
"variant_position",
210+
data=np.array([1, 2], dtype=np.int32),
211+
chunks=(2,),
212+
dimension_names=["variants"],
213+
compressors=None,
214+
filters=None,
215+
)
216+
root1.create_array(
217+
"variant_allele",
218+
data=np.array([["A", "T"], ["C", "G"]]),
219+
chunks=(2, 2),
220+
dimension_names=["variants", "alleles"],
221+
compressors=None,
222+
filters=None,
223+
)
224+
root1.create_array(
225+
"sample_id",
226+
data=np.array(["S1"]),
227+
dimension_names=["samples"],
228+
compressors=None,
229+
filters=None,
230+
)
231+
root1.create_array(
232+
"call_genotype",
233+
data=np.array([[[0, 1]], [[1, 1]]], dtype=np.int8),
234+
chunks=(2, 1, 2),
235+
dimension_names=["variants", "samples", "ploidy"],
236+
compressors=None,
237+
filters=None,
238+
)
239+
240+
store2 = zarr.storage.MemoryStore()
241+
root2 = zarr.create_group(store=store2)
242+
for name in ("contig_id", "variant_contig", "variant_position", "variant_allele"):
243+
root2.create_array(
244+
name,
245+
data=root1[name][:],
246+
chunks=root1[name].chunks,
247+
dimension_names=root1[name].metadata.dimension_names,
248+
compressors=None,
249+
filters=None,
250+
)
251+
root2.create_array(
252+
"sample_id",
253+
data=np.array(["S2"]),
254+
dimension_names=["samples"],
255+
compressors=None,
256+
filters=None,
257+
)
258+
259+
with pytest.raises(ValueError, match="present in both stores"):
260+
append(store1, store2)
261+
262+
root1_after = zarr.open_group(store=store1, mode="r")
263+
np.testing.assert_array_equal(root1_after["sample_id"][:], np.array(["S1"]))
264+
assert root1_after["call_genotype"].shape == (2, 1, 2)
265+
266+
267+
def test_append_fails_for_misaligned_source_variant_chunks():
268+
store1 = zarr.storage.MemoryStore()
269+
root1 = zarr.create_group(store=store1)
270+
root1.create_array(
271+
"contig_id",
272+
data=np.array(["0"]),
273+
dimension_names=["contigs"],
274+
compressors=None,
275+
filters=None,
276+
)
277+
root1.create_array(
278+
"variant_contig",
279+
data=np.array([0, 0], dtype=np.int32),
280+
chunks=(2,),
281+
dimension_names=["variants"],
282+
compressors=None,
283+
filters=None,
284+
)
285+
root1.create_array(
286+
"variant_position",
287+
data=np.array([1, 2], dtype=np.int32),
288+
chunks=(2,),
289+
dimension_names=["variants"],
290+
compressors=None,
291+
filters=None,
292+
)
293+
root1.create_array(
294+
"variant_allele",
295+
data=np.array([["A", "T"], ["C", "G"]]),
296+
chunks=(2, 2),
297+
dimension_names=["variants", "alleles"],
298+
compressors=None,
299+
filters=None,
300+
)
301+
root1.create_array(
302+
"sample_id",
303+
data=np.array(["S1"]),
304+
dimension_names=["samples"],
305+
compressors=None,
306+
filters=None,
307+
)
308+
root1.create_array(
309+
"call_genotype",
310+
data=np.array([[[0, 1]], [[1, 1]]], dtype=np.int8),
311+
chunks=(2, 1, 2),
312+
dimension_names=["variants", "samples", "ploidy"],
313+
compressors=None,
314+
filters=None,
315+
)
316+
317+
store2 = zarr.storage.MemoryStore()
318+
root2 = zarr.create_group(store=store2)
319+
for name in ("contig_id", "variant_contig", "variant_position", "variant_allele"):
320+
root2.create_array(
321+
name,
322+
data=root1[name][:],
323+
chunks=root1[name].chunks,
324+
dimension_names=root1[name].metadata.dimension_names,
325+
compressors=None,
326+
filters=None,
327+
)
328+
root2.create_array(
329+
"sample_id",
330+
data=np.array(["S2"]),
331+
dimension_names=["samples"],
332+
compressors=None,
333+
filters=None,
334+
)
335+
root2.create_array(
336+
"call_genotype",
337+
data=np.array([[[0, 0]], [[0, 1]]], dtype=np.int8),
338+
chunks=(1, 1, 2),
339+
dimension_names=["variants", "samples", "ploidy"],
340+
compressors=None,
341+
filters=None,
342+
)
343+
344+
with pytest.raises(ValueError, match="VCZ-aligned variant chunks"):
345+
append(store1, store2)
346+
347+
root1_after = zarr.open_group(store=store1, mode="r")
348+
np.testing.assert_array_equal(root1_after["sample_id"][:], np.array(["S1"]))
349+
assert root1_after["call_genotype"].shape == (2, 1, 2)
350+
351+
104352
def test_append_multiple_chunks(tmp_path):
105353
vcz1 = convert_vcf_to_vcz(
106354
"chr22-part1.vcf.gz", tmp_path, variants_chunk_size=10, samples_chunk_size=50

‎tests/test_remove.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
import numpy as np
12
import pytest
3+
import zarr
24

35
from vczstore.remove import remove
46

@@ -113,3 +115,87 @@ def test_remove_icechunk(tmp_path):
113115
session = repo.readonly_session("main")
114116
store = session.store
115117
check_removed_sample(store, "NA00002")
118+
119+
120+
def test_remove_fails_for_misaligned_variant_chunks():
121+
store = zarr.storage.MemoryStore()
122+
root = zarr.create_group(store=store)
123+
root.create_array(
124+
"variant_contig",
125+
data=np.array([0, 0], dtype=np.int32),
126+
chunks=(2,),
127+
dimension_names=["variants"],
128+
compressors=None,
129+
filters=None,
130+
)
131+
root.create_array(
132+
"variant_position",
133+
data=np.array([1, 2], dtype=np.int32),
134+
chunks=(2,),
135+
dimension_names=["variants"],
136+
compressors=None,
137+
filters=None,
138+
)
139+
root.create_array(
140+
"sample_id",
141+
data=np.array(["S1", "S2"]),
142+
dimension_names=["samples"],
143+
compressors=None,
144+
filters=None,
145+
)
146+
root.create_array(
147+
"call_genotype",
148+
data=np.array([[[0, 1], [1, 1]], [[0, 0], [0, 1]]], dtype=np.int8),
149+
chunks=(1, 2, 2),
150+
dimension_names=["variants", "samples", "ploidy"],
151+
compressors=None,
152+
filters=None,
153+
)
154+
155+
with pytest.raises(ValueError, match="VCZ-aligned variant chunks"):
156+
remove(store, "S1")
157+
158+
root_after = zarr.open_group(store=store, mode="r")
159+
np.testing.assert_array_equal(root_after["sample_id"][:], np.array(["S1", "S2"]))
160+
161+
162+
def test_remove_fails_for_malformed_call_array_dimensions():
163+
store = zarr.storage.MemoryStore()
164+
root = zarr.create_group(store=store)
165+
root.create_array(
166+
"variant_contig",
167+
data=np.array([0, 0], dtype=np.int32),
168+
chunks=(2,),
169+
dimension_names=["variants"],
170+
compressors=None,
171+
filters=None,
172+
)
173+
root.create_array(
174+
"variant_position",
175+
data=np.array([1, 2], dtype=np.int32),
176+
chunks=(2,),
177+
dimension_names=["variants"],
178+
compressors=None,
179+
filters=None,
180+
)
181+
root.create_array(
182+
"sample_id",
183+
data=np.array(["S1", "S2"]),
184+
dimension_names=["samples"],
185+
compressors=None,
186+
filters=None,
187+
)
188+
root.create_array(
189+
"call_quality",
190+
data=np.array([[10, 20], [30, 40]], dtype=np.int16),
191+
chunks=(2, 2),
192+
dimension_names=["variants", "not_samples"],
193+
compressors=None,
194+
filters=None,
195+
)
196+
197+
with pytest.raises(ValueError, match="variants/samples dimensions"):
198+
remove(store, "S1")
199+
200+
root_after = zarr.open_group(store=store, mode="r")
201+
np.testing.assert_array_equal(root_after["sample_id"][:], np.array(["S1", "S2"]))

0 commit comments

Comments
 (0)