From 576aac4c7408e074d96b49624a37440ac2a047be Mon Sep 17 00:00:00 2001 From: NingyuSun Date: Fri, 9 Oct 2026 13:59:12 +0900 Subject: [PATCH] Add the first expert-review round (#23) One blinded workbook for the three reviews that wait on experts: 40 ClinVar classifications (half divergent, half control, unmarked), the 60-claim extraction sample, and a fresh single-cell audit sample (59 auto-admitted, 10 others). Every answer is a dropdown with plain options, the start sheet gives six rules in English and Chinese, and items of parts B and C get new codes. import_round.py writes each part back into its review's own format and refuses half-answered rows, unknown options and a second import of the same reviewer. --- CHANGELOG.md | 4 + evaluation/expert_round/README.md | 47 ++++ evaluation/expert_round/import_round.py | 187 +++++++++++++++ evaluation/expert_round/make_round.py | 291 ++++++++++++++++++++++++ tests/test_expert_round.py | 110 +++++++++ 5 files changed, 639 insertions(+) create mode 100644 evaluation/expert_round/README.md create mode 100644 evaluation/expert_round/import_round.py create mode 100644 evaluation/expert_round/make_round.py create mode 100644 tests/test_expert_round.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4019a80..c10ea8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Add the first expert-review round (`evaluation/expert_round`, #23): one blinded workbook for the three reviews + that wait on experts (40 ClinVar classifications, the 60-claim extraction sample, a fresh 69-cluster single-cell + audit sample), with every answer a dropdown and instructions in English and Chinese, and an importer that + writes each part back into its review's own format, refusing half-answered rows and repeat imports. - Add model reviewers for the blinded ClinVar review packet (#17, #22). `model_reviewers.py` gives the six benchmark models exactly what experts get, one isolated session per case, with tools disabled or detected, stops a model whose account is out of quota, and lets Claude run on a different install than Codex and Antigravity. All six diff --git a/evaluation/expert_round/README.md b/evaluation/expert_round/README.md new file mode 100644 index 0000000..9e2e7fc --- /dev/null +++ b/evaluation/expert_round/README.md @@ -0,0 +1,47 @@ +# Expert review, round 1 + +One workbook that experts fill in, for the three reviews that wait on them (#23). Each expert does the parts in +their field: + +| Part | Items | For | Feeds | +|---|---:|---|---| +| A · Variants | 40 ClinVar germline classifications (September 2023): 20 where the validator and NCBI's review status disagree, 20 where they agree, unmarked | Clinical or molecular geneticists | The [ClinVar review kit](../clinvar_review/README.md), and the scoring of the six model reviewers (#22) | +| B · Paper claims | The 60-claim expert sample of the [extraction experiment](../../examples/civic_extraction/README.md) | Cancer genomics, variant curation | Natural extraction errors, labelled with the [error taxonomy](../../docs/ERROR_TAXONOMY.md) (#21) | +| C · Single-cell | A fresh audit sample: 59 auto-admitted cluster annotations and 10 others, unmarked | Single-cell biologists | The [audit](../../docs/GOLD_STANDARD.md#auditing-what-is-admitted-automatically) of what bioevidence admits (#24) | + +Every answer is a dropdown with plain options ("yes / no / unsure", "usable as is / needs a curator's check / not +usable"); a comment is optional. The start sheet gives six rules in English and Chinese. + +## For the maintainer + +```bash +# Build the round. Keep the seed and maintainer/ private: they unblind the items. +uv run --frozen python evaluation/expert_round/make_round.py \ + --clinvar-packet artifacts/clinvar-review/reviewer_packet \ + --clinvar-key artifacts/clinvar-review/maintainer/key.json \ + --seed --output artifacts/expert-round-1 + +# Import each returned workbook (one per expert) +uv run --frozen python evaluation/expert_round/import_round.py returned_R1.xlsx \ + --key artifacts/expert-round-1/maintainer/round_key.json \ + --clinvar-key artifacts/clinvar-review/maintainer/key.json --output artifacts/expert-round-1/returned +``` + +Then each part goes on in its own tools: +- **Part A:** `bioevidence review agreement` and `adjudication-sheet` on `clinvar_annotations.csv`; `bioevidence + review score` against `maintainer/predictions.csv`; `analyze_models.py` with the expert labels. +- **Part B:** `extraction_labels.csv`, joined with `results/extraction-test/expert_sample/key.jsonl`. +- **Part C:** `bioevidence review audit-score --manifest maintainer/singlecell_audit_manifest.json` on + `singlecell_annotations.csv`. + +Blinding. The workbook shows no model, no route, no validator decision and no reference label. Parts B and C get +new item codes, so the public results cannot be looked up by code. The experts are also asked not to look at this +project's results until they are done. + +Strictness. The importer rejects: +- a half-answered row; +- an answer that is not a dropdown option; +- a missing reviewer ID or qualification; +- a second import of the same reviewer. + +Nothing is written unless every part is clean. diff --git a/evaluation/expert_round/import_round.py b/evaluation/expert_round/import_round.py new file mode 100644 index 0000000..bd120bf --- /dev/null +++ b/evaluation/expert_round/import_round.py @@ -0,0 +1,187 @@ +"""Read a returned expert-review workbook back into each review's own format (the counterpart of make_round.py). + + uv run --frozen python evaluation/expert_round/import_round.py returned_R1.xlsx \ + --key artifacts/expert-round-1/maintainer/round_key.json \ + --clinvar-key artifacts/clinvar-review/maintainer/key.json \ + --reviewer-id R1 --output artifacts/expert-round-1/returned + +Each part the expert filled in is appended to its own file; blank rows (parts outside their field) are skipped: +- Part A -> `clinvar_annotations.csv`, the protocol annotation format of the ClinVar review kit + (`bioevidence review agreement`, `adjudication-sheet`, `score`); +- Part B -> `extraction_labels.csv`, the labelling format of the extraction expert sample; +- Part C -> `singlecell_annotations.csv`, protocol annotations for `bioevidence review audit-score` with + `maintainer/singlecell_audit_manifest.json`. + +A half-answered row, or an answer that is not one of the dropdown's options, is an error; nothing is written then. +""" +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(REPO / "evaluation" / "clinvar_review")) +import import_sheets # noqa: E402 +import make_round as mk # noqa: E402 + +from bioevidence_validator import review # noqa: E402 + +EXTRACTION_COLUMNS = ["sample_id", "pmid", "claim_correct", "error_codes", "rationale", "reviewer_id", + "reviewer_qualification", "annotated_at", "minutes_spent"] + + +def read_sheet(sheet) -> list[dict[str, Any]]: + header = [c.value for c in sheet[mk.HEADER_ROW]] + rows = [] + for values in sheet.iter_rows(min_row=mk.HEADER_ROW + 1, values_only=True): + row = {h: ("" if v is None else str(v).strip()) for h, v in zip(header, values, strict=False) if h} + if row.get("Item"): + rows.append(row) + return rows + + +def answers(row: dict[str, str], name: str) -> dict[str, str]: + """The answer columns of one row (questions only), checked against the dropdown options.""" + out = {} + for title, _, choices in mk.COLUMNS[name]: + if isinstance(choices, dict): + value = row.get(title, "") + if value and value not in choices: + raise ValueError(f"{row['Item']}: {value!r} is not an option for {title!r}") + out[title] = value + return out + + +def reviewer(book, override: dict[str, str]) -> dict[str, str]: + start = book["Start here"] + found = {"id": start["B4"].value, "qualification": start["B5"].value, "date": start["B6"].value} + found = {k: ("" if v is None else str(v).strip()) for k, v in found.items()} + found.update({k: v for k, v in override.items() if v}) + if not found["id"] or not found["qualification"]: + raise ValueError("The reviewer's ID and one-line qualification are needed (on the Start sheet or as options)") + date = found["date"][:10] or dt.date.today().isoformat() + try: + dt.date.fromisoformat(date) + except ValueError: + raise ValueError(f"Date finished must be YYYY-MM-DD, not {found['date']!r}") from None + found["date"] = date + return found + + +def convert(book, key: dict, clinvar_key: dict | None, who: dict[str, str]) -> dict[str, list[dict[str, str]]]: + out: dict[str, list[dict[str, str]]] = {"A": [], "B": [], "C": []} + errors: list[str] = [] + for name, part in mk.PARTS.items(): + for row in read_sheet(book[part["sheet"]]): + try: + given = answers(row, name) + except ValueError as exc: + errors.append(str(exc)) + continue + if not any(given.values()): + continue # not reviewed: a part outside this expert's field + questions = [t for t, _, c in mk.COLUMNS[name] if isinstance(c, dict)] + needed = questions if name != "B" else questions[:1] # B's Q2 only applies when Q1 is "no" + missing = [q for q in needed if not given[q]] + if missing: + errors.append(f"{row['Item']}: unanswered {missing}") + continue + source = key["items"][name].get(row["Item"]) + if source is None: + errors.append(f"{row['Item']}: not an item of this round") + continue + comment = row.get("Comment (optional)", "") + if name == "A": + q = [given[t] for t in questions] + out["A"].append({"case_code": source["case_code"], "statement_label": mk.STATEMENT[q[0]], + **{use: mk.USE[v] for use, v in zip(import_sheets.USES, q[1:], strict=True)}, + "statement_rationale": comment, "sources_consulted": "", "later_information_seen": "no"}) + elif name == "B": + verdict, problem = given[questions[0]], given[questions[1]] + out["B"].append({"sample_id": source["sample_id"], "pmid": source["pmid"], "claim_correct": verdict, + "error_codes": mk.PROBLEMS[problem] if problem and verdict == "no" else "", + "rationale": comment, "reviewer_id": who["id"], + "reviewer_qualification": who["qualification"], "annotated_at": who["date"], + "minutes_spent": ""}) + else: + verdict, use = given[questions[0]], given[questions[1]] + out["C"].append({**{c: "" for c in review.ANNOTATION_COLUMNS}, **source, + "annotation_id": f"{who['id']}:{row['Item']}", "group_id": source["case_id"], + "reviewer_id": who["id"], "reviewer_qualification": who["qualification"], + "annotated_at": who["date"], "mapping_label": mk.STATEMENT[verdict], + "mapping_rationale": comment, "admission_label": mk.USE[use], + "admission_rationale": comment, "evidence_refs_json": json.dumps(["expert review round 1"])}) + if errors: + raise ValueError("Cannot import:\n" + "\n".join(errors[:30])) + if out["A"]: + if clinvar_key is None: + raise ValueError("Part A was answered: --clinvar-key is needed to import it") + out["A"], _ = import_sheets.convert(out["A"], clinvar_key, reviewer_id=who["id"], + qualification=who["qualification"], annotated_at=who["date"]) + return out + + +def append_csv(path: Path, rows: list[dict[str, str]], columns: list[str], key_columns: tuple[str, ...]) -> None: + existing = [] + if path.exists(): + with path.open(encoding="utf-8", newline="") as handle: + existing = list(csv.DictReader(handle)) + seen = {tuple(r[c] for c in key_columns) for r in existing} + clash = [r for r in rows if tuple(r[c] for c in key_columns) in seen] + if clash: + raise ValueError(f"{path.name} already holds this reviewer's answer for {clash[0][key_columns[0]]}") + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=columns, lineterminator="\n") + writer.writeheader() + writer.writerows(existing + rows) + + +def main(argv: list[str] | None = None) -> int: + from openpyxl import load_workbook + + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("workbook", type=Path) + parser.add_argument("--key", type=Path, required=True, help="maintainer/round_key.json from make_round.py") + parser.add_argument("--clinvar-key", type=Path, help="The ClinVar kit's maintainer key (for part A)") + parser.add_argument("--reviewer-id", default="", help="Overrides the Start sheet") + parser.add_argument("--qualification", default="", help="Overrides the Start sheet") + parser.add_argument("--annotated-at", default="", help="YYYY-MM-DD; overrides the Start sheet") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + book = load_workbook(args.workbook, read_only=False) + who = reviewer(book, {"id": args.reviewer_id, "qualification": args.qualification, "date": args.annotated_at}) + key = json.loads(args.key.read_text(encoding="utf-8")) + clinvar_key = json.loads(args.clinvar_key.read_text(encoding="utf-8")) if args.clinvar_key else None + parts = convert(book, key, clinvar_key, who) + args.output.mkdir(parents=True, exist_ok=True) + # Refuse a second import of the same reviewer before anything is written, so no part is imported alone. + for name, file, columns in (("A", "clinvar_annotations.csv", ("annotation_id",)), + ("B", "extraction_labels.csv", ("sample_id", "reviewer_id")), + ("C", "singlecell_annotations.csv", ("case_id", "reviewer_id"))): + if parts[name] and (args.output / file).exists(): + with (args.output / file).open(encoding="utf-8", newline="") as handle: + seen = {tuple(r[c] for c in columns) for r in csv.DictReader(handle)} + if any(tuple(r[c] for c in columns) in seen for r in parts[name]): + raise ValueError(f"{file} already holds answers from {who['id']}; nothing was imported") + if parts["A"]: + import_sheets.append(args.output / "clinvar_annotations.csv", parts["A"]) + if parts["B"]: + append_csv(args.output / "extraction_labels.csv", parts["B"], EXTRACTION_COLUMNS, ("sample_id", "reviewer_id")) + if parts["C"]: + columns = review.ANNOTATION_COLUMNS + review.OPTIONAL_ANNOTATION_COLUMNS + append_csv(args.output / "singlecell_annotations.csv", parts["C"], columns, ("case_id", "reviewer_id")) + review.load_annotations(args.output / "singlecell_annotations.csv") # the protocol's strict check + print(f"{who['id']}: part A {len(parts['A']) // 3} case(s), part B {len(parts['B'])} claim(s), " + f"part C {len(parts['C'])} cluster(s) imported into {args.output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evaluation/expert_round/make_round.py b/evaluation/expert_round/make_round.py new file mode 100644 index 0000000..fa3d573 --- /dev/null +++ b/evaluation/expert_round/make_round.py @@ -0,0 +1,291 @@ +"""Build one expert-review workbook for three pending reviews, and the private key that maps it back (#23). + + uv run --frozen python evaluation/expert_round/make_round.py \ + --clinvar-packet artifacts/clinvar-review/reviewer_packet \ + --clinvar-key artifacts/clinvar-review/maintainer/key.json \ + --seed --output artifacts/expert-round-1 + +The workbook (`expert_review_round1.xlsx`) is what experts get. It has one sheet per part, so each expert does the +parts in their field, and every answer is a dropdown: +- Part A: ClinVar germline classifications as of September 2023: half where the validator and NCBI's review + status disagree, half where they agree, in random order (the ClinVar review kit, #23); +- Part B: claims models extracted from cancer papers (the 60-claim expert sample of the extraction test, #21); +- Part C: single-cell cluster annotations: a fresh random sample of auto-admitted records with a few others + mixed in (the audit of #24). + +Nothing in the workbook shows which model made a claim, how bioevidence decided, or what the reference says. Items +of parts B and C get new codes, so the public results cannot be looked up by code. `maintainer/` holds the key, +the audit manifest and the seed: **keep it from the experts.** `import_round.py` reads returned workbooks back +into each review's own format. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import random +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +BENCH = REPO / "evaluation" / "llm_benchmark" +sys.path.insert(0, str(BENCH)) + +ROUND = "expert-review-round-1" +CLINVAR_CASES = 40 # half divergent, half control, from the test split +WORKBOOK = "expert_review_round1.xlsx" + +# What experts pick, and what each choice means in the review formats. Plain words, so the dropdowns read easily. +STATEMENT = {"yes": "correct", "no": "incorrect", "unsure": "uncertain"} +USE = {"usable as is": "admitted", "needs a curator's check": "review_required", "not usable": "rejected"} +CLAIM = {"yes": "yes", "no": "no", "unsure": "unsure"} +PROBLEMS = { # the extraction-relevant failure modes of evaluation/error_taxonomy.yaml + "Wrong gene, variant or disease": "EXT-5", "Wrong relation or effect": "EXT-6", "Opposite direction": "EXT-2", + "Overstated certainty": "EXT-3", "Wrong population (e.g. only cell lines or mice)": "EXT-4", + "The quotes do not show it": "EXT-1", "Not this paper's own result": "SRC-2", "Other (see comment)": "OTHER", +} + +PARTS = { + "A": {"sheet": "A · Variants", "title": "Part A · Germline variant classifications (ClinVar, September 2023)", + "who": "Clinical or molecular geneticist, variant curator", "minutes": "about 2 hours"}, + "B": {"sheet": "B · Paper claims", "title": "Part B · Claims extracted from cancer papers", + "who": "Cancer genomics, clinical variant curation", "minutes": "about 1.5 hours"}, + "C": {"sheet": "C · Single-cell", "title": "Part C · Single-cell cluster annotations", + "who": "Single-cell biology, cell types", "minutes": "about 1 hour"}, +} +RULES = [ + ("Do only the parts in your field; leave the other sheets blank.", "只做您专业相关的部分，其他工作表留空即可。"), + ("Work alone. Do not discuss items with other reviewers until everyone has sent their file.", + "请独立完成；所有人提交之前，不要和其他审稿人讨论。"), + ("Judge only what is shown. Do not look up the variant's current ClinVar page or this project's results.", + "只根据表中给出的信息判断；不要查询变异当前的 ClinVar 页面，也不要查看本项目的结果。"), + ("Pick each answer from the dropdown in the yellow cells. 'unsure' is a valid answer.", + "在黄色单元格的下拉菜单里选择答案；“unsure（不确定）”也是有效答案。"), + ("A comment is optional, but one phrase helps when you answer 'no' or 'not usable'.", + "备注可选；选择 no 或 not usable 时，写一句理由会很有帮助。"), + ("Fill in your name or ID and one line on your experience above, then save and send the file back.", + "在上方填写姓名或代号和一句话的专业背景，保存后把文件发回即可。"), +] +GUIDES = { + "A": ["Evidence: every submission ClinVar held for the variant in September 2023 (submitter, criteria, " + "classification, method, date). Variant IDs and ClinVar's own rating are hidden on purpose.", + "Q1: taken together, do these submissions support calling the variant pathogenic or likely pathogenic " + "(germline)? Q2–Q4: may the P/LP classification be used, as it stands, for:", + "Q2 a research summary (cited as reported, with ordinary caveats) · Q3 a clinical knowledge base, without " + "its own re-curation · Q4 an expert-level reference, such as a benchmark or training label."], + "B": ["A model read the paper and extracted the claim. The quotes are the sentences it cited, with the paragraph " + "each comes from; the PMID links to the paper if you need more.", + "Q1: does the paper report this claim as stated: gene, variant, kind of effect, disease and therapies? " + "Q2: if not, what is the main problem?"], + "C": ["Each row is one cluster of a single-cell RNA-seq dataset: its top marker genes (log fold change; share " + "of cells expressing it inside vs outside the cluster) and the cell type a model assigned.", + "Q1: is that cell type right for this cluster? A correct but broader type counts as 'yes'. " + "Q2: may this annotation be used, as it stands, in a research summary?"], +} +COLUMNS = { + "A": [("Item", 10, None), ("Gene", 10, None), ("Submissions", 12, None), ("ClinVar submissions, September 2023", 80, None), + ("Q1 · P/LP is the right summary?", 18, STATEMENT), ("Q2 · Research summary", 22, USE), + ("Q3 · Clinical reference", 22, USE), ("Q4 · Expert reference", 22, USE), ("Comment (optional)", 36, "text")], + "B": [("Item", 10, None), ("Paper", 14, None), ("Claim", 60, None), ("Quotes cited, with their paragraphs", 90, None), + ("Q1 · The paper reports this claim?", 18, CLAIM), ("Q2 · If no: main problem", 34, PROBLEMS), + ("Comment (optional)", 36, "text")], + "C": [("Item", 10, None), ("Dataset", 26, None), ("Top marker genes", 70, None), ("Assigned cell type", 30, None), + ("Markers the model cited", 40, None), ("Q1 · Right cell type?", 16, CLAIM), + ("Q2 · Usable in a research summary?", 24, USE), ("Comment (optional)", 36, "text")], +} +HEADER_ROW = len(max(GUIDES.values(), key=len)) + 2 # guide lines, a blank line, then the header + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def clinvar_items(packet: Path, key: dict, rng: random.Random) -> list[dict[str, Any]]: + with (packet / "labels.csv").open(encoding="utf-8", newline="") as handle: + rows = {r["case_code"]: r for r in csv.DictReader(handle)} + picked = [] + for role in ("divergent", "control"): + codes = sorted(c for c, case in key["cases"].items() if case["split"] == "test" and case["role"] == role) + picked += rng.sample(codes, CLINVAR_CASES // 2) + rng.shuffle(picked) + return [{"item": code, "source": {"case_code": code}, + "cells": [code, rows[code]["gene"], rows[code]["submissions"], rows[code]["summary"]]} for code in picked] + + +def claim_items(rng: random.Random) -> list[dict[str, Any]]: + import extraction_loop as x + case = x.Case() + results = BENCH / "results" / "extraction-test" + rows = {(r["model"], r["task_id"]): r for r in x.load_jsonl(results / "episodes.jsonl")} + sample = x.load_jsonl(results / "expert_sample" / "key.jsonl") + words = {"predicts_sensitivity_in": "predicts sensitivity or response, in", "predicts_resistance_in": + "predicts resistance, in", "poor_outcome_in": "is associated with poor outcome in", "better_outcome_in": + "is associated with better outcome in", "diagnostic_of": "is diagnostic of", "predisposes_to": "predisposes to"} + rng.shuffle(sample) + items = [] + for number, entry in enumerate(sample, start=1): + task = case.tasks[entry["task_id"]] + attempts = rows[(entry["model"], entry["task_id"])]["claims"][entry["claim"]]["attempts"] + claim = (attempts[0] if entry["group"] == "stopped by the chain" else attempts[-1])["claim"] + s = claim["statement"] + text = (f"{s['subject']['label']}{' ' + claim['variant'] if claim['variant'] else ''} " + f"{words.get(s['predicate'], s['predicate'])} {s['object']['label']}" + + (f" (therapies: {', '.join(claim['therapies'])})" if claim["therapies"] else "")) + quotes = [] + for n, e in enumerate(claim["evidence"] or [], start=1): + context = case.paragraph(task["pmid"], e["locator"], e["text"]) + against = " [cited as evidence AGAINST the claim]" if e["direction"] == "contradicts" else "" + quotes.append(f"{n}. \"{e['text']}\"{against}" + (f"\n Paragraph: {context}" if context and context != e["text"] else "")) + items.append({"item": f"B{number:03d}", "source": {"sample_id": entry["sample_id"], "pmid": task["pmid"]}, + "cells": [f"B{number:03d}", f"PMID {task['pmid']}", text, "\n\n".join(quotes) or "(no quote given)"], + "link": f"https://pubmed.ncbi.nlm.nih.gov/{task['pmid']}/"}) + return items + + +def singlecell_items(seed: int, maintainer: Path) -> list[dict[str, Any]]: + import audit_celltype as ac + import celltype_loop as cl + + from bioevidence_validator import audit, review + case = cl.Case() + cases = ac.export(case) + predictions = review.load_predictions(BENCH / "results" / "celltype-audit" / "predictions.csv") + sheet, manifest = audit.audit_sample(predictions, method=ac.METHOD, use=ac.USE, size=audit.sample_size(ac.TARGET), + seed=seed, profile_id=ac.PROFILE, controls=ac.CONTROLS, + predictions_sha256=review.sha256_file(BENCH / "results" / "celltype-audit" / + "predictions.csv")) + (maintainer / "singlecell_audit_manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + items = [] + for number, row in enumerate(sheet, start=1): + c = cases[row["case_id"]] + task, answer = c["task"], c["answer"] + markers = ", ".join(f"{m['gene']} ({float(m['logfc']):.1f}; {float(m['pct_in']):.0%} vs {float(m['pct_out']):.0%})" + for m in task["markers"]) + cited = ", ".join(f"{m['gene']}{'' if m['stance'] == 'supports' else ' (against)'}" for m in answer["markers"]) + items.append({"item": f"S{number:03d}", "source": {k: row[k] for k in ("case_id", "record_sha256", "profile_id", + "profile_sha256", "requested_use", "split")}, + "cells": [f"S{number:03d}", f"{task['species']} {task['tissue']}, {task['assay']} (cluster {task['cluster']})", + markers, f"{answer['cell_type_label']} ({answer['cell_type_id']})", cited or "none"]}) + return items + + +def write_workbook(path: Path, parts: dict[str, list[dict[str, Any]]]) -> None: + from openpyxl import Workbook + from openpyxl.styles import Alignment, Border, Font, PatternFill, Side + from openpyxl.worksheet.datavalidation import DataValidation + + yellow, grey = PatternFill("solid", fgColor="FFF4CC"), PatternFill("solid", fgColor="F2F2F2") + wrap = Alignment(wrap_text=True, vertical="top") + thin = Border(bottom=Side(style="thin", color="BFBFBF")) + book = Workbook() + start = book.active + start.title = "Start here" + start.column_dimensions["A"].width = 30 + start.column_dimensions["B"].width = 70 + start.column_dimensions["C"].width = 55 + start["A1"] = "Expert review · round 1 专家审阅 · 第一轮" + start["A1"].font = Font(bold=True, size=15) + start["A2"] = ("We test whether AI-produced curation can be trusted. Your judgment is the reference the AI is " + "measured against. 我们在检验 AI 生成的生物学标注是否可信，您的判断就是衡量 AI 的标准。") + start["A2"].alignment = wrap + start.merge_cells("A2:C2") + start.row_dimensions[2].height = 32 + for row, label in enumerate(("Your name or ID 姓名或代号", "Your field, one line 专业背景（一句话）", + "Date finished 完成日期"), start=4): + start.cell(row=row, column=1, value=label).font = Font(bold=True) + start.cell(row=row, column=2).fill = yellow + start.cell(row=8, column=1, value="Part 部分").font = Font(bold=True) + start.cell(row=8, column=2, value="For 适合").font = Font(bold=True) + start.cell(row=8, column=3, value="Items · time 条目 · 时间").font = Font(bold=True) + for row, (name, part) in enumerate(PARTS.items(), start=9): + start.cell(row=row, column=1, value=f"{part['sheet']}") + start.cell(row=row, column=2, value=part["who"]) + start.cell(row=row, column=3, value=f"{len(parts[name])} items · {part['minutes']}") + start.cell(row=13, column=1, value="How 怎么做").font = Font(bold=True) + for row, (english, chinese) in enumerate(RULES, start=14): + start.cell(row=row, column=1, value=f"{row - 13}.") + start.cell(row=row, column=2, value=english).alignment = wrap + start.cell(row=row, column=3, value=chinese).alignment = wrap + start.row_dimensions[row].height = 30 + start.cell(row=21, column=1, value="Thank you! 谢谢！").font = Font(bold=True) + + # Dropdown options live on a hidden sheet: an inline list would split options at their commas. + options = book.create_sheet("Lists") + lists: dict[int, str] = {} + for column, choices in enumerate((STATEMENT, USE, CLAIM, PROBLEMS), start=1): + for row, label in enumerate(choices, start=1): + options.cell(row=row, column=column, value=label) + letter = options.cell(row=1, column=column).column_letter + lists[id(choices)] = f"=Lists!${letter}$1:${letter}${len(choices)}" + options.sheet_state = "hidden" + + for name, items in parts.items(): + sheet = book.create_sheet(PARTS[name]["sheet"]) + columns = COLUMNS[name] + sheet.cell(row=1, column=1, value=PARTS[name]["title"]).font = Font(bold=True, size=13) + for n, line in enumerate(GUIDES[name], start=2): + cell = sheet.cell(row=n, column=1, value=line) + cell.alignment = wrap + sheet.merge_cells(start_row=n, start_column=1, end_row=n, end_column=len(columns)) + sheet.row_dimensions[n].height = 30 + for index, (title, width, choices) in enumerate(columns, start=1): + cell = sheet.cell(row=HEADER_ROW, column=index, value=title) + cell.font, cell.alignment, cell.fill = Font(bold=True), wrap, yellow if choices else grey + sheet.column_dimensions[cell.column_letter].width = width + sheet.row_dimensions[HEADER_ROW].height = 34 + first, last = HEADER_ROW + 1, HEADER_ROW + len(items) + for offset, item in enumerate(items): + row = first + offset + for index, value in enumerate(item["cells"], start=1): + cell = sheet.cell(row=row, column=index, value=value) + cell.alignment, cell.border = wrap, thin + if item.get("link"): + sheet.cell(row=row, column=2).hyperlink = item["link"] + sheet.cell(row=row, column=2).font = Font(color="0563C1", underline="single") + for index, (_, _, choices) in enumerate(columns, start=1): + if choices: + cell = sheet.cell(row=row, column=index) + cell.fill, cell.alignment, cell.border = yellow, wrap, thin + for index, (_, _, choices) in enumerate(columns, start=1): + if isinstance(choices, dict): + letter = sheet.cell(row=HEADER_ROW, column=index).column_letter + rule = DataValidation(type="list", formula1=lists[id(choices)], allow_blank=True, + showErrorMessage=True, errorTitle="Please pick from the list", + error="Choose one of the options in the dropdown.") + rule.add(f"{letter}{first}:{letter}{last}") + sheet.add_data_validation(rule) + sheet.freeze_panes = sheet.cell(row=first, column=2) + book.save(path) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--clinvar-packet", type=Path, required=True) + parser.add_argument("--clinvar-key", type=Path, required=True) + parser.add_argument("--seed", type=int, required=True, help="Keep it private: it fixes which items are drawn") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + if args.output.exists() and any(args.output.iterdir()): + raise SystemExit(f"{args.output} is not empty; refusing to overwrite a round") + maintainer = args.output / "maintainer" + maintainer.mkdir(parents=True, exist_ok=True) + rng = random.Random(args.seed) + clinvar_key = json.loads(args.clinvar_key.read_text(encoding="utf-8")) + parts = {"A": clinvar_items(args.clinvar_packet, clinvar_key, rng), "B": claim_items(rng), + "C": singlecell_items(args.seed, maintainer)} + write_workbook(args.output / WORKBOOK, parts) + key = {"round": ROUND, "warning": "Keep from the experts: it maps items to their sources.", "seed": args.seed, + "clinvar_key_sha256": sha256_text(args.clinvar_key.read_text(encoding="utf-8")), + "choices": {"statement": STATEMENT, "use": USE, "claim": CLAIM, "problems": PROBLEMS}, + "items": {name: {i["item"]: i["source"] for i in items} for name, items in parts.items()}} + (maintainer / "round_key.json").write_text(json.dumps(key, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Wrote {args.output / WORKBOOK}: " + ", ".join(f"part {n} {len(i)} items" for n, i in parts.items()) + + f". Keep {maintainer} private.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_expert_round.py b/tests/test_expert_round.py new file mode 100644 index 0000000..956571e --- /dev/null +++ b/tests/test_expert_round.py @@ -0,0 +1,110 @@ +"""The expert-review round: one blinded workbook for three reviews, and its import back into each review's format.""" +import importlib.util +import json +import sys +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +from bioevidence_validator import audit, review + +ROOT = Path(__file__).resolve().parents[1] +ROUND = ROOT / "evaluation" / "expert_round" +KIT = ROOT / "evaluation" / "clinvar_review" +sys.path.insert(0, str(ROUND)) +sys.path.insert(0, str(KIT)) + + +def load(folder, name): + spec = importlib.util.spec_from_file_location(f"er_{name}", folder / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +mk, imp, prepare = load(ROUND, "make_round"), load(ROUND, "import_round"), load(KIT, "prepare_packets") + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + tmp = tmp_path_factory.mktemp("round") + prepare.prepare(tmp / "kit", salt="test-salt", calibration=20) + assert mk.main(["--clinvar-packet", str(tmp / "kit" / "reviewer_packet"), "--clinvar-key", + str(tmp / "kit" / "maintainer" / "key.json"), "--seed", "7", "--output", str(tmp / "round")]) == 0 + return tmp + + +def fill(src, dst, answers, who=("R1", "curator, 5 years", "2026-10-15"), rows=None): + book = load_workbook(src) + start = book["Start here"] + start["B4"], start["B5"], start["B6"] = who + for name, values in answers.items(): + sheet = book[mk.PARTS[name]["sheet"]] + columns = [i for i, (_, _, c) in enumerate(mk.COLUMNS[name], start=1) if isinstance(c, dict)] + last = sheet.max_row if rows is None else mk.HEADER_ROW + rows + for row in range(mk.HEADER_ROW + 1, last + 1): + for column, value in zip(columns, values, strict=False): + sheet.cell(row=row, column=column, value=value) + book.save(dst) + return dst + + +def test_workbook_is_blinded_and_every_answer_is_a_dropdown(built): + book = load_workbook(built / "round" / mk.WORKBOOK) + assert book.sheetnames == ["Start here", "Lists", *(p["sheet"] for p in mk.PARTS.values())] + assert book["Lists"].sheet_state == "hidden" + key = json.loads((built / "round" / "maintainer" / "round_key.json").read_text(encoding="utf-8")) + sizes = {name: len(items) for name, items in key["items"].items()} + assert sizes == {"A": mk.CLINVAR_CASES, "B": 60, "C": audit.sample_size(0.05) + 10} + hidden = ("claude", "gpt", "gemini", "validator", "divergent", "review_required", "bioevidence", "llm-feedback") + for name, part in mk.PARTS.items(): + sheet = book[part["sheet"]] + text = " ".join(str(c.value) for row in sheet.iter_rows(max_col=4) for c in row if c.value).lower() + if name != "B": # paper quotes may say anything + assert not any(word in text for word in hidden) + questions = [t for t, _, c in mk.COLUMNS[name] if isinstance(c, dict)] + assert len(sheet.data_validations.dataValidation) == len(questions) + assert all(dv.formula1.startswith("=Lists!") for dv in sheet.data_validations.dataValidation) + clinvar = json.loads((built / "kit" / "maintainer" / "key.json").read_text(encoding="utf-8")) + roles = [clinvar["cases"][code]["role"] for code in key["items"]["A"]] + assert roles.count("divergent") == roles.count("control") == mk.CLINVAR_CASES // 2 + + +def test_returned_workbook_imports_into_each_review(built, tmp_path): + filled = fill(built / "round" / mk.WORKBOOK, tmp_path / "R1.xlsx", + {"A": ["yes", "usable as is", "needs a curator's check", "not usable"], + "B": ["no", "Opposite direction"], "C": ["unsure", "needs a curator's check"]}) + out = tmp_path / "out" + args = [str(filled), "--key", str(built / "round" / "maintainer" / "round_key.json"), + "--clinvar-key", str(built / "kit" / "maintainer" / "key.json"), "--output", str(out)] + assert imp.main(args) == 0 + clinvar = review.load_annotations(out / "clinvar_annotations.csv") + assert len(clinvar) == 3 * mk.CLINVAR_CASES and {a["reviewer_id"] for a in clinvar} == {"R1"} + assert {a["requested_use"]: a["admission_label"] for a in clinvar} == { + "research_summary": "admitted", "clinical_reference": "review_required", "expert_reference": "rejected"} + extraction = (out / "extraction_labels.csv").read_text(encoding="utf-8").splitlines() + assert len(extraction) == 61 and extraction[1].split(",")[2:4] == ["no", "EXT-2"] + singlecell = review.load_annotations(out / "singlecell_annotations.csv") + manifest = json.loads((built / "round" / "maintainer" / "singlecell_audit_manifest.json").read_text(encoding="utf-8")) + report = audit.audit_score(manifest, review.resolve(singlecell, [], min_reviewers=1), singlecell) + assert report["routes"]["admitted"]["audited"] == audit.sample_size(0.05) + with pytest.raises(ValueError, match="already holds"): + imp.main(args) # the same reviewer twice + + +def test_partial_and_invalid_answers(built, tmp_path): + key = ["--key", str(built / "round" / "maintainer" / "round_key.json"), "--output", str(tmp_path / "out")] + partial = fill(built / "round" / mk.WORKBOOK, tmp_path / "p.xlsx", {"C": ["yes", "usable as is"]}, rows=5) + assert imp.main([str(partial), *key]) == 0 # only part C, five rows; no ClinVar key needed + assert len(review.load_annotations(tmp_path / "out" / "singlecell_annotations.csv")) == 5 + half = fill(built / "round" / mk.WORKBOOK, tmp_path / "h.xlsx", {"A": ["yes"]}, rows=1) + with pytest.raises(ValueError, match="unanswered"): + imp.main([str(half), *key, "--reviewer-id", "R2"]) + wrong = fill(built / "round" / mk.WORKBOOK, tmp_path / "w.xlsx", {"C": ["maybe", "usable as is"]}, rows=1) + with pytest.raises(ValueError, match="not an option"): + imp.main([str(wrong), *key, "--reviewer-id", "R3"]) + anonymous = fill(built / "round" / mk.WORKBOOK, tmp_path / "a.xlsx", {"C": ["yes", "usable as is"]}, + who=(None, None, None), rows=1) + with pytest.raises(ValueError, match="ID and one-line qualification"): + imp.main([str(anonymous), *key])