From 3d42f2f01db8113a57b1338bc066b51a9a4a1a87 Mon Sep 17 00:00:00 2001 From: George Date: Wed, 29 Jul 2026 12:36:18 -0400 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=93=8A=20telemetry:=20restore=20round?= =?UTF-8?q?=20+=20per-miner=20telemetry=20on=20every=20recovery=20(schema?= =?UTF-8?q?=20v3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Startup recovery restored telemetry at most ONCE, and `val_loss` had no recovery path at all. Two Watchtower restarts 25 minutes apart on 2026-07-31 left every per-miner family at zero series for 17 minutes and lost cycle 16684's losses permanently. Three coupled gaps, one schema bump: 1. `val_loss` is journaled (v3 `uid_to_val_loss`). It is published at eval time and is NOT derivable from `scores`: `delta = max(0.0, baseline - val_loss)` clamps at zero, so every miner scoring 0 — the majority in many rounds — would be underivable. Plumbed through `MinerEvalJob.val_loss` → `Round.mark_scored(..., val_loss=)` → `Round.val_losses` → journal; both eval paths (foreground + background) pass it. 2. Recovery now re-emits telemetry on EVERY restart, not just the first. The replay loop only touches unfinalized journals, and replaying one marks it finalized — so the second restart found nothing and emitted nothing. New `republish_telemetry_from_journal` reads the newest finalized journal and sets the gauges directly. It is deliberately METRICS-ONLY. Re-running `finalize_round_scores` would keep the aggregator's point set correct (`drop_round` runs first) but re-stamp those points with fresh `_utc_now()` timestamps, shoving them to the end of the time-ordered series — and the rolling average is "last N by timestamp", which is what drives weight submission. A re-finalize would therefore silently reshuffle the scoring window. A test asserts the pass makes zero mutating aggregator calls. 3. The round-level counters (`validator_round_miners_{scored,pending, failed}`, `lifecycle_step`, `current_round_id`) now restore via that same pass, so they cover the finalized path too rather than inheriting the once-only limitation. `roster_size` + `lifecycle_step` are journaled (v3) to feed them; pre-v3 journals clamp pending to 0 rather than inventing a denominator. `verdict_uids` mirrors finalize's entry set exactly — scored ∪ validation-failed ∪ freeze-zero, excluding operational failures, which get no aggregator entry so telemetry must not imply they were judged. `from_json` still accepts v1/v2 (missing fields default). Verified against the v0.3.4 image: the suite's failure set is byte-identical before and after this change (4 pre-existing failures, unrelated), and the 49 tests across the journal/telemetry suites pass. Co-Authored-By: Claude Opus 5 --- .../test/test_cycle_consistent_telemetry.py | 72 ++++++ .../test/test_journal_telemetry_republish.py | 221 ++++++++++++++++++ connito/validator/background_eval_worker.py | 2 +- connito/validator/evaluator.py | 24 +- connito/validator/round.py | 25 +- connito/validator/round_journal.py | 166 ++++++++++++- connito/validator/run.py | 45 ++++ 7 files changed, 549 insertions(+), 6 deletions(-) create mode 100644 connito/test/test_journal_telemetry_republish.py diff --git a/connito/test/test_cycle_consistent_telemetry.py b/connito/test/test_cycle_consistent_telemetry.py index e563dc1..6fd4c4c 100644 --- a/connito/test/test_cycle_consistent_telemetry.py +++ b/connito/test/test_cycle_consistent_telemetry.py @@ -155,6 +155,78 @@ def test_commit_map_from_checkpoints_skips_incomplete(): assert m == {1: ("a/r", "v1")} +# --------------------------------------------------------------------------- +# Journal v3: roster_size / lifecycle_step for round-gauge recovery +# --------------------------------------------------------------------------- + +def test_journal_v3_roundtrips_roster_and_step(tmp_path): + j = RJ.RoundJournal( + round_id=555, + scored_uids=(1, 2, 3), + failed_uids=(4,), + roster_size=165, + lifecycle_step=3, + finalized=True, + ) + p = tmp_path / "round_555.json" + RJ.write_atomic(p, j) + loaded = RJ.load(p) + assert loaded is not None + assert loaded.schema_version == 3 + assert loaded.roster_size == 165 + assert loaded.lifecycle_step == 3 + + +def test_journal_v2_loads_with_zero_roster(tmp_path): + # A v2 file (no roster_size/lifecycle_step) must still load, defaulting + # both to 0 so recovery degrades gracefully rather than crashing. + v2 = { + "round_id": 42, "schema_version": 2, + "scored_uids": [1, 2], "failed_uids": [3], + "uid_to_commit": {}, "finalized": False, + } + p = tmp_path / "round_42.json" + p.write_text(json.dumps(v2), encoding="utf-8") + loaded = RJ.load(p) + assert loaded is not None + assert loaded.roster_size == 0 + assert loaded.lifecycle_step == 0 + + +def test_recovery_round_carries_v3_fields(tmp_path): + j = RJ.RoundJournal(round_id=777, roster_size=100, lifecycle_step=2) + stub = RJ._RecoveryRound.from_journal(j, tmp_path / "round_777.json") + assert stub.roster_size == 100 + assert stub.lifecycle_step == 2 + + +def test_finalize_preserves_roster_in_journal_rewrite(tmp_path): + # finalize re-writes the journal with finalized=True; the v3 fields must + # survive that round-trip (they feed the round-gauge restore next boot). + rid = 8_888 + jpath = tmp_path / f"round_{rid}.json" + RJ.write_atomic( + jpath, + RJ.RoundJournal( + round_id=rid, + uid_to_hotkey={9201: "hk", 9202: "hk2"}, + scores={9201: 2.5}, + scored_uids=(9201,), + failed_uids=(9202,), + roster_size=165, + lifecycle_step=3, + ), + ) + stub = RJ._RecoveryRound.from_journal(RJ.load(jpath), jpath) + finalize_round_scores( + round_obj=stub, score_aggregator=_FakeAggregator(), score_path=None, + ) + reloaded = RJ.load(jpath) + assert reloaded.finalized is True + assert reloaded.roster_size == 165 + assert reloaded.lifecycle_step == 3 + + # --------------------------------------------------------------------------- # finalize_round_scores emission (live-shaped round + recovery path) # --------------------------------------------------------------------------- diff --git a/connito/test/test_journal_telemetry_republish.py b/connito/test/test_journal_telemetry_republish.py new file mode 100644 index 0000000..5c87b25 --- /dev/null +++ b/connito/test/test_journal_telemetry_republish.py @@ -0,0 +1,221 @@ +"""Tests for the metrics-only telemetry republish on startup recovery. + +Startup recovery restores telemetry by *replaying* `finalize_round_scores`, +which marks the journal finalized — and the replay loop skips finalized +journals. So recovery only ever restored telemetry ONCE: a second restart +found nothing to replay and emitted nothing, leaving the dashboard blank +for the entire last completed cycle (observed 2026-07-31, two Watchtower +restarts 25 minutes apart). + +`republish_telemetry_from_journal` closes that. It must be metrics-only: +re-running finalize would keep the aggregator's point *set* correct +(`drop_round` runs first) but re-stamp those points with fresh timestamps, +reshuffling the "last N by timestamp" rolling average that drives weight +submission. + +The prometheus registry is process-global, so these tests use uids in a +dedicated 94xx range and round ids in a 94xxxx range. +""" +from __future__ import annotations + +import json + +from connito.shared import telemetry as T +from connito.validator import round_journal as RJ + + +def _sample(gauge, family: str, **labels) -> float | None: + for metric in gauge.collect(): + for s in metric.samples: + if s.name == family and all(s.labels.get(k) == v for k, v in labels.items()): + return s.value + return None + + +def _journal(round_id: int, **overrides) -> RJ.RoundJournal: + kwargs = dict( + round_id=round_id, + uid_to_hotkey={9401: "hk1", 9402: "hk2", 9403: "hk3", 9404: "hk4"}, + scores={9401: 2.5, 9402: 1.5, 9403: 0.0}, + scored_uids=(9401, 9402, 9403), + failed_uids=(9404,), # operational failure — no verdict + validation_failed_uids=(), + freeze_zero_uids=(9405,), + freeze_zero_hotkeys={9405: "hk5"}, + uid_to_commit={9401: ("miner/one", "aaa1")}, + uid_to_val_loss={9401: 1.25, 9402: 1.75}, + roster_size=10, + lifecycle_step=3, + finalized=True, + ) + kwargs.update(overrides) + return RJ.RoundJournal(**kwargs) + + +class _RecordingAggregator: + """Read-only probe: records every mutating call so a test can assert + the republish pass made none.""" + + def __init__(self): + self.mutations: list[str] = [] + + # --- read side (what republish is allowed to use) --- + def uid_score_pairs(self, how="avg"): + return {9401: 2.25 if how == "latest" else 1.1, 9402: 1.5} + + def record_count(self, uid): + return 4 + + # --- write side (must never be called) --- + def add_score(self, **kwargs): + self.mutations.append("add_score") + + def drop_round(self, round_id): + self.mutations.append("drop_round") + + def persist_atomic(self, path): + self.mutations.append("persist_atomic") + + def prune_before_round(self, min_round_id): + self.mutations.append("prune_before_round") + + +# --------------------------------------------------------------------------- +# The guard the backend specifically asked for +# --------------------------------------------------------------------------- + +def test_republish_does_not_mutate_the_aggregator(): + agg = _RecordingAggregator() + RJ.republish_telemetry_from_journal(_journal(940_001), score_aggregator=agg) + assert agg.mutations == [] + + +def test_republish_works_without_an_aggregator(): + # Snapshot gauges are skipped, everything else still emits. + n = RJ.republish_telemetry_from_journal(_journal(940_002), score_aggregator=None) + assert n == 4 # 3 scored + 1 freeze_zero (the failed uid gets no verdict) + + +# --------------------------------------------------------------------------- +# What gets emitted +# --------------------------------------------------------------------------- + +def test_republish_emits_last_scored_round_for_verdict_uids(): + rid = 940_010 + RJ.republish_telemetry_from_journal(_journal(rid), score_aggregator=_RecordingAggregator()) + for uid in (9401, 9402, 9403, 9405): + assert _sample( + T.VALIDATOR_MINER_LAST_SCORED_ROUND_ID, + "validator_miner_last_scored_round_id", + miner_uid=str(uid), + ) == float(rid) + + +def test_republish_excludes_operational_failures(): + """`failed_uids` minus validation failures get NO aggregator entry at + finalize, so telemetry must not imply they were judged.""" + j = _journal(940_011, uid_to_hotkey={9410: "hk"}, scored_uids=(), scores={}, + failed_uids=(9410,), freeze_zero_uids=(), freeze_zero_hotkeys={}, + uid_to_commit={}, uid_to_val_loss={}) + assert RJ.verdict_uids(j) == set() + assert RJ.republish_telemetry_from_journal(j) == 0 + + +def test_republish_emits_val_loss(): + rid = 940_020 + RJ.republish_telemetry_from_journal(_journal(rid)) + assert _sample( + T.VALIDATOR_MINER_VAL_LOSS, "validator_miner_val_loss", miner_uid="9401" + ) == 1.25 + assert _sample( + T.VALIDATOR_MINER_VAL_LOSS, "validator_miner_val_loss", miner_uid="9402" + ) == 1.75 + + +def test_republish_emits_round_counters_and_lifecycle(): + rid = 940_030 + RJ.republish_telemetry_from_journal(_journal(rid)) + label = {"round_id": str(rid)} + assert _sample(T.VALIDATOR_ROUND_MINERS_SCORED, "validator_round_miners_scored", **label) == 3.0 + assert _sample(T.VALIDATOR_ROUND_MINERS_FAILED, "validator_round_miners_failed", **label) == 1.0 + # roster 10 - 3 scored - 1 failed + assert _sample(T.VALIDATOR_ROUND_MINERS_PENDING, "validator_round_miners_pending", **label) == 6.0 + assert _sample( + T.VALIDATOR_ROUND_LIFECYCLE_STEP, "validator_round_lifecycle_step", **label + ) == 3.0 + + +def test_republish_emits_round_delta_and_commit(): + rid = 940_040 + RJ.republish_telemetry_from_journal(_journal(rid)) + assert _sample( + T.VALIDATOR_MINER_ROUND_DELTA, "validator_miner_round_delta", miner_uid="9401" + ) == 2.5 + assert _sample( + T.VALIDATOR_MINER_EVALUATED_COMMIT_INFO, + "validator_miner_evaluated_commit_info", + miner_uid="9401", hf_repo_id="miner/one", hf_revision="aaa1", + ) == float(rid) + + +def test_republish_pre_v3_journal_clamps_pending_and_skips_val_loss(): + """A v2 journal has no roster_size and no losses — republish must not + invent a denominator or crash.""" + rid = 940_050 + j = _journal(rid, roster_size=0, lifecycle_step=0, uid_to_val_loss={}) + assert RJ.republish_telemetry_from_journal(j) == 4 + assert _sample( + T.VALIDATOR_ROUND_MINERS_PENDING, "validator_round_miners_pending", + round_id=str(rid), + ) == 0.0 + + +def test_republish_is_idempotent(): + rid = 940_060 + a = RJ.republish_telemetry_from_journal(_journal(rid)) + b = RJ.republish_telemetry_from_journal(_journal(rid)) + assert a == b + assert _sample( + T.VALIDATOR_ROUND_MINERS_SCORED, "validator_round_miners_scored", + round_id=str(rid), + ) == 3.0 + + +def test_republish_registers_round_for_eviction(): + rid = 940_070 + RJ.republish_telemetry_from_journal(_journal(rid)) + assert rid in T._EMITTED_ROUND_IDS + + +def test_republish_never_raises_on_malformed_journal(): + class Broken: + round_id = "not-an-int" + assert RJ.republish_telemetry_from_journal(Broken()) == 0 # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# val_loss journalling (Gap 1) +# --------------------------------------------------------------------------- + +def test_journal_v3_roundtrips_val_loss(tmp_path): + j = _journal(940_080) + p = tmp_path / "round.json" + RJ.write_atomic(p, j) + loaded = RJ.load(p) + assert loaded is not None + assert loaded.uid_to_val_loss == {9401: 1.25, 9402: 1.75} + + +def test_journal_v2_file_loads_with_empty_val_loss(tmp_path): + v2 = {"round_id": 42, "schema_version": 2, "scored_uids": [1], "finalized": True} + p = tmp_path / "round_42.json" + p.write_text(json.dumps(v2), encoding="utf-8") + loaded = RJ.load(p) + assert loaded is not None + assert loaded.uid_to_val_loss == {} + assert loaded.roster_size == 0 + + +def test_recovery_round_carries_val_losses(tmp_path): + stub = RJ._RecoveryRound.from_journal(_journal(940_090), tmp_path / "r.json") + assert stub.val_losses == {9401: 1.25, 9402: 1.75} diff --git a/connito/validator/background_eval_worker.py b/connito/validator/background_eval_worker.py index d198a22..a6f9704 100644 --- a/connito/validator/background_eval_worker.py +++ b/connito/validator/background_eval_worker.py @@ -518,7 +518,7 @@ def _run_eval(): self._prune_non_top(round_obj) return - round_obj.mark_scored(uid, evaluated.score) + round_obj.mark_scored(uid, evaluated.score, val_loss=evaluated.val_loss) logger.info( "bg-eval: success", round_id=round_obj.round_id, diff --git a/connito/validator/evaluator.py b/connito/validator/evaluator.py index 9123264..7201955 100644 --- a/connito/validator/evaluator.py +++ b/connito/validator/evaluator.py @@ -347,6 +347,19 @@ def finalize_round_scores( uid_to_commit=_rj.commit_map_from_checkpoints( getattr(round_obj, "uid_to_chain_checkpoint", None) or {} ), + # v3 round-level gauge inputs. A live Round exposes + # foreground/background uids; the recovery stub carries + # `roster_size` forward from the journal it was hydrated + # from — so a re-finalize preserves them. + roster_size=int( + getattr(round_obj, "roster_size", 0) + or ( + len(getattr(round_obj, "foreground_uids", ())) + + len(getattr(round_obj, "background_uids", ())) + ) + ), + lifecycle_step=int(getattr(round_obj, "lifecycle_step", 0)), + uid_to_val_loss=dict(getattr(round_obj, "val_losses", None) or {}), finalized=True, ), ) @@ -628,6 +641,14 @@ class MinerEvalJob: model_path: str step: int score: float = 0.0 + # Raw evaluation loss for this miner this round. Carried alongside the + # delta-based `score` so the caller can journal it: `val_loss` is + # published to Prometheus at eval time and is NOT recoverable from + # `score` alone, because `delta = max(0.0, baseline - val_loss)` clamps + # at zero — every miner scoring 0 (the majority in many rounds) would + # be underivable. Without journaling it, a mid-round restart loses the + # cycle's losses permanently. + val_loss: float | None = None # -------------------------- Pipeline Config ----------------------------------- @@ -876,6 +897,7 @@ def evaluate_one_miner_sync( model_path=str(model_path), step=int(step), score=float(score), + val_loss=float(val_loss), ) except EvalDeadlineExceeded as e: logger.warning( @@ -1193,7 +1215,7 @@ async def evaluate_foreground_round( round_obj=round_obj, ) continue - round_obj.mark_scored(uid, evaluated.score) + round_obj.mark_scored(uid, evaluated.score, val_loss=evaluated.val_loss) round_obj.publish_progress() completed.append(evaluated) _prune_non_top_after_eval( diff --git a/connito/validator/round.py b/connito/validator/round.py index aa16db3..1eb394b 100644 --- a/connito/validator/round.py +++ b/connito/validator/round.py @@ -77,6 +77,11 @@ class Round: # rounds and would let history pull a non-top-this-round miner into # the keep set). scores: dict[int, float] = field(default_factory=dict) + # Per-uid raw evaluation loss, recorded by `mark_scored` alongside the + # score. Journaled so a mid-round restart doesn't lose the cycle's + # losses — `validator_miner_val_loss` is emitted at eval time and is + # not derivable from `scores` (the delta clamps at 0). + val_losses: dict[int, float] = field(default_factory=dict) claimed_uids: set[int] = field(default_factory=set) failed_uids: set[int] = field(default_factory=set) # UIDs the miner is at fault for: explicit validation failures @@ -96,6 +101,11 @@ class Round: freeze_zero_uids: set[int] = field(default_factory=set) freeze_zero_hotkeys: dict[int, str] = field(default_factory=dict) weights_submitted: bool = False + # Last live lifecycle step this round reached (set by run.py alongside + # the VALIDATOR_ROUND_LIFECYCLE_STEP gauge: 0 freeze / 2 post-foreground + # / 3 eval-window). Persisted to the journal so startup recovery can + # restore the round-level gauges that only the live loop writes. + lifecycle_step: int = 0 # Round-group construction scheme (gated by # `config.evaluation.enable_round_group_construction`). All default @@ -588,6 +598,9 @@ def _journal_snapshot_locked(self) -> dict | None: "freeze_zero_uids": tuple(sorted(self.freeze_zero_uids)), "freeze_zero_hotkeys": dict(self.freeze_zero_hotkeys), "uid_to_commit": commit_map_from_checkpoints(self.uid_to_chain_checkpoint), + "uid_to_val_loss": dict(self.val_losses), + "roster_size": len(self.foreground_uids) + len(self.background_uids), + "lifecycle_step": int(self.lifecycle_step), "finalized": False, } @@ -640,12 +653,20 @@ def _record_in_cycle_score(self, uid: int, hotkey: str, score: float) -> None: error=str(e), uid=uid, round_id=self.round_id, ) - def mark_scored(self, uid: int, score: float = 0.0) -> None: + def mark_scored( + self, uid: int, score: float = 0.0, val_loss: float | None = None + ) -> None: """Record a successful evaluation. `score` is this-round's score (e.g. ``delta ** 1.2`` from `evaluate_one_miner`); it is stored in `self.scores` so per-round ranking — used by post-eval submission cleanup — never has to consult the global aggregator. + `val_loss` is the raw evaluation loss. It is recorded purely so the + journal can carry it: `validator_miner_val_loss` is published at + eval time and cannot be reconstructed from `score`, because + ``delta = max(0.0, baseline - val_loss)`` clamps at zero. Optional + so existing callers and test fixtures keep working. + Also writes to the per-round journal and (if configured) the score aggregator with the raw delta tagged with this round_id, so a kill before `finalize_round_scores` runs does not lose the @@ -656,6 +677,8 @@ def mark_scored(self, uid: int, score: float = 0.0) -> None: with self._lock: self.scored_uids.add(uid) self.scores[uid] = score_f + if val_loss is not None: + self.val_losses[uid] = float(val_loss) self.claimed_uids.discard(uid) hotkey = self.uid_to_hotkey.get(uid) self._persist_journal() diff --git a/connito/validator/round_journal.py b/connito/validator/round_journal.py index 41e2250..ec4450b 100644 --- a/connito/validator/round_journal.py +++ b/connito/validator/round_journal.py @@ -27,9 +27,18 @@ # v2 adds `uid_to_commit` (uid -> (hf_repo_id, hf_revision)) so the # journal-recovery finalize can re-publish evaluated-commit telemetry. -# `from_json` accepts v1 files (missing map -> empty) so leftover journals -# written by an older build still recover. -SCHEMA_VERSION = 2 +# v3 adds `roster_size` + `lifecycle_step` so recovery can also restore the +# round-level gauges (validator_round_miners_{scored,pending,failed}, +# lifecycle_step, current_round_id) — those are written only by the live +# eval workers, so without persistence the dashboard's "Evaluated N of M" +# column blanks for a full cycle after every restart. v3 also adds +# `uid_to_val_loss`, the only per-miner field with no recovery path at all: +# `validator_miner_val_loss` is emitted at eval time and cannot be derived +# from `scores`, because `delta = max(0.0, baseline - val_loss)` clamps at +# zero, so every miner scoring 0 would be underivable. +# `from_json` accepts v1/v2 files (missing fields default) so leftover +# journals written by an older build still recover. +SCHEMA_VERSION = 3 JOURNAL_DIR_NAME = "round_journal" JOURNAL_FILENAME_PREFIX = "round_" JOURNAL_FILENAME_SUFFIX = ".json" @@ -59,6 +68,17 @@ class RoundJournal: # v2: uid -> (hf_repo_id, hf_revision) evaluated for the round. Only # uids whose chain checkpoint carried BOTH values are recorded. uid_to_commit: dict[int, tuple[str, str]] = field(default_factory=dict) + # v3: round-level gauge inputs. `roster_size` = len(foreground_uids) + + # len(background_uids) at freeze; `lifecycle_step` = the last live + # lifecycle step the round reached (0 freeze / 2 post-foreground / + # 3 eval-window). Both let startup recovery restore the round-level + # gauges. Default 0 for v1/v2 journals (recovery then leaves pending at + # 0 rather than inventing a roster). + roster_size: int = 0 + lifecycle_step: int = 0 + # v3: uid -> raw evaluation loss, recorded at `mark_scored`. Only + # populated for uids actually evaluated this round. + uid_to_val_loss: dict[int, float] = field(default_factory=dict) finalized: bool = False schema_version: int = SCHEMA_VERSION @@ -75,6 +95,9 @@ def to_json(self) -> str: payload["uid_to_commit"] = { str(k): [str(v[0]), str(v[1])] for k, v in self.uid_to_commit.items() } + payload["uid_to_val_loss"] = { + str(k): float(v) for k, v in self.uid_to_val_loss.items() + } return json.dumps(payload) @classmethod @@ -106,11 +129,139 @@ def from_json(cls, data: str) -> "RoundJournal": for k, v in raw.get("uid_to_commit", {}).items() if isinstance(v, (list, tuple)) and len(v) == 2 }, + roster_size=int(raw.get("roster_size", 0)), + lifecycle_step=int(raw.get("lifecycle_step", 0)), + uid_to_val_loss={ + int(k): float(v) for k, v in raw.get("uid_to_val_loss", {}).items() + }, finalized=bool(raw.get("finalized", False)), schema_version=version, ) +def verdict_uids(journal: "RoundJournal") -> set[int]: + """UIDs that received a finalize verdict for this round. + + Mirrors which uids `finalize_round_scores` writes an aggregator entry + for: every scored uid, every explicit validation failure, and every + freeze-zero uid not already in those sets. Operational failures + (`failed_uids` minus `validation_failed_uids`) are deliberately + excluded — finalize writes nothing for them so the miner keeps its + prior EMA, and telemetry must not imply otherwise. + + A uid with no known hotkey is skipped, matching finalize's `continue`. + """ + scored = set(journal.scored_uids) + validation_failed = set(journal.validation_failed_uids) + freeze_zero = set(journal.freeze_zero_uids) - scored - validation_failed + out: set[int] = set() + for uid in scored | validation_failed | freeze_zero: + if uid in journal.uid_to_hotkey or uid in journal.freeze_zero_hotkeys: + out.add(int(uid)) + return out + + +def republish_telemetry_from_journal( + journal: "RoundJournal", score_aggregator=None +) -> int: + """Re-emit a finalized round's Prometheus series from its journal. + + **Metrics only — this must never mutate scoring state.** It exists + because startup recovery works by *replaying* `finalize_round_scores`, + which marks the journal finalized; the recovery loop then skips + finalized journals, so a second restart re-emits nothing and the + dashboard loses the whole last completed cycle (observed 2026-07-31: + two Watchtower restarts 25 minutes apart left every per-miner family at + zero series for 17 minutes). + + Re-running finalize instead would be actively harmful, and not for the + obvious reason: `finalize_round_scores` calls `drop_round` first, so the + aggregator's *point set* would stay correct — but `add_score` stamps + `_utc_now()`, so the round's points would get fresh timestamps and jump + to the end of the time-ordered series. The rolling average is "last + `max_points` **by timestamp**", and that average is what drives weight + submission, so a re-finalize would silently reshuffle the scoring + window. Hence: read the journal, set gauges, touch nothing else. + + `score_aggregator` is read (never written) for the latest/avg/samples + snapshot, which lives in the aggregator rather than the journal. Pass + `None` to skip those three gauges. + + Returns the number of uids republished. Best-effort — never raises. + """ + from connito.shared.telemetry import ( + VALIDATOR_CURRENT_ROUND_ID, + VALIDATOR_MINER_VAL_LOSS, + VALIDATOR_ROUND_LIFECYCLE_STEP, + set_miner_evaluated_commit, + set_miner_last_scored_round, + set_miner_round_delta, + set_miner_score_snapshot, + set_round_progress, + ) + + try: + rid = int(journal.round_id) + uids = verdict_uids(journal) + + latest_scores: dict[int, float] = {} + avg_scores: dict[int, float] = {} + if score_aggregator is not None: + try: + latest_scores = score_aggregator.uid_score_pairs(how="latest") + avg_scores = score_aggregator.uid_score_pairs(how="avg") + except Exception: + latest_scores, avg_scores = {}, {} + + for uid in uids: + set_miner_last_scored_round(uid, rid) + samples = None + if score_aggregator is not None: + try: + samples = score_aggregator.record_count(uid) + except Exception: + samples = None + set_miner_score_snapshot( + uid, + latest=latest_scores.get(uid), + avg=avg_scores.get(uid), + samples=samples, + ) + + # Raw per-round delta, for every uid actually evaluated. + for uid in journal.scored_uids: + set_miner_round_delta(int(uid), float(journal.scores.get(uid, 0.0))) + + # val_loss — the field with no other recovery path (v3+ journals). + for uid, loss in journal.uid_to_val_loss.items(): + try: + VALIDATOR_MINER_VAL_LOSS.labels(miner_uid=str(int(uid))).set(float(loss)) + except Exception: + pass + + for uid, (repo, rev) in journal.uid_to_commit.items(): + set_miner_evaluated_commit(int(uid), repo, rev, rid) + + # Round-level counters. `roster_size` is 0 on pre-v3 journals, so + # pending clamps to 0 rather than inventing a denominator. + scored_n = len(journal.scored_uids) + failed_n = len(journal.failed_uids) + set_round_progress( + rid, + scored=scored_n, + failed=failed_n, + pending=max(0, int(journal.roster_size) - scored_n - failed_n), + ) + if journal.lifecycle_step: + VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(rid)).set( + int(journal.lifecycle_step) + ) + VALIDATOR_CURRENT_ROUND_ID.set(float(rid)) + return len(uids) + except Exception: + return 0 + + def commit_map_from_checkpoints( uid_to_chain_checkpoint: dict[int, object], ) -> dict[int, tuple[str, str]]: @@ -218,6 +369,12 @@ class _RecoveryRound: # `uid_to_commit` map (empty for v1 journals) so a recovered finalize # re-publishes evaluated-commit telemetry too. uid_to_chain_checkpoint: dict[int, object] = field(default_factory=dict) + # v3: carried through so the finalize re-write (finalized=True) preserves + # them, and so the recovery pass can restore the round-level gauges. + roster_size: int = 0 + lifecycle_step: int = 0 + # Carried so the finalize journal-rewrite preserves the losses. + val_losses: dict[int, float] = field(default_factory=dict) _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) @classmethod @@ -238,6 +395,9 @@ def from_journal(cls, journal: "RoundJournal", journal_path: str | os.PathLike) int(uid): SimpleNamespace(hf_repo_id=repo, hf_revision=rev) for uid, (repo, rev) in journal.uid_to_commit.items() }, + roster_size=int(journal.roster_size), + lifecycle_step=int(journal.lifecycle_step), + val_losses=dict(journal.uid_to_val_loss), ) def processed_uids_snapshot(self) -> tuple[set[int], set[int]]: diff --git a/connito/validator/run.py b/connito/validator/run.py index 925b80a..a529b6c 100644 --- a/connito/validator/run.py +++ b/connito/validator/run.py @@ -927,6 +927,48 @@ def run(rank: int, world_size: int, config: ValidatorConfig, pkg_version: str = journals_finalized=_recovered, journals_seen=len(_journals), ) + + # Re-emit the most recent finalized round's telemetry. + # + # Runs unconditionally, and this is the point: the replay loop above + # only touches *unfinalized* journals, and replaying one marks it + # finalized. So a second restart finds nothing to replay and used to + # emit nothing at all — leaving the dashboard blank for the whole + # last completed cycle until the next round's evaluations arrived + # (observed 2026-07-31: two Watchtower restarts 25 min apart, every + # per-miner family at zero series for 17 minutes). + # + # This is a METRICS-ONLY pass — it never re-runs finalize. Re-running + # finalize would keep the aggregator's point set correct (drop_round + # runs first) but would re-stamp those points with fresh timestamps, + # reshuffling the "last N by timestamp" rolling average that drives + # weight submission. See `republish_telemetry_from_journal`. + # + # Journals are scanned ascending, so the last finalized one is the + # most recent round — which is also the one just replayed on a first + # restart, making the re-emit a harmless idempotent gauge write. + try: + _newest_finalized = None + for _journal_file in reversed(_journals): + _j = _rj_recover.load(_journal_file) + if _j is not None and _j.finalized: + _newest_finalized = _j + break + if _newest_finalized is not None: + _republished = _rj_recover.republish_telemetry_from_journal( + _newest_finalized, score_aggregator=score_aggregator, + ) + logger.info( + "Startup recovery: republished telemetry for last finalized round", + round_id=_newest_finalized.round_id, + uids=_republished, + schema_version=_newest_finalized.schema_version, + val_losses=len(_newest_finalized.uid_to_val_loss), + ) + except Exception as e: + logger.warning( + "Startup recovery: telemetry republish failed", error=str(e), + ) except Exception as e: logger.warning( "Startup recovery: scan failed", error=str(e), @@ -1473,6 +1515,7 @@ def run(rank: int, world_size: int, config: ValidatorConfig, pkg_version: str = download_window_closed.clear() try: note_round_series(new_round.round_id) + new_round.lifecycle_step = 0 VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(new_round.round_id)).set(0) # Seed the progress counters at freeze so the round exists in # the metric from the moment it is frozen (scored=0, failed=0, @@ -1559,6 +1602,7 @@ async def _bounded_foreground_eval(): eval_worker.set_eval_base_model(copy.deepcopy(global_model)) try: note_round_series(new_round.round_id) + new_round.lifecycle_step = 2 VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(new_round.round_id)).set(2) except Exception: pass @@ -1791,6 +1835,7 @@ async def _bounded_foreground_eval(): eval_window_active.set() try: note_round_series(new_round.round_id) + new_round.lifecycle_step = 3 VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(new_round.round_id)).set(3) except Exception: pass From b9772af18dcba95a6dedf0a35d81d66565d2dc0e Mon Sep 17 00:00:00 2001 From: George Date: Fri, 31 Jul 2026 17:49:56 -0400 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=94=84=20dataset:=20revert=20subnet?= =?UTF-8?q?=20to=20Nemotron-CC-Math=20+=20C4=20(exp=5Fnemotron=5Fc4,=20gro?= =?UTF-8?q?up=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #186 switched the subnet from math data to legal data. Despite touching 9 files, the switch itself was one line: the locked `TaskCfg.expert_group_name` default. This reverts the DATASET while keeping everything layered on top since — the anti-memorization eval pipeline, the telemetry/recovery work, and the bittensor 10.5 chain fix all live in shared code and are dataset-agnostic. New expert group `exp_nemotron_c4` (group_id 4) rather than reactivating exp_math (group 0), so the cutover is clean and doesn't inherit group 0's history. `exp_math` and `exp_legal` are left on disk for reference. Two improvements are retained deliberately rather than by accident: - `dataset_class` is intentionally UNSET, which resolves to `DefaultStreamingTorchDataset` and therefore `tokenize_windowed`. `exp_math/dataset.py` still prefix-truncates, so pointing at it would have silently dropped the windowing fix ("exp_math left for follow-up", d7e979c). - Both sources are pinned to commit SHAs via `eval_source_revision_pin`. `_KNOWN_SOURCES` declares `revision="main"` for c4/Nemotron, and `main` is a moving target — an HF re-upload mid-rollout would make two validators read different rows for the same seed and break weight consensus. The legal-tuned eval gates are inherited unchanged after measuring them against the new mix (600 rows/source): c4 0% empty / 5.5% under 200 chars, Nemotron 0% / 0%, prefix-dedup 0% on both — versus Multi_Legal_Pile's 38% empty and 75% duplicate prefixes. Effectively inert here, so no override. Expert assignment is copied from exp_math, verified valid for the current post-#188 (2Fnat) architecture rather than assumed: #188 regenerated that file itself (68a878e, 208->187 experts) and nothing architectural has landed since. Confirmed by building the partial model — 52 MoE modules across layers 1..26 for trainable group 4 + helper group 2, n_routed_experts=64, expert ids 0..63 all in range. Docs: miner-faq updated for the new group, plus three stale claims corrected while in there — "8 experts per MoE layer" (it is 64), "miners self-select their group" (the field is locked and resets on load), and `sequence_length=4096` (1024 since #188). exp_legal README and the migration plan are marked historical. ROLLOUT — fleet-wide flag day, not a rolling change: - Validators only score miners whose chain commit `expert_group` matches, so miners still on group 3 become invisible and earn nothing until they upgrade. Announce before tagging. - `nvidia/Nemotron-CC-Math-v1` is GATED. Every validator and miner HF_TOKEN must belong to an account that accepted the license — metadata reads fine without it, so this fails late at dataloader build with GatedRepoError, not at startup. - Fresh checkpoint dir (checkpoint_path is group-scoped), so the model restarts from base and baseline loss re-converges over several cycles. Verified in the v0.3.4 image: config resolution, locked-field reset from a stale exp_legal YAML, seeded shard-pick on both sources honouring the pins, non-prefix windowing, live eval stream, and the full suite showing exactly the 4 pre-existing master failures and no new ones. Co-Authored-By: Claude Opus 5 --- connito/shared/config.py | 19 +- connito/test/test_locked_task_rederive.py | 50 +- docs/exp-legal-migration-plan.md | 8 + docs/miner-faq/architecture.md | 71 +- docs/miner-faq/faq.md | 35 +- docs/miner-faq/miner-config.md | 30 +- expert_groups/exp_legal/README.md | 14 +- expert_groups/exp_nemotron_c4/config.yaml | 61 ++ .../exp_nemotron_c4/expert_assignment.json | 802 ++++++++++++++++++ 9 files changed, 1010 insertions(+), 80 deletions(-) create mode 100644 expert_groups/exp_nemotron_c4/config.yaml create mode 100644 expert_groups/exp_nemotron_c4/expert_assignment.json diff --git a/connito/shared/config.py b/connito/shared/config.py index a410e72..3a8edd4 100644 --- a/connito/shared/config.py +++ b/connito/shared/config.py @@ -508,16 +508,23 @@ class ExpertCfg(BaseConfig): class TaskCfg(BaseConfig): - # `expert_group_name` is locked so operators can't drift back to - # `exp_math` after the subnet-wide switch to the legal expert group — + # `expert_group_name` is locked so the whole fleet evaluates the same task: # `auto_update_config` resets any non-default value on load and logs a - # one-time reset warning (see docs/exp-legal-migration-plan.md for the - # activation checklist). `helper_group_id` and `routing_mode` stay - # locked for the natural-routing (2Fnat) consensus contract. + # one-time reset warning. Validators only score miners whose chain commit + # carries a matching `expert_group`, so a drifting operator would simply + # stop seeing (and stop being seen by) everyone else. + # + # Currently `exp_nemotron_c4` (group 4): Nemotron-CC-Math + C4. This + # replaces the `exp_legal` switch made in #186 — see + # docs/exp-legal-migration-plan.md for that history and the flag-day + # mechanics, which apply identically in this direction. + # + # `helper_group_id` and `routing_mode` stay locked for the natural-routing + # (2Fnat) consensus contract and are independent of the dataset. _LOCKED_FIELDS: ClassVar[frozenset[str]] = frozenset({ "expert_group_name", "helper_group_id", "routing_mode", }) - expert_group_name: str = "exp_legal" + expert_group_name: str = "exp_nemotron_c4" load_all_expert_groups: bool = False base_path: Path = Path("expert_groups") path: Path | None = None diff --git a/connito/test/test_locked_task_rederive.py b/connito/test/test_locked_task_rederive.py index 4ff31bf..682de87 100644 --- a/connito/test/test_locked_task_rederive.py +++ b/connito/test/test_locked_task_rederive.py @@ -1,14 +1,19 @@ """Activation-order regression: when the locked `task.expert_group_name` -is auto-reset on load (the exp_legal activation path), the derived -`task.path` / `task.exp` must be re-derived in the same load. +is auto-reset on load, the derived `task.path` / `task.exp` must be +re-derived in the same load. -Observed live on the pioneer validator (2026-07-11 11:49 UTC): a config -YAML still saying `exp_math` was reset to the locked default `exp_legal` -and persisted to disk — but the process kept RUNNING exp_math -(`task_path=/app/expert_groups/exp_math`, chain commits `group_id: 0`) -because `task.path`/`task.exp` had been derived at construction, before -`check_and_prompt_locked` ran. Every fleet validator would have needed a -second restart to actually switch groups. +Observed live on the pioneer validator (2026-07-11 11:49 UTC) during the +exp_legal activation: a config YAML still saying `exp_math` was reset to +the then-locked default `exp_legal` and persisted to disk — but the +process kept RUNNING exp_math (`task_path=/app/expert_groups/exp_math`, +chain commits `group_id: 0`) because `task.path`/`task.exp` had been +derived at construction, before `check_and_prompt_locked` ran. Every +fleet validator would have needed a second restart to actually switch +groups. + +The locked default is now `exp_nemotron_c4` (group 4), so this test +exercises the same path in the reverse direction: an operator YAML left +on `exp_legal` must be reset AND re-derived in one load. """ from __future__ import annotations @@ -34,37 +39,38 @@ def _write_cfg(tmp_path: Path, expert_group_name: str) -> Path: def test_locked_reset_rederives_task_path(tmp_path: Path) -> None: # Run from the repo root so relative expert_groups/ resolves. - assert Path("expert_groups/exp_legal/config.yaml").exists(), ( + assert Path("expert_groups/exp_nemotron_c4/config.yaml").exists(), ( f"run from repo root (cwd={os.getcwd()})" ) - cfg_path = _write_cfg(tmp_path, "exp_math") + # An operator YAML left behind on the previous locked default. + cfg_path = _write_cfg(tmp_path, "exp_legal") config = MinerConfig.from_path(cfg_path, auto_update_config=True) # The locked default flipped the name... - assert config.task.expert_group_name == "exp_legal" + assert config.task.expert_group_name == "exp_nemotron_c4" # ...and the DERIVED task state must have followed in the same load: - assert config.task.path is not None and config.task.path.name == "exp_legal" - assert config.task.exp.group_id == 3, ( + assert config.task.path is not None and config.task.path.name == "exp_nemotron_c4" + assert config.task.exp.group_id == 4, ( f"stale task.exp — still group_id={config.task.exp.group_id} " - "(exp_math=0): check_and_prompt_locked reset the name without " - "re-deriving task.path/task.exp" + "(exp_math=0, exp_legal=3): check_and_prompt_locked reset the name " + "without re-deriving task.path/task.exp" ) # ...including the checkpoint path, which is group-scoped so the switch # writes/resumes from a fresh dir instead of the prior group's checkpoints. assert config.ckpt.checkpoint_path is not None - assert config.ckpt.checkpoint_path.name == "exp_legal", ( + assert config.ckpt.checkpoint_path.name == "exp_nemotron_c4", ( f"checkpoint_path leaf must track the effective group, got " f"{config.ckpt.checkpoint_path}" ) # And the persisted YAML matches what the process actually runs. persisted = yaml.safe_load(cfg_path.read_text()) - assert persisted["task"]["expert_group_name"] == "exp_legal" + assert persisted["task"]["expert_group_name"] == "exp_nemotron_c4" def test_no_reset_no_rederive_noise(tmp_path: Path) -> None: - # A config already on the locked default loads once, stays exp_legal. - cfg_path = _write_cfg(tmp_path, "exp_legal") + # A config already on the locked default loads once and stays put. + cfg_path = _write_cfg(tmp_path, "exp_nemotron_c4") config = MinerConfig.from_path(cfg_path, auto_update_config=True) - assert config.task.expert_group_name == "exp_legal" - assert config.task.exp.group_id == 3 + assert config.task.expert_group_name == "exp_nemotron_c4" + assert config.task.exp.group_id == 4 diff --git a/docs/exp-legal-migration-plan.md b/docs/exp-legal-migration-plan.md index 82cf0cf..c93d7a3 100644 --- a/docs/exp-legal-migration-plan.md +++ b/docs/exp-legal-migration-plan.md @@ -1,5 +1,13 @@ # Migration plan: `exp_math` → `exp_legal` (MultiLegalPile) +> **Historical record — this migration has since been reverted.** The subnet +> now runs `exp_nemotron_c4` (`group_id 4`, Nemotron-CC-Math + C4). This +> document is kept because its mechanics still apply verbatim to *any* expert +> group change: the locked-field activation path, the group-scoped checkpoint +> directory, and above all the flag-day risk in +> "Operational notes for the launch period" — a validator on a new group sees +> only miners that have already migrated, so the fleet must move together. + Plan for adding a new expert group that trains on [joelniklaus/Multi_Legal_Pile](https://huggingface.co/datasets/joelniklaus/Multi_Legal_Pile), running alongside the existing math expert group during a transition period. diff --git a/docs/miner-faq/architecture.md b/docs/miner-faq/architecture.md index cebdeea..306eb2a 100644 --- a/docs/miner-faq/architecture.md +++ b/docs/miner-faq/architecture.md @@ -73,25 +73,31 @@ Code: `connito/validator/run.py` (the single-loop main function), ## Expert-group sharding -The base model (DeepSeek-V2-Lite) has 8 experts per MoE layer. Connito -partitions those experts into **2 worker groups** (`exp_math` and `exp_dummy` -in the current configuration; `group_id = 0` and `group_id = 1`). A miner -trains exactly one group's experts and uploads only that group's shard. - -Group assignment is **declared by the miner in its config**, not chosen by -chain: `config.task.expert_group_name` picks the directory under -`expert_groups/`, and that directory's `config.yaml` defines the `group_id` and -the per-group dataset. There is no chain-level placement — the miner is free -to switch groups by changing its config (though doing so resets its score -history; see `scoring.md`). +The base model (DeepSeek-V2-Lite) has 64 routed experts per MoE layer, indexed +0–63, on MoE layers 1–26 (layer 0 is dense). Connito assigns a subset of those +experts to each **worker group**. A miner trains exactly one group's experts and +uploads only that group's shard. The active group is **`exp_nemotron_c4`** +(`group_id = 4`); other directories under `expert_groups/` are inactive or serve +as frozen helpers. + +The active group is **not** a free choice. `config.task.expert_group_name` is a +locked field: loading a config with `auto_update_config` resets any other value +back to the built-in default and rewrites it to your YAML, logging a one-time +reset warning. Setting it to something else does not switch groups — it is +overwritten on the next load. This is deliberate, because validators only +evaluate miners whose group matches their own. The validator filters all chain commits by `expert_group == config.task.exp.group_id` -when building its roster, so a validator running `exp_math` only evaluates -miners that also committed to `exp_math`. +when building its roster, so a validator on `exp_nemotron_c4` (`group_id = 4`) +only evaluates miners that also committed under `group_id = 4`. A miner still +committing under a previous group is invisible to the fleet and scores nothing — +so group changes are coordinated subnet-wide, announced in advance, and take +effect when you upgrade to the release that carries them. Code: `connito/shared/expert_manager.py:ExpertManager.load_expert_group_assignment`, -`connito/shared/config.py:TaskCfg`, -`connito/shared/cycle.py:get_miners_from_commit`. +`connito/shared/config.py:TaskCfg` (`_LOCKED_FIELDS`), +`connito/shared/cycle.py:get_miners_from_commit`, +`expert_groups/exp_nemotron_c4/expert_assignment.json`. ## Foundation / global checkpoint flow @@ -125,18 +131,35 @@ miner's HF shard. Validators evaluate miners against two HuggingFace streaming datasets, mixed 50/50: -- `allenai/c4` (split `en`) -- `nvidia/Nemotron-CC-Math-v1` (split `4plus`) - -Both use a per-source shuffle buffer of **50 000 rows** and a random skip up -to **50 000 rows** to prevent the eval set from being memorized. All -validators in a cycle use the same `combined_seed` derived from chain state -plus the block hash of the last MinerCommit2 block, so scores are reproducible -across validators but unpredictable to miners during their commit window. +- `allenai/c4` (config `en`) +- `nvidia/Nemotron-CC-Math-v1` (config `4plus`) + +**`nvidia/Nemotron-CC-Math-v1` is a gated dataset.** Your `HF_TOKEN` must +belong to a HuggingFace account that has accepted the license on that dataset's +page. Dataset metadata is readable without it, so the failure does not appear at +startup — file reads fail later with a `GatedRepoError` (HTTP 403) when the +dataloader is built. If your miner cannot stream training data, check this +first. Both datasets are pinned to a specific commit SHA via +`eval_source_revision_pin` so every validator reads identical rows. + +The eval slice is chosen by **seeded shard-pick**: the round's seed selects +which shard to open and a hash-derived offset into that shard, so across +rotating seeds every shard and every row is eventually reachable and no fixed +window can be memorized. Rows shorter than 200 characters are dropped and rows +repeating an already-seen 200-character prefix are skipped, so duplicated +boilerplate cannot inflate a score. Documents longer than the 1024-token +sequence length contribute a content-hash-derived window rather than always +their opening tokens. + +All validators in a cycle use the same `combined_seed`, derived from the block +hash of the last MinerCommit2 block, so scores are reproducible across +validators but unpredictable to miners during their commit window. Code: `connito/shared/cycle.py:get_combined_validator_seed`, `connito/shared/dataloader.py:get_dataloader`, -`expert_groups/exp_math/config.yaml` (dataset definition). +`connito/shared/eval_shard_pick.py:pick_shard_for_source`, +`connito/shared/dataloader.py:tokenize_windowed`, +`expert_groups/exp_nemotron_c4/config.yaml` (dataset definition). ## What miners can and cannot influence diff --git a/docs/miner-faq/faq.md b/docs/miner-faq/faq.md index 3363c7f..483010e 100644 --- a/docs/miner-faq/faq.md +++ b/docs/miner-faq/faq.md @@ -58,25 +58,28 @@ Code: `connito/validator/evaluator.py:finalize_round_scores`, ## Which expert group am I in, and how is that assigned? -You declare your expert group in your local config: +Your expert group is set in your local config, but the value is **locked**: ```yaml task: - expert_group_name: exp_math # directory under expert_groups/ + expert_group_name: exp_nemotron_c4 # locked — resets if you change it ``` -The directory's `config.yaml` defines `group_id`. The validator filters its -miner roster by `expert_group == config.task.exp.group_id` from each -`MinerChainCommit`, so the chain commit you write in MinerCommit2 must -agree with the directory you trained under. +The directory's `config.yaml` defines `group_id` (currently `4`). The validator +filters its miner roster by `expert_group == config.task.exp.group_id` from each +`MinerChainCommit`, so the chain commit you write in MinerCommit2 must agree +with the directory you trained under. -There is **no chain-level placement** — miners self-select. You can switch -groups by changing the config, but only validators running that group will -score you. +There is **no chain-level placement and no self-selection**. Loading a config +with `auto_update_config` resets `expert_group_name` to the built-in default and +rewrites your YAML, so editing it does not move you to another group. The entire +subnet runs one group at a time; a miner committing under any other group is +invisible to validators and earns nothing. Group changes are announced in +advance and take effect when you upgrade to the release that carries them. -Code: `connito/shared/config.py:TaskCfg`, +Code: `connito/shared/config.py:TaskCfg` (`_LOCKED_FIELDS`), `connito/shared/cycle.py:get_miners_from_commit`, -`expert_groups/exp_math/config.yaml`. +`expert_groups/exp_nemotron_c4/config.yaml`. ## How do I commit a checkpoint correctly? @@ -244,16 +247,18 @@ are trainable. Total memory is much smaller than a full DeepSeek-V2-Lite fine-tune. A 24 GB consumer GPU (3090 / 4090) is a reasonable minimum at the default -config (`fp16-mixed`, `batch_size=4`, `sequence_length=4096`, +config (`fp16-mixed`, `batch_size=4`, `sequence_length=1024`, `gradient_accumulation_steps=4`). Validators run on 40 GB GPUs because they have to load the full model for evaluation plus host the eval dataloader's buffers. If you're getting OOM during Train, reduce `task.exp.data.batch_size` -in the expert-group config; don't reduce `sequence_length` (the -validator's eval is at 4096 and you'll lose comparability). +in the expert-group config; don't reduce `sequence_length` (it is a +locked field and the validator's eval runs at 1024, so a different value +loses comparability). -Code: `connito/shared/config.py:ParallelismCfg`, `expert_groups/exp_math/config.yaml`. +Code: `connito/shared/config.py:ParallelismCfg`, +`expert_groups/exp_nemotron_c4/config.yaml`. ## Should I upload `.safetensors` or `.pt`? diff --git a/docs/miner-faq/miner-config.md b/docs/miner-faq/miner-config.md index 06dde85..f7e3c48 100644 --- a/docs/miner-faq/miner-config.md +++ b/docs/miner-faq/miner-config.md @@ -28,22 +28,28 @@ Code: `connito/shared/config.py:ChainCfg`. ```yaml task: - expert_group_name: exp_math # or exp_dummy — must match a dir under expert_groups/ + expert_group_name: exp_nemotron_c4 # locked — see below # exp.group_id is loaded from expert_groups//config.yaml ``` - The directory under `expert_groups/` defines the `group_id`, the dataset, - and the routing assignment. -- **Choose a group where at least one validator is running.** A miner - committing to a group no validator covers will never be evaluated and - will never earn rewards. The current production groups are - `exp_math` (`group_id = 0`) and `exp_dummy` (`group_id = 1`). -- Switching expert groups **does not** reset your aggregator history if the - hotkey stays the same — but the validator only evaluates miners whose - chain-committed `expert_group` matches its own active group, so a config - switch only takes effect after MinerCommit2 in the cycle of the change. - -Code: `connito/shared/config.py:TaskCfg`, + and the routing assignment. The active group is `exp_nemotron_c4` + (`group_id = 4`). +- **This field is locked, not a choice.** Loading a config with + `auto_update_config` resets any other value back to the built-in default, + rewrites your YAML, and logs a one-time reset warning. Editing it does not + switch groups. The whole subnet runs one group at a time so that validators + and miners always match. +- **A miner committing under a different group is never evaluated.** The + validator filters its roster by `expert_group == config.task.exp.group_id`, + so a stale group means no evaluation and no rewards — not a partial score. + When the subnet changes groups, it is announced in advance and takes effect + when you upgrade to the release carrying the new default. +- Changing groups **does not** reset your aggregator history if the hotkey + stays the same, but your accumulated scores age out of the rolling window + within a few rounds because the loss scale on a new dataset differs. + +Code: `connito/shared/config.py:TaskCfg` (`_LOCKED_FIELDS`), `connito/shared/expert_manager.py:ExpertManager.load_expert_group_assignment`, `connito/shared/cycle.py:get_miners_from_commit`. diff --git a/expert_groups/exp_legal/README.md b/expert_groups/exp_legal/README.md index 86d5951..4cfcffa 100644 --- a/expert_groups/exp_legal/README.md +++ b/expert_groups/exp_legal/README.md @@ -1,4 +1,16 @@ -# exp_legal — legal expert group +# exp_legal — legal expert group (INACTIVE) + +> **Status: inactive.** The subnet moved back to the Nemotron-CC-Math + C4 mix +> under `exp_nemotron_c4` (`group_id 4`), which is the current locked value of +> `TaskCfg.expert_group_name`. This group is retained for reference — its +> config, expert assignment, and the `Multi_Legal_Pile` policy in +> `connito/shared/eval_shard_pick.py:_KNOWN_SOURCES` (29-shard verified +> row-count table) all remain valid if legal data is revisited. +> +> Two details below are stale and kept only as historical record: this README +> predates the switch of `eval_source_seeded_shard_pick` to `true` and the +> regeneration of the expert assignment via `--min-share 0.035` (179 experts, +> variable per layer — not the "8 experts per layer" described here). Expert group that trains on [joelniklaus/Multi_Legal_Pile](https://huggingface.co/datasets/joelniklaus/Multi_Legal_Pile) diff --git a/expert_groups/exp_nemotron_c4/config.yaml b/expert_groups/exp_nemotron_c4/config.yaml new file mode 100644 index 0000000..3c4db11 --- /dev/null +++ b/expert_groups/exp_nemotron_c4/config.yaml @@ -0,0 +1,61 @@ +# group_id 4: 0=exp_math, 1=exp_dummy, 2=exp_c4_p02 (the standing frozen helper +# slot loaded alongside every active task via TaskCfg.helper_group_id), +# 3=exp_legal. This group carries the same Nemotron+C4 mix exp_math has always +# used, but as a fresh slot so the switch away from exp_legal is a clean cutover +# rather than a reuse of group 0's history. +group_id: 4 +data: + dataset_sources: + - path: "allenai/c4" + name: "en" + weight: 0.5 + text_column: "text" + - path: "nvidia/Nemotron-CC-Math-v1" + name: "4plus" + weight: 0.5 + text_column: "text" + # GATED DATASET. Every validator's HF_TOKEN must belong to an account + # that has accepted the license at + # https://huggingface.co/datasets/nvidia/Nemotron-CC-Math-v1 — metadata + # is readable without it, but FILE reads 403 with GatedRepoError, so a + # validator without access fails at dataloader build, not at startup. + # No `trust_remote_code` — neither source ships a loading script. It was + # only ever needed for Multi_Legal_Pile (see DatasetSourceCfg). + per_device_train_batch_size: 1 + batch_size: 4 + # 1024 matches the fleet paradigm and the VRAM envelope miners/validators are + # sized for under natural routing (locked DataCfg default; exp_math was moved + # 4096 -> 1024 by the 2Fnat PR). + sequence_length: 1024 + world_size: 10 # TODO: approximately how many miners are out there + rank: 1 # TODO: based on your uid + + # NOTE: `dataset_class` is deliberately NOT set. Leaving it unset resolves to + # `DefaultStreamingTorchDataset`, whose `tokenize_and_format` delegates to + # `connito.shared.dataloader.tokenize_windowed` — long documents contribute a + # content-hash-derived window instead of always their (most templated) prefix. + # `expert_groups/exp_math/dataset.py` still prefix-truncates, so pointing at it + # would silently drop that anti-memorization fix. + + # Whole-shard eval reach: seed picks the shard AND the in-shard offset, so + # across rotating seeds every shard and every row is eventually reachable. + # Inherits the DataCfg default (True); stated explicitly because it is part of + # the anti-memorization contract rather than an incidental default. + eval_source_seeded_shard_pick: true + + # The global eval gates (`eval_min_text_chars: 200`, + # `eval_dedup_prefix_chars: 200`) are inherited unchanged. They were tuned + # against Multi_Legal_Pile's pathologies (38% empty rows, 75% sharing a + # 200-char prefix), so they were re-measured against this mix before reuse + # (600-row sample per source, 2026-07-31): + # c4/en: 0% empty, 5.5% below 200 chars, 0% prefix-duplicate + # Nemotron/4plus: 0% empty, 0.0% below 200 chars, 0% prefix-duplicate + # i.e. effectively inert here — no per-group override needed. + + # Pin both sources to a commit SHA. `_KNOWN_SOURCES` declares `revision="main"` + # for these two, and `main` is a moving target: an HF re-upload mid-rollout + # would make two validators pick different rows for the same seed and break + # weight consensus for that round. Resolved 2026-07-31; bump deliberately. + eval_source_revision_pin: + "allenai/c4": "1588ec454efa1a09f29cd18ddd04fe05fc8653a2" + "nvidia/Nemotron-CC-Math-v1": "397a2502f2028c659ba411a6c4935b464a7f03aa" diff --git a/expert_groups/exp_nemotron_c4/expert_assignment.json b/expert_groups/exp_nemotron_c4/expert_assignment.json new file mode 100644 index 0000000..4a76bbd --- /dev/null +++ b/expert_groups/exp_nemotron_c4/expert_assignment.json @@ -0,0 +1,802 @@ +{ + "1": [ + [ + 0, + 17 + ], + [ + 1, + 27 + ], + [ + 2, + 13 + ], + [ + 3, + 57 + ], + [ + 4, + 60 + ], + [ + 5, + 25 + ], + [ + 6, + 24 + ], + [ + 7, + 55 + ] + ], + "2": [ + [ + 0, + 35 + ], + [ + 1, + 47 + ], + [ + 2, + 27 + ], + [ + 3, + 34 + ], + [ + 4, + 62 + ], + [ + 5, + 38 + ], + [ + 6, + 60 + ], + [ + 7, + 42 + ], + [ + 8, + 63 + ] + ], + "3": [ + [ + 0, + 22 + ], + [ + 1, + 62 + ], + [ + 2, + 45 + ], + [ + 3, + 7 + ], + [ + 4, + 27 + ], + [ + 5, + 48 + ], + [ + 6, + 1 + ], + [ + 7, + 49 + ], + [ + 8, + 46 + ] + ], + "4": [ + [ + 0, + 42 + ], + [ + 1, + 59 + ], + [ + 2, + 40 + ], + [ + 3, + 18 + ], + [ + 4, + 26 + ], + [ + 5, + 3 + ], + [ + 6, + 13 + ], + [ + 7, + 10 + ] + ], + "5": [ + [ + 0, + 14 + ], + [ + 1, + 39 + ], + [ + 2, + 25 + ], + [ + 3, + 53 + ], + [ + 4, + 59 + ], + [ + 5, + 40 + ], + [ + 6, + 15 + ], + [ + 7, + 12 + ] + ], + "6": [ + [ + 0, + 25 + ], + [ + 1, + 55 + ], + [ + 2, + 6 + ], + [ + 3, + 59 + ], + [ + 4, + 51 + ], + [ + 5, + 11 + ], + [ + 6, + 56 + ], + [ + 7, + 19 + ] + ], + "7": [ + [ + 0, + 22 + ], + [ + 1, + 8 + ], + [ + 2, + 28 + ], + [ + 3, + 16 + ], + [ + 4, + 41 + ], + [ + 5, + 61 + ], + [ + 6, + 5 + ] + ], + "8": [ + [ + 0, + 1 + ], + [ + 1, + 6 + ], + [ + 2, + 8 + ], + [ + 3, + 7 + ], + [ + 4, + 27 + ], + [ + 5, + 37 + ], + [ + 6, + 45 + ] + ], + "9": [ + [ + 0, + 56 + ], + [ + 1, + 10 + ], + [ + 2, + 11 + ], + [ + 3, + 41 + ], + [ + 4, + 61 + ], + [ + 5, + 1 + ] + ], + "10": [ + [ + 0, + 6 + ], + [ + 1, + 63 + ], + [ + 2, + 21 + ], + [ + 3, + 50 + ], + [ + 4, + 45 + ], + [ + 5, + 30 + ], + [ + 6, + 60 + ] + ], + "11": [ + [ + 0, + 30 + ], + [ + 1, + 18 + ], + [ + 2, + 63 + ], + [ + 3, + 5 + ], + [ + 4, + 28 + ] + ], + "12": [ + [ + 0, + 35 + ], + [ + 1, + 27 + ], + [ + 2, + 15 + ], + [ + 3, + 46 + ], + [ + 4, + 58 + ], + [ + 5, + 41 + ] + ], + "13": [ + [ + 0, + 39 + ], + [ + 1, + 33 + ], + [ + 2, + 61 + ], + [ + 3, + 40 + ], + [ + 4, + 30 + ], + [ + 5, + 44 + ], + [ + 6, + 25 + ] + ], + "14": [ + [ + 0, + 53 + ], + [ + 1, + 11 + ], + [ + 2, + 56 + ], + [ + 3, + 22 + ], + [ + 4, + 33 + ], + [ + 5, + 43 + ] + ], + "15": [ + [ + 0, + 59 + ], + [ + 1, + 9 + ], + [ + 2, + 19 + ], + [ + 3, + 16 + ], + [ + 4, + 58 + ] + ], + "16": [ + [ + 0, + 55 + ], + [ + 1, + 21 + ], + [ + 2, + 14 + ], + [ + 3, + 17 + ], + [ + 4, + 15 + ], + [ + 5, + 24 + ], + [ + 6, + 10 + ] + ], + "17": [ + [ + 0, + 6 + ], + [ + 1, + 25 + ], + [ + 2, + 17 + ], + [ + 3, + 57 + ], + [ + 4, + 13 + ] + ], + "18": [ + [ + 0, + 27 + ], + [ + 1, + 9 + ], + [ + 2, + 17 + ], + [ + 3, + 63 + ], + [ + 4, + 46 + ], + [ + 5, + 30 + ], + [ + 6, + 8 + ] + ], + "19": [ + [ + 0, + 54 + ], + [ + 1, + 62 + ], + [ + 2, + 32 + ], + [ + 3, + 7 + ], + [ + 4, + 47 + ], + [ + 5, + 14 + ], + [ + 6, + 61 + ] + ], + "20": [ + [ + 0, + 18 + ], + [ + 1, + 50 + ], + [ + 2, + 19 + ], + [ + 3, + 27 + ], + [ + 4, + 6 + ], + [ + 5, + 4 + ], + [ + 6, + 21 + ] + ], + "21": [ + [ + 0, + 48 + ], + [ + 1, + 35 + ], + [ + 2, + 25 + ], + [ + 3, + 53 + ], + [ + 4, + 44 + ], + [ + 5, + 42 + ], + [ + 6, + 37 + ] + ], + "22": [ + [ + 0, + 62 + ], + [ + 1, + 21 + ], + [ + 2, + 61 + ], + [ + 3, + 9 + ], + [ + 4, + 2 + ], + [ + 5, + 60 + ], + [ + 6, + 40 + ] + ], + "23": [ + [ + 0, + 54 + ], + [ + 1, + 25 + ], + [ + 2, + 35 + ], + [ + 3, + 23 + ], + [ + 4, + 38 + ], + [ + 5, + 6 + ], + [ + 6, + 43 + ], + [ + 7, + 57 + ] + ], + "24": [ + [ + 0, + 52 + ], + [ + 1, + 51 + ], + [ + 2, + 28 + ], + [ + 3, + 27 + ], + [ + 4, + 58 + ], + [ + 5, + 23 + ], + [ + 6, + 22 + ], + [ + 7, + 61 + ], + [ + 8, + 36 + ] + ], + "25": [ + [ + 0, + 1 + ], + [ + 1, + 44 + ], + [ + 2, + 40 + ], + [ + 3, + 45 + ], + [ + 4, + 28 + ], + [ + 5, + 25 + ], + [ + 6, + 54 + ], + [ + 7, + 51 + ], + [ + 8, + 39 + ] + ], + "26": [ + [ + 0, + 28 + ], + [ + 1, + 16 + ], + [ + 2, + 29 + ], + [ + 3, + 23 + ], + [ + 4, + 58 + ], + [ + 5, + 56 + ], + [ + 6, + 36 + ], + [ + 7, + 24 + ] + ] +} \ No newline at end of file