Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 46 additions & 16 deletions .github/scripts/hf_release_readiness_terminal.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import requests
from huggingface_hub import HfApi
from kernel_hub_git import KernelHubGitTransport

REPORT_SCHEMA = "szl.hf-release-readiness/v1"
PRERELEASE_SCHEMA = "szl.hf-release-readiness/v1-prerelease"
Expand Down Expand Up @@ -125,6 +126,7 @@ def __init__(self, *, token: str, generation: str) -> None:
)
self.token = token
self.api = HfApi(token=token)
self.kernel_transport = KernelHubGitTransport(token=token)
self.actions: list[Action] = []
self.results: dict[str, Any] = {}

Expand Down Expand Up @@ -271,27 +273,51 @@ def _kernel_tree_paths(self, repo_id: str, revision: str) -> tuple[str, ...]:
)
return paths

def verify_kernel(self, repo_id: str) -> None:
revision = _immutable_revision(
getattr(self.api.kernel_info(repo_id), "sha", ""),
label=repo_id,
)
paths = self._kernel_tree_paths(repo_id, revision)
def _kernel_revision(self, repo_id: str) -> tuple[str, str]:
try:
revision = getattr(self.api.kernel_info(repo_id), "sha", "")
source = "kernel-api"
except ValueError as exc:
if str(exc) != "min() iterable argument is empty":
raise
revision = self.kernel_transport.snapshot(repo_id).revision
source = "authenticated-kernel-hub-git-fallback"
return _immutable_revision(revision, label=repo_id), source

from kernels import get_kernel
def _kernel_selfcheck(self, repo_id: str, revision: str) -> tuple[Any, str]:
from kernels import get_kernel, get_local_kernel

module = get_kernel(repo_id, revision=revision, trust_remote_code=True)
check = getattr(module, "selfcheck", None)
if not callable(check):
try:
module = get_kernel(
repo_id,
revision=revision,
trust_remote_code=True,
)
except ValueError as exc:
if str(exc) != "min() iterable argument is empty":
raise
with self.kernel_transport.materialize_build(repo_id, revision) as repo:
module = get_local_kernel(repo)
result = module.selfcheck() if callable(getattr(module, "selfcheck", None)) else None
transport = "authenticated-kernel-hub-git-fallback"
else:
check = getattr(module, "selfcheck", None)
result = check() if callable(check) else None
transport = "kernel-api"

if result is None:
raise RuntimeError(f"{repo_id}@{revision} does not expose selfcheck()")
result = check()
if not _selfcheck_passed(result):
raise RuntimeError(f"{repo_id}@{revision} selfcheck did not pass: {result}")
return result, transport

revision_after = _immutable_revision(
getattr(self.api.kernel_info(repo_id), "sha", ""),
label=repo_id,
)
def verify_kernel(self, repo_id: str) -> None:
revision, metadata_source = self._kernel_revision(repo_id)
paths = self._kernel_tree_paths(repo_id, revision)

result, selfcheck_transport = self._kernel_selfcheck(repo_id, revision)

revision_after, metadata_source_after = self._kernel_revision(repo_id)
if revision_after != revision:
raise RuntimeError(
f"kernel revision moved during selfcheck: {repo_id}; "
Expand All @@ -303,13 +329,17 @@ def verify_kernel(self, repo_id: str) -> None:
"remote_file_count": len(paths),
"build_variants_present": True,
"metadata_stable": True,
"metadata_revision_source": metadata_source,
"metadata_revision_source_after": metadata_source_after,
"selfcheck": result,
"selfcheck_transport": selfcheck_transport,
}
self.record(
repo_id,
"kernel-tree-and-selfcheck",
"validated",
f"revision={revision}; files={len(paths)}; metadata_stable=true",
f"revision={revision}; files={len(paths)}; metadata_stable=true; "
f"metadata_source={metadata_source}; selfcheck_transport={selfcheck_transport}",
)

def report(self) -> dict[str, Any]:
Expand Down
86 changes: 86 additions & 0 deletions .github/scripts/test_hf_release_readiness_terminal.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
import pathlib
import sys
import unittest
import unittest.mock
from contextlib import contextmanager
from types import SimpleNamespace

HERE = pathlib.Path(__file__).resolve().parent
ROOT = HERE.parents[1]
Expand Down Expand Up @@ -70,6 +73,89 @@ def test_selfcheck_parser_requires_positive_evidence(self) -> None:
)
self.assertFalse(terminal._selfcheck_passed({"checks": {}}))

def test_empty_kernel_metadata_uses_exact_git_revision(self) -> None:
verifier = terminal.TerminalReadiness(
token="test-token",
generation="a" * 40,
)
verifier.api = SimpleNamespace(
kernel_info=lambda repo_id: (_ for _ in ()).throw(
ValueError("min() iterable argument is empty")
)
)
verifier.kernel_transport = SimpleNamespace(
snapshot=lambda repo_id: SimpleNamespace(revision="b" * 40)
)
revision, source = verifier._kernel_revision("SZLHOLDINGS/example")
self.assertEqual(revision, "b" * 40)
self.assertEqual(source, "authenticated-kernel-hub-git-fallback")

def test_unrelated_kernel_metadata_error_fails_closed(self) -> None:
verifier = terminal.TerminalReadiness(
token="test-token",
generation="a" * 40,
)
verifier.api = SimpleNamespace(
kernel_info=lambda repo_id: (_ for _ in ()).throw(
ValueError("malformed kernel metadata")
)
)
with self.assertRaisesRegex(ValueError, "malformed kernel metadata"):
verifier._kernel_revision("SZLHOLDINGS/example")

def test_empty_kernel_loader_uses_exact_git_build(self) -> None:
verifier = terminal.TerminalReadiness(
token="test-token",
generation="a" * 40,
)
materialize_calls = []

@contextmanager
def materialize_build(repo_id, revision):
materialize_calls.append((repo_id, revision))
yield pathlib.Path("materialized-kernel")

verifier.kernel_transport = SimpleNamespace(
materialize_build=materialize_build
)
module = SimpleNamespace(selfcheck=lambda: {"ok": True})
kernels = SimpleNamespace(
get_kernel=unittest.mock.Mock(
side_effect=ValueError("min() iterable argument is empty")
),
get_local_kernel=unittest.mock.Mock(return_value=module),
)
with unittest.mock.patch.dict(sys.modules, {"kernels": kernels}):
result, source = verifier._kernel_selfcheck(
"SZLHOLDINGS/example",
"b" * 40,
)
self.assertEqual(result, {"ok": True})
self.assertEqual(source, "authenticated-kernel-hub-git-fallback")
self.assertEqual(
materialize_calls,
[("SZLHOLDINGS/example", "b" * 40)],
)
kernels.get_local_kernel.assert_called_once_with(
pathlib.Path("materialized-kernel")
)

def test_unrelated_kernel_loader_error_fails_closed(self) -> None:
verifier = terminal.TerminalReadiness(
token="test-token",
generation="a" * 40,
)
kernels = SimpleNamespace(
get_kernel=unittest.mock.Mock(
side_effect=ValueError("malformed build metadata")
),
get_local_kernel=unittest.mock.Mock(),
)
with unittest.mock.patch.dict(sys.modules, {"kernels": kernels}):
with self.assertRaisesRegex(ValueError, "malformed build metadata"):
verifier._kernel_selfcheck("SZLHOLDINGS/example", "b" * 40)
kernels.get_local_kernel.assert_not_called()

def test_issue_body_contains_machine_readable_readiness_report(self) -> None:
report = {
"schema": terminal.REPORT_SCHEMA,
Expand Down
Loading