feat(notebook): add a notebook plugin

.gitignore

@@ -37,3 +37,20 @@ charts/taugrid/charts
 # Portal frontend dependencies and local build metadata
 portal/frontend/node_modules/
 portal/frontend/*.tsbuildinfo
+# Agent scratch: briefs, reports, throwaway environments.
+.codex-review/
+sdk/python/python/share/
+sdk/python/python/build/
+sdk/python/python/tau.egg-info/
+
+# Local Jupyter extension build output and e2e artifacts
+sdk/python/python/labextension/node_modules/
+sdk/python/python/labextension/lib/
+sdk/python/python/labextension/tsconfig.tsbuildinfo
+jupyter-*-e2e*.png
+tools/jupyter-*.png
+# Python bytecode
+__pycache__/
+*.pyc
+# Jupyter checkpoints
+.ipynb_checkpoints/

examples/notebook-loss-curve-demo.ipynb

@@ -0,0 +1,9 @@
+{
+ "cells": [
+  {"cell_type": "markdown", "metadata": {}, "source": ["# Deterministic CPU loss curve\n", "Pure Python gradient descent fits y = 2x + 1. Forty real observations span about forty seconds so the existing status poll can collect multiple windows. No downloads or GPU packages are needed. Submit using an explicitly reviewed CPU profile.\n"]},
+  {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["import time\n", "\n", "inputs = (-2.0, -1.0, 0.0, 1.0, 2.0)\n", "targets = tuple(2.0 * value + 1.0 for value in inputs)\n", "weight, bias = 0.0, 0.0\n", "for step in range(40):\n", "    errors = tuple(weight * value + bias - target for value, target in zip(inputs, targets))\n", "    loss = sum(error * error for error in errors) / len(inputs)\n", "    print(f'step={step} loss={loss:.12g}', flush=True)\n", "    weight -= 0.1 * 2.0 * sum(error * value for error, value in zip(errors, inputs)) / len(inputs)\n", "    bias -= 0.1 * 2.0 * sum(errors) / len(inputs)\n", "    time.sleep(1.0)\n", "print(f'Finished: weight={weight:.6f}, bias={bias:.6f}', flush=True)\n"]}
+ ],
+ "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.11"}},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/notebook-ray-cpu-demo.ipynb

@@ -0,0 +1,80 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "intro",
+   "metadata": {},
+   "source": [
+    "# TauGrid CPU Ray demo\n",
+    "\n",
+    "Submitted through the TauGrid JupyterLab plugin. It runs on a CPU-only Ray worker\n",
+    "profile, so it needs no GPU quota.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "connect",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "from math import sqrt\n",
+    "\n",
+    "import ray\n",
+    "\n",
+    "# On the cluster the RayJob starts the cluster, so connect to it; locally, start one.\n",
+    "try:\n",
+    "    ray.init(address=\"auto\", ignore_reinit_error=True, logging_level=\"ERROR\")\n",
+    "except Exception:\n",
+    "    ray.init(ignore_reinit_error=True, logging_level=\"ERROR\")\n",
+    "\n",
+    "resources = ray.cluster_resources()\n",
+    "print(\"ray\", ray.__version__)\n",
+    "print(\"nodes\", len(ray.nodes()))\n",
+    "print(\"cpus\", resources.get(\"CPU\"))\n",
+    "print(\"gpus\", resources.get(\"GPU\", 0))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "workload",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@ray.remote(num_cpus=1)\n",
+    "def busy_sum(n: int) -> float:\n",
+    "    return sum(sqrt(i) for i in range(n))\n",
+    "\n",
+    "started = time.time()\n",
+    "results = ray.get([busy_sum.remote(2_000_000) for _ in range(8)])\n",
+    "elapsed = time.time() - started\n",
+    "print(f\"CPU_DEMO tasks={len(results)} total={sum(results):.2f} elapsed={elapsed:.2f}s\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "done",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"CPU_DEMO_OK\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/notebook-ray-cpu-demo.md

@@ -0,0 +1,98 @@
+# CPU Ray demo for the TauGrid notebook plugin
+
+Runs a CPU-only Ray job through the TauGrid JupyterLab plugin, so it needs no GPU
+quota, and shows the plugin tracking it from admission to completion.
+
+- Notebook: [notebook-ray-cpu-demo.ipynb](notebook-ray-cpu-demo.ipynb)
+- Driver: `tools/run-cpu-ray-demo.py` (drives the same endpoints the panel uses)
+
+## What the job does
+
+Connects to the Ray cluster the RayJob starts, then runs eight `@ray.remote`
+CPU tasks and prints the Ray version, node and CPU counts, GPU count, and the
+result. It finishes with `CPU_DEMO_OK`.
+
+## Prerequisites
+
+1. **A notebook runtime image.** The platform's AI runtime image ships Ray but no
+   notebook executor, and cluster pods cannot reach PyPI on a locked-down
+   network, so build the executor in:
+
+   ```bash
+   python -m pip download --dest images/notebook-runtime/wheels \
+       --platform manylinux2014_x86_64 --python-version 3.12 --implementation cp --abi cp312 \
+       --only-binary=:all: "nbformat>=5.10" "nbconvert>=7.16" "ipykernel>=6.29" \
+       "pexpect>4.6" "ptyprocess"
+   docker build -t taugrid-notebook-runtime:local images/notebook-runtime
+   ```
+
+   The extra `pexpect`/`ptyprocess` downloads are required because pip
+   evaluates `sys_platform` markers against the host, not the target platform.
+
+2. **Point the server at it** and enable submission:
+
+   ```bash
+   TAUGRID_RUNTIME_IMAGE=taugrid-notebook-runtime:local \
+   TAUGRID_SUBMISSION_ENABLED=1 \
+   jupyter server --no-browser --port=8888 --ServerApp.token=<token>
+   ```
+
+3. **A CPU workload profile.** The shipped profiles all request GPUs, so add a
+   CPU one to the TauCluster:
+
+   ```bash
+   kubectl patch cluster cluster --type=json -p '[{"op":"add","path":"/spec/workloadProfiles/-","value":{
+     "name":"azure.research.cpu.small","description":"CPU-only Ray workers.",
+     "mode":"fixed","workerCount":1,"gpusPerWorker":0,"defaultLocalQueue":"jobqueue",
+     "executionTarget":"singleCluster","placement":"independent",
+     "applicability":{"lanes":["training"],"teams":["research"],"namespaces":["tau-notebook-e2e"]},
+     "priorities":{"podPriorityClassName":"taugrid-default","workloadPriorityClassName":"taugrid-default"}}}]'
+   ```
+
+4. **A LocalQueue in the run namespace**, and the namespace label the ClusterQueue
+   selects on. Without both, the workload stays unadmitted:
+
+   ```bash
+   kubectl create namespace tau-notebook-e2e --dry-run=client -o yaml | kubectl apply -f -
+   kubectl label ns tau-notebook-e2e tau.azure.com/workspace=tau-notebook-e2e --overwrite
+   kubectl apply -f - <<'YAML'
+   apiVersion: kueue.x-k8s.io/v1beta2
+   kind: LocalQueue
+   metadata:
+     name: jobqueue
+     namespace: tau-notebook-e2e
+   spec:
+     clusterQueue: jobqueue
+   YAML
+   ```
+
+## Run it
+
+```bash
+python tools/run-cpu-ray-demo.py --token <token> --name cpu-demo --timeout 900
+```
+
+Expected: the plan resolves `azure.research.cpu.small` on queue `jobqueue`, then
+`state=queued` -> `state=running` -> `state=complete` with `RESULT: SUCCEEDED`.
+
+In the JupyterLab panel: open **TauGrid: Open runs**, set the namespace, press
+**List runs**, and open the run. The detail tab shows Finished, Admitted, the
+queue, and 2/2 pods; **Open logs** shows the bounded pod log snapshot.
+
+## Notes
+
+- The RayJob sets `ttlSecondsAfterFinished: 15`, so pods are removed 15 seconds
+  after the job ends. Patch the RayJob to a larger TTL if you want to read the
+  executed notebook or the Ray driver log afterwards:
+
+  ```bash
+  kubectl patch rayjob <name> -n <ns> --type=merge -p '{"spec":{"ttlSecondsAfterFinished":900}}'
+  kubectl exec -n <ns> <head-pod> -c ray-head -- \
+      cat /tmp/ray/session_latest/logs/job-driver-*.log
+  kubectl exec -n <ns> <head-pod> -c ray-head -- python3 -c \
+      "import json; nb=json.load(open('/data/analysis.executed.ipynb')); print(''.join(t for c in nb['cells'] for o in c.get('outputs',[]) for t in o.get('text',[])))"
+  ```
+
+- The entrypoint still runs a `pip install nbconvert ipykernel` preamble. With a
+  runtime image that already contains them it is a no-op; on an image without
+  them it fails, because pods have no PyPI access here.

images/notebook-runtime/Dockerfile

@@ -0,0 +1,33 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# TauGrid notebook runtime: ray plus the notebook executor.
+#
+# The base AI runtime image ships ray but no notebook execution stack, and both
+# pods and image builds can be cut off from PyPI on a locked-down network, so the
+# executor is baked in at build time. Populate wheels/ (gitignored) to build
+# offline:
+#
+#   python -m pip download --dest images/notebook-runtime/wheels \
+#       "nbformat>=5.10" "nbconvert>=7.16" "ipykernel>=6.29"
+#   docker build -t taugrid-notebook-runtime:local images/notebook-runtime
+#
+# With an empty wheels/ the build falls back to the package index. Publish the
+# image through the normal pipeline; notebook submission stays disabled until a
+# runtime image like this is published and recorded.
+
+FROM mcr.microsoft.com/aks/ai-runtime/ray:py3.12-ray2.56.0-cuda13.0
+
+COPY wheels/ /wheels/
+
+RUN if ls /wheels/*.whl >/dev/null 2>&1; then \
+        python3 -m pip install --no-cache-dir --no-index --find-links=/wheels \
+            "nbformat>=5.10" "nbconvert>=7.16" "ipykernel>=6.29"; \
+    else \
+        python3 -m pip install --no-cache-dir \
+            "nbformat>=5.10" "nbconvert>=7.16" "ipykernel>=6.29"; \
+    fi
+
+# Prove the executor is importable at build time, so a bad image fails here
+# rather than in a user's job.
+RUN python3 -c "import nbformat, nbconvert, ipykernel; print('notebook executor ready')"

images/notebook-runtime/wheels/.gitignore

@@ -0,0 +1,2 @@
+# Vendored only for local/offline image builds; never committed.
+*.whl

scripts/license-header-exclusions.txt

@@ -17,3 +17,8 @@ charts/adx-mon/charts/
 # Generated portal bundles contain third-party JavaScript; licenses are shipped
 # alongside them in assets/THIRD_PARTY_LICENSES.txt. Check first-party source.
 portal/internal/portalapi/assets/assets/
+
+# Generated JupyterLab extension bundles contain third-party JavaScript;
+# licenses are shipped alongside them in static/third-party-licenses.json.
+# Check first-party source instead.
+sdk/python/python/tau/labextension/static/

sdk/python/python/examples/nb_helpers.py

@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""A sibling module a notebook can ship with, chosen in the submit review."""
+
+VALUE = 41
+
+
+def total(values):
+    return sum(values) + VALUE

sdk/python/python/examples/notebook-button-e2e.ipynb

@@ -0,0 +1,82 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b0e2d001",
+   "metadata": {},
+   "source": [
+    "# TauGrid plugin button E2E\n",
+    "\n",
+    "The panel below carries the real interactive Submit button. Its submit is\n",
+    "backed by an in-kernel recording fake, so a browser click can be asserted\n",
+    "end-to-end without a live cluster: the click drives the same handler the\n",
+    "button registers, and the status line repaints into run view."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "b0e2d002",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext tau.widgets.ipython"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b0e2d003",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tau.widgets.panel import TauGridPanel\n",
+    "from tau._backend import SubmittedRun\n",
+    "from IPython.display import display\n",
+    "\n",
+    "recorded = []\n",
+    "\n",
+    "def fake_submit(notebook=None, **kwargs):\n",
+    "    recorded.append(notebook)\n",
+    "    panel.run_name = \"submitted-button-demo\"  # status repaint proves the click ran\n",
+    "    return SubmittedRun(name=panel.run_name, namespace=panel.namespace, kind=\"RayJob\")\n",
+    "\n",
+    "panel = TauGridPanel(namespace=\"ray\")\n",
+    "panel.set_notebook_path(\"examples/notebook-button-e2e.ipynb\")\n",
+    "panel.submit = fake_submit\n",
+    "display(panel.build())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b0e2d004",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "recorded"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.14.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}