Feature/omnihub integration

.claude/commands/init-cluster.md

@@ -69,9 +69,9 @@ echo "HTTPS_PROXY=${HTTPS_PROXY:-<unset>}"
 find "$HOME" /scratch /projects /opt -maxdepth 4 -name "*.sif" 2>/dev/null | head -20
 
 # 14. Performance tooling (perf-analysis / perf-optimizer-loop recipes — optional)
-# Look for an existing shared perf-tools dir (omnistat + TraceLens venv "perf-inspect/").
+# Look for an existing shared perf-tools dir (omnistat + TraceLens venv "omnihub-inspect/").
 for d in /shared/*/tools /shared/perf-tools "$HOME/perf-tools"; do
-    [[ -d "$d/perf-inspect" || -d "$d/omnistat-src" ]] && echo "perf_tools: $d"
+    [[ -d "$d/omnihub-inspect" || -d "$d/omnistat-src" ]] && echo "perf_tools: $d"
 done
 ```
 
@@ -127,8 +127,8 @@ If no internet, ask for proxy settings.
 
 **Q9. Performance tooling** (optional — only for `perf-analysis` / `perf-optimizer-loop` recipes)
 If a perf-tools dir was discovered (step 14), show it and ask to confirm; otherwise ask for the path
-or leave blank. This is exported to perf scripts as `PERF_TOOLS_DIR`; expected layout under it:
-`perf-inspect/` (omnistat + TraceLens venv), `omnistat-src/`, `victoriametrics/victoria-metrics-prod`.
+or leave blank. This is exported to perf scripts as `OMNIHUB_TOOLS_DIR`; expected layout under it:
+`omnihub-inspect/` (omnistat + TraceLens venv), `omnistat-src/`, `victoriametrics/victoria-metrics-prod`.
 
 **Q10. Config file location**
 - `.cluster-config.yaml` in this repo (recommended — per-checkout)
@@ -173,9 +173,9 @@ network:
   libionic_path: ""      # path to libionic.so.1 (default: /usr/lib/x86_64-linux-gnu/libionic.so.1)
 
 # Performance tooling — only if using perf-analysis / perf-optimizer-loop recipes (Q9).
-# Exported to scripts as PERF_TOOLS_DIR. Leave dir blank if not used.
+# Exported to scripts as OMNIHUB_TOOLS_DIR. Leave dir blank if not used.
 perf_tools:
-  dir: "<perf-tools dir or empty>"   # layout: perf-inspect/, omnistat-src/, victoriametrics/
+  dir: "<perf-tools dir or empty>"   # layout: omnihub-inspect/, omnistat-src/, victoriametrics/
 
 discovered_sifs:
   # SIF files found during init (informational — confirm they are under sif_cache)
@@ -196,4 +196,4 @@ Suggest what the user can do next:
 - If **Q9 (Performance tooling)** was configured: run ORBIT-2 or HydraGNN perf recipes with `/run-perf-orbit2` or `/run-perf-hydragnn`
 - The run commands will read cluster config automatically for SLURM partition/account defaults
 
-$ARGUMENTS
+$ARGUMENTS

.claude/commands/run-perf-hydragnn.md

@@ -15,7 +15,7 @@ Read `.cluster-config.yaml` (repo root) or `~/.config/ai4science-studio/cluster.
 **Required for perf recipes:**
 
 - Set **`AI4S_SHARED_DIR`** from cluster config (`paths.scratch` or documented scratch root).
-- Set **`PERF_TOOLS_DIR`** from `perf_tools.dir`. If missing, stop and send the user to **`/init-cluster`** Q9 (Performance tooling).
+- Set **`OMNIHUB_TOOLS_DIR`** from `omnihub.tools_dir`. If missing, stop and send the user to **`/init-cluster`** Q9 (Performance tooling).
 
 Pre-fill SLURM partition/account from config.
 
@@ -92,7 +92,7 @@ Confirm branch **`Predictive_GFM_2024`** if user cares about upstream alignment
 
 ```bash
 export AI4S_SHARED_DIR=<from cluster config>
-export PERF_TOOLS_DIR=<from cluster config perf_tools.dir>
+export OMNIHUB_TOOLS_DIR=<from cluster config omnihub.tools_dir>
 # Optional overrides:
 # export HG_SIF=...  HG_OVERLAY=...  HG_REPO_DIR=...  HG_DATASETS=...  HG_DATA_DIR=...
 ```

.claude/commands/run-perf-orbit2.md

@@ -13,7 +13,7 @@ Read `.cluster-config.yaml` (repo root) or `~/.config/ai4science-studio/cluster.
 **Required for perf recipes:**
 
 - `paths.scratch` (or equivalent) → use as **`AI4S_SHARED_DIR`** (same convention as other studio scripts).
-- `perf_tools.dir` → use as **`PERF_TOOLS_DIR`** (layout: `perf-inspect/`, `omnistat-src/`, `victoriametrics/`). If missing or empty, stop and tell the user to re-run **`/init-cluster`** and answer **Q9 (Performance tooling)**.
+- `omnihub.tools_dir` → use as **`OMNIHUB_TOOLS_DIR`** (layout: `omnihub-inspect/`, `omnistat-src/`, `victoriametrics/`). If missing or empty, stop and tell the user to re-run **`/init-cluster`** and answer **Q9 (Performance tooling)**.
 
 Pre-fill SLURM partition/account from `slurm.partition` / `slurm.account` when present.
 
@@ -97,7 +97,7 @@ Export from cluster config + user answers:
 
 ```bash
 export AI4S_SHARED_DIR=<from cluster config paths.scratch>
-export PERF_TOOLS_DIR=<from cluster config perf_tools.dir>
+export OMNIHUB_TOOLS_DIR=<from cluster config omnihub.tools_dir>
 export ORBIT2_ROOT=<path to Bayes-CAST or ORBIT-2 tree with launch/train_edm.py>
 export ORBIT2_SIF=<path>                    # if not using default under $AI4S_SHARED_DIR/images/...
 export ORBIT2_OVERLAY=<path>                # optional if script defaults suffice

.cluster-config.example.yaml

@@ -41,9 +41,18 @@ network:
   rccl_anp_plugin: ""    # Path to librccl-anp.so on host (default: /opt/rocm/lib/librccl-anp.so)
   libionic_path: ""      # Path to libionic.so.1 on host (default: /usr/lib/x86_64-linux-gnu/libionic.so.1)
 
-# Performance tooling (perf-analysis / perf-optimizer-loop recipes)
-# Shared location of the omnistat + TraceLens perf tools, exported to scripts as
-# PERF_TOOLS_DIR. Expected layout under this dir: perf-inspect/ (Python venv with
-# omnistat + TraceLens), omnistat-src/, victoriametrics/victoria-metrics-prod.
-perf_tools:
-  dir: ""                # e.g. "/shared/perf-tools"; leave blank if not using perf recipes
+# OmniHub integration (sibling repo — see integrations/omnihub/README.md)
+omnihub:
+  repo_dir: ""             # e.g. /home/$USER/git/omnihub
+  cluster: vultr           # maps to omnihub config/vultr.yaml on Lux
+  shared_dir: /shared/omnihub
+  results_dir: ""          # default: /shared/$USER/results/omnihub
+  rocm_version: "7.2.2"
+  # Shared perf/telemetry tools. Cluster-specific — set per site, do not hardcode
+  # these into recipes/scripts. Default: <shared_dir>/tools.
+  # The HydraGNN/ORBIT-2 perf recipes read this value via the OMNIHUB_TOOLS_DIR
+  # env var (export OMNIHUB_TOOLS_DIR=<tools_dir> before running them).
+  tools_dir: ""            # default: /shared/omnihub/tools
+  #   <tools_dir>/omnistat              GPU telemetry, no PMC          (--tools omnistat)
+  #   <tools_dir>/omnistat-rocprofiler  adds rocprofiler PMC counters  (--tools omnistat-rocprofiler-pmc1)
+  #   <tools_dir>/omnihub-inspect       omnistat-inspect + TraceLens venv (legacy deep-dive)

.cursor/skills/ai4science-omnihub/SKILL.md

@@ -0,0 +1,97 @@
+---
+name: ai4science-omnihub
+description: Launch AI4Science Studio models via OmniHub job generation, SLURM submission, and standardized result processing. Use when the user asks for OmniHub, train-omnihub, perf-analysis-omnihub, omnihub-generate-job, or post-processing OmniHub results for ai4science models.
+---
+
+# AI4Science Studio + OmniHub
+
+Integrates this repo with a sibling [OmniHub](https://github.com/AMDResearch/omnihub) checkout **without merging repos**.
+
+## Prerequisites
+
+1. Read `.cluster-config.yaml` (or `~/.config/ai4science-studio/cluster.yaml`) — especially `omnihub:` and `paths`.
+2. Set `export OMNIHUB_DIR=/path/to/omnihub` and `export AI4S_SHARED_DIR=/shared/$USER` (or your site path).
+3. HydraGNN assets: SIF, overlay, datasets, code clone under `$AI4S_SHARED_DIR/models/HydraGNN/`.
+
+> **Omnistat note:** OmniHub's `/shared/omnihub/tools/omnistat` has no `omnistat-inspect`, so don't expect inspect JSON from OmniHub jobs — read `omnihub-process` output under `processed-data/` instead. For deep `omnistat-inspect`/PromQL workflows use the `omnihub-inspect` build at `/shared/omnihub/tools/omnihub-inspect` (see perf-analysis recipe).
+
+## Workflow: HydraGNN train-omnihub
+
+1. Read `material_science/models/HydraGNN/model.yaml` recipe `train-omnihub`.
+2. Sync app stubs and generate SLURM script:
+
+```bash
+./integrations/omnihub/generate-job.sh \
+  --num-nodes 2 \
+  --partition lux \
+  --time-limit 2h \
+  --output /tmp/hydragnn-omnihub.slurm
+```
+
+3. Submit (account on `sbatch`, not in the generated script):
+
+```bash
+sbatch -A vultr_lux /tmp/hydragnn-omnihub.slurm
+```
+
+4. Monitor: `squeue -j <jobid>`, `tail -f /shared/$USER/results/omnihub/<jobid>/*.out`
+
+5. After completion — post-process:
+
+```bash
+$OMNIHUB_DIR/omnihub-process --results-dir /shared/$USER/results/omnihub -j 4
+$OMNIHUB_DIR/omnihub-index --results-dir /shared/$USER/results/omnihub --output index
+```
+
+## Workflow: perf-analysis-omnihub (Phase 2)
+
+Same as train, but add `--perf` to enable `omnistat`, `pytorch-trace`, and `tracelens`:
+
+```bash
+./integrations/omnihub/generate-job.sh --perf --num-nodes 2 --time-limit 30m \
+  --output /tmp/hydragnn-perf.slurm
+sbatch -A vultr_lux /tmp/hydragnn-perf.slurm
+```
+
+**Agent result reading order** (minimal tokens):
+
+1. `<jobdir>/processed-data/tracelens-summary.json`
+2. `<jobdir>/processed-data/omnistat-range.yaml`
+3. `<jobdir>/processed-data/report-card.yaml`
+4. Raw traces under `<jobdir>/tools/` only if needed
+
+Legacy HydraGNN perf (`sbatch_train_perf_amd.sh` + `omnihub-inspect` + subagent claims) remains for deep PromQL/inspect workflows. See `material_science/models/HydraGNN/recipes/perf-analysis/README.md`.
+
+## Key paths
+
+Cluster-specific paths come from `.cluster-config.yaml` (`omnihub.shared_dir`, `omnihub.tools_dir`, `omnihub.results_dir`). Resolve them from there rather than assuming literals. The values below are the **Lux / vultr** defaults (`shared_dir: /shared/omnihub`, `tools_dir: /shared/omnihub/tools`).
+
+| Resource | Path (`<tools_dir>` = `omnihub.tools_dir`) |
+|----------|------|
+| OmniHub shared | `omnihub.shared_dir` (Lux: `/shared/omnihub`) |
+| Omnistat | `<tools_dir>/omnistat/` → use `--tools omnistat` |
+| Omnistat + PMC | `<tools_dir>/omnistat-rocprofiler/` → use `--tools omnistat-rocprofiler-pmc1` for hardware counters |
+| Omnistat inspect | `<tools_dir>/omnihub-inspect/` → has `omnistat-inspect` + TraceLens CLI (legacy deep-dive venv) |
+| Results | `omnihub.results_dir` (Lux: `/shared/$USER/results/omnihub/$SLURM_JOB_ID/`) |
+| HydraGNN overlay | `$AI4S_SHARED_DIR/models/HydraGNN/overlays/hydragnn-overlay.img` |
+
+## Runner
+
+HydraGNN uses MPI — default `--runner manual-mpi --tasks-per-node 8` (OmniHub fork). Do **not** use `--runner manual` (that enables torch DDP init).
+
+## Related OmniHub skills
+
+When working inside `$OMNIHUB_DIR`, also use:
+
+- `.cursor/skills/generate-job-and-run/SKILL.md`
+- `.cursor/skills/post-process-results/SKILL.md`
+- `.cursor/skills/check-job-results/SKILL.md`
+
+## Troubleshooting
+
+| Issue | Fix |
+|-------|-----|
+| App config not found | Run `sync-to-omnihub.sh` first |
+| ROCm / overlay missing | Verify `HG_OVERLAY`, `HG_SIF`; check generate-job injected binds |
+| Omnistat empty | Confirm `/shared/omnihub/tools/omnistat/` exists and VictoriaMetrics binary is present |
+| TraceLens skipped | Install venv at `/shared/omnihub/tools/tracelens/venv` or set `TRACELENS_VENV` |

.cursor/skills/ai4science-run-models/SKILL.md

@@ -33,6 +33,7 @@ Match the user's request to a recipe in `model.yaml`:
 - "run inference" / "predict" / "generate" → look for `task: inference`
 - "train" / "fine-tune" / "pair-tune" → look for `task: train` or `task: finetune`
 - "ensemble" → look for `task: ensemble`
+- "omnihub" / "train-omnihub" → look for `task: train-omnihub` and use `.cursor/skills/ai4science-omnihub/SKILL.md`
 
 ## Step 3: Ask the user for required inputs
 

.cursor/skills/ai4science-studio/SKILL.md

@@ -677,18 +677,18 @@ Omnistat exposes **two** independent rocprofiler-sdk integrations, and there's a
 
 **Cross-runtime gotcha:** `enable_kernel_trace=True` without the tool library loaded does *nothing* — the omnistat collector just sits with an empty endpoint. The `OMNISTAT_KERNEL_TRACE=1` switch in `sbatch_train_perf_amd.sh` enforces this with a hard `exit 2` if `libomnistat_trace.so` isn't found at the configured path; do **not** loosen that check, otherwise you re-create the exact "silent zero counters" failure mode from §8.
 
-**Build location:** Both omnistat artifacts (the Python extension and the kernel-trace tool library) must be built on a **compute node** inside the same SIF the workload runs in — login nodes don't have apptainer or the ROCm headers, and a host-side build will pick up the wrong libcurl / glibc. `PERF_TOOLS_DIR` is the shared tools dir from `.cluster-config.yaml` `perf_tools.dir` (e.g. `/path/to/perf-tools`); export it first. Standard build:
+**Build location:** Both omnistat artifacts (the Python extension and the kernel-trace tool library) must be built on a **compute node** inside the same SIF the workload runs in — login nodes don't have apptainer or the ROCm headers, and a host-side build will pick up the wrong libcurl / glibc. `OMNIHUB_TOOLS_DIR` is the shared tools dir from `.cluster-config.yaml` `omnihub.tools_dir` (e.g. `/shared/omnihub/tools`); export it first. Standard build:
 
 ```bash
 salloc -p <partition> -A <account> -N 1 --time=00:15:00 --gpus-per-node=1 \
   apptainer exec --rocm \
     "${AI4S_SHARED_DIR}/images/pytorch_rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0.sif" \
-    bash -c "cd ${PERF_TOOLS_DIR}/omnistat-src && \
+    bash -c "cd ${OMNIHUB_TOOLS_DIR}/omnistat-src && \
       cmake -S rocprofiler-sdk/ -B build-trace/ -DBUILD_KERNEL_TRACE_LIB=ON && \
       cmake --build build-trace/ -j 8"
 ```
 
-Verify with `ls -la ${PERF_TOOLS_DIR}/omnistat-src/build-trace/libomnistat_trace.so` (~MB-class file, not the tiny 4 kB stub you get when CMake fails halfway).
+Verify with `ls -la ${OMNIHUB_TOOLS_DIR}/omnistat-src/build-trace/libomnistat_trace.so` (~MB-class file, not the tiny 4 kB stub you get when CMake fails halfway).
 
 **SIF build prerequisites (one-time gotchas):** the `pytorch_rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0.sif` ships **without** `cmake` and **without** libcurl headers (only the `.so.4` runtime). Both omnistat artifacts need cmake; the kernel-trace library additionally needs `<curl/curl.h>`. Working build recipe inside the SIF:
 
@@ -737,12 +737,12 @@ When the cluster is up but you only need a one-shot interactive build inside the
 srun -p <partition> -A <account> -N 1 --time=00:15:00 --gpus-per-node=1 --cpus-per-task=8 \
   --job-name=<descriptive> \
   apptainer exec --rocm \
-    --bind ${PERF_TOOLS_DIR}:${PERF_TOOLS_DIR} \
+    --bind ${OMNIHUB_TOOLS_DIR}:${OMNIHUB_TOOLS_DIR} \
     "$HG_SIF" \
     bash -c '<your build commands>'
 ```
 
-Why: `salloc` doesn't auto-bind `${PERF_TOOLS_DIR}/` into the apptainer namespace. Direct `srun … apptainer exec --bind …` forces the bind explicitly and writes the build artifact to its persistent NFS path in one shot. Verified during the kernel-trace library build (job 7030).
+Why: `salloc` doesn't auto-bind `${OMNIHUB_TOOLS_DIR}/` into the apptainer namespace. Direct `srun … apptainer exec --bind …` forces the bind explicitly and writes the build artifact to its persistent NFS path in one shot. Verified during the kernel-trace library build (job 7030).
 
 ### 15. Dual-failure debug pattern — separate "tool wired correctly?" from "workload happy?"
 

earth_science/models/ORBIT-2/examples/interm_8m_lux.yaml

@@ -0,0 +1,129 @@
+# ORBIT-2 Lux/MI355X training config — real 10.0_arcmin PRISM grid (same-dir mode).
+#
+# Both low_res_dir and high_res_dir point at the same 10.0_arcmin tree so we can
+# exercise the full training pipeline for perf/scaling without 2.5_arcmin targets.
+# Loss and PSNR are NOT scientifically meaningful in this mode — use for timing only.
+#
+# Placeholders substituted by render_orbit2_config.py at job submit time:
+#   __DATA_ROOT__   — ORBIT2_DATA_ROOT (10.0_arcmin directory)
+#   __FSDP__        — fsdp parallelism (typically = node count)
+#   __SIMPLE_DDP__  — simple_ddp parallelism (typically 8)
+#   __MAX_EPOCHS__  — ORBIT2_MAX_EPOCH override
+#   __BATCH_SIZE__  — ORBIT2_BATCH_SIZE override
+#   __DATA_TYPE__   — ORBIT2_DATA_TYPE (same intent as upstream `configs/interm_8m.yaml` /
+#                     Bayes-CAST `edm_8m_era5.yaml`: **bfloat16**; override float32 only for debug)
+
+#-----------   TRAINER -------------
+trainer:
+  max_epochs: __MAX_EPOCHS__
+  checkpoint:
+  pretrain:
+  batch_size: __BATCH_SIZE__
+  buffer_size: 40
+  num_workers: 0
+  data_type: __DATA_TYPE__
+  gpu_type: "amd"
+  train_loss: "bayesian_tv"
+
+#----------  Parallelism -------------
+parallelism:
+  fsdp: __FSDP__
+  simple_ddp: __SIMPLE_DDP__
+  tensor_par: 1
+  seq_par: 1
+
+### Tiling
+tiling:
+  do_tiling: False
+  div: 4
+  overlap: 3
+
+# ---------------------------- MODEL -------------------------------------------
+model:
+  preset: res_slimvit
+  lr: 2e-3
+  weight_decay: 1e-5
+  beta_1: 0.9
+  beta_2: 0.99
+  warmup_epochs: 1
+  warmup_start_lr: 1e-7
+  eta_min: 1e-8
+  # Same-dir 10.0_arcmin perf mode: no 4× upscale (loss not meaningful; timing only).
+  superres_mag: 1
+  cnn_ratio: 4
+  patch_size: 2
+  embed_dim: 256
+  depth: 6
+  decoder_depth: 4
+  num_heads: 4
+  mlp_ratio: 4
+  drop_path: 0.1
+  drop_rate: 0.1
+
+# ---------------------------- DATA -------------------------------------------
+data:
+  low_res_dir: {
+    'PRISM': "__DATA_ROOT__",
+  }
+  high_res_dir: {
+    'PRISM': "__DATA_ROOT__",
+  }
+
+  spatial_resolution: {
+    'PRISM': 18,
+  }
+
+  default_vars: [
+    "land_sea_mask",
+    "orography",
+    "lattitude",
+    "landcover",
+    "2m_temperature",
+    "2m_temperature_max",
+    "2m_temperature_min",
+    "temperature_200",
+    "temperature_500",
+    "temperature_850",
+    "10m_u_component_of_wind",
+    "u_component_of_wind_200",
+    "u_component_of_wind_500",
+    "u_component_of_wind_850",
+    "10m_v_component_of_wind",
+    "v_component_of_wind_200",
+    "v_component_of_wind_500",
+    "v_component_of_wind_850",
+    "specific_humidity_200",
+    "specific_humidity_500",
+    "specific_humidity_850",
+    "total_precipitation_24hr",
+    "volumetric_soil_water_layer_1",
+  ]
+
+  dict_in_variables: {
+    'PRISM': [
+      "land_sea_mask",
+      "orography",
+      "lattitude",
+      "landcover",
+      "total_precipitation_24hr",
+      "2m_temperature_min",
+      "2m_temperature_max",
+    ],
+  }
+
+  dict_out_variables: {
+    'PRISM': [
+      "total_precipitation_24hr",
+      "2m_temperature_min",
+      "2m_temperature_max",
+    ],
+  }
+
+  var_weights: {
+    "2m_temperature": 10,
+    "10m_u_component_of_wind": 1,
+    "10m_v_component_of_wind": 1,
+    "total_precipitation_24hr": 1,
+    "2m_temperature_min": 10,
+    "2m_temperature_max": 10,
+  }