feat(multi-modal): Phase 0 + Phase 1 — cross-architecture vision capt…

AGENTS.md

@@ -15,17 +15,23 @@ Cargo workspace at repo root with a strict dependency chain — respect this whe
 ```
 # LARQL-specific (depend on vindex, LQL, etc.)
 larql-models      model config, architecture traits, weight loading, quant/dequant,
-                  shared test_fixtures (behind `test-utils` feature)
+                  multi-modal trait surface (ModalEncoder, Connector,
+                  MultiModalProtocol, EmbeddingPlan types), vision tower
+                  config+weights+loader (encoders/vision_tower.rs), projector
+                  weights+loader (connectors/projector.rs), shared
+                  test_fixtures (behind `test-utils` feature)
     ↓
 larql-compute     CPU substrate: BLAS kernels, residual norms, attention spine
                   (rope/gqa/block/decode/gpu), forward-pass primitives (embed,
-                  ops, hooks, ple, layer, predict/raw), kquant_forward Q4_K/Q6_K
-                  decode helpers, FfnBackend trait + dense WeightFfn impl,
-                  KvDispatch + AsyncComputeBackend traits + CpuBackend impls,
-                  KvIndex trait (abstracts VectorIndex for substrate callers),
-                  forward_overrides env-var registry, PerLayerDecodeState.
-                  ADR-0022 moved all of this down from larql-inference; the
-                  substrate is now self-contained.
+                  embed_plan, EmbeddingPlan, ops, hooks, ple, layer, predict/raw),
+                  kquant_forward Q4_K/Q6_K decode helpers, FfnBackend trait +
+                  dense WeightFfn impl, KvDispatch + AsyncComputeBackend traits +
+                  CpuBackend impls, KvIndex trait (abstracts VectorIndex for
+                  substrate callers), forward_overrides env-var registry,
+                  PerLayerDecodeState, vision encoder CPU forward
+                  (encoders/vision_tower.rs), projector CPU forward
+                  (connectors/projector.rs). ADR-0022 moved substrate down from
+                  larql-inference; the substrate is now self-contained.
     ↓
 larql-compute-metal  Metal GPU backend (first-class peer, NOT a thin layer).
                      Implements ComputeBackend / KvDispatch / AsyncComputeBackend
@@ -38,17 +44,21 @@ larql-vindex      vindex lifecycle: extract, load, query, mutate, patch, save,
 larql-core        graph algorithms (merge, diff, BFS, pagerank, shortest-path)
 larql-inference   engines (Standard, MarkovResidual, Apollo, etc.), chat,
                   sessions, tokenizer, FFN routing impls (Graph/Remote/MoE),
-                  layer_executor, layer_graph orchestration. Substrate moves
-                  to larql-compute; this crate is the inference-shaped layer
-                  that composes substrate primitives + engine state. Re-export
-                  shims preserve `crate::{residual, forward, attention,
-                  kv_dispatch, async_compute_backend, kquant_forward,
-                  forward_overrides}::*` paths for back-compat.
+                  layer_executor, layer_graph orchestration. KvEngine trait
+                  (with supports_multimodal + prefill_from_hidden per
+                  ADR-0023), AnyEngine dispatch enum (KvEngine | RetrievalEngine).
+                  Substrate moves to larql-compute; this crate is the
+                  inference-shaped layer that composes substrate primitives +
+                  engine state.
     ↓
 larql-lql         lexer/parser/executor/REPL + USE REMOTE client
     ↓
 larql-server      HTTP + gRPC server serving vindexes
-larql-cli         top-level `larql` binary (every subcommand lives in commands/)
+larql-cli         top-level `larql` binary (every subcommand lives in commands/).
+                  Multi-modal: `--image` + `--mm-weights` flags on `larql run`,
+                  image decode/resize (image_input.rs), plan assembly
+                  (run_cmd_image.rs). 3-image regression test in
+                  tests/multimodal_e2e.rs (#[ignore], NOT FOR CI).
 larql-python      PyO3 bindings (maturin-built, module name `larql._native`)
 
 # Portable (no LARQL deps; extract to sibling repo later, name stable)

README.md

@@ -42,6 +42,11 @@ larql list
 larql run gemma-3-4b-it-vindex "The capital of France is"
 larql run gemma-3-4b-it-vindex          # drops into chat mode
 
+# Multi-modal — describe an image (Gemma 3 + SigLIP, prefix-only)
+larql run gemma3-4b-v2 --image photo.jpg \
+    --mm-weights ~/.cache/huggingface/hub/models--google--gemma-3-4b-it/snapshots/<hash> \
+    "Describe this image in one sentence."
+
 # Or extract locally — inference-ready at f16 by default
 larql extract google/gemma-3-4b-it -o gemma3-4b.vindex
 larql run gemma3-4b.vindex "Einstein is known for"

ROADMAP.md

@@ -35,6 +35,8 @@ Every invention in the codebase serves this aim:
 | Shannon arc (1 bit/char on Frankenstein) | Theoretical compression ceiling — how far this can go |
 | Mech-interp surface (M1–M8) | Discover *which* weights actually do the work; rest stays on disk |
 | Cross-arch coverage | The technique stack must generalise |
+| Multi-modal (vision / audio) | Accept images + audio alongside text; same sparse-retrieval story applies to the LM portion of multimodal models |
+| KV engine trait split (KvEngine / RetrievalEngine / AnyEngine) | Uniform dispatch across production KV-cache engines + retrieval-only engines (Apollo) via typed enum |
 
 Combined effect (rough math, conservative): hash routing 5× × FP4 2× × KV
 compression 10× = **100× effective bandwidth reduction** on the right

crates/larql-cli/Cargo.toml

@@ -33,6 +33,10 @@ thiserror = { workspace = true }
 minijinja = "2"
 libc = "0.2"
 rayon = "1.10"
+# Multi-modal Phase 1d: image decode + bicubic resize for the `--image` flag.
+# Pruned features — we only need JPEG/PNG/WebP. Skipping TIFF/AVIF/HDR/etc
+# avoids dragging in libavif, libtiff, openexr, and a 2 MB binary tax.
+image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp"] }
 
 [features]
 default = ["gpu"]
@@ -45,3 +49,7 @@ gpu = [
 
 [dev-dependencies]
 tempfile = "3"
+# `test-utils` exposes `larql_models::test_fixtures::make_test_weights`,
+# used by run_cmd_image tests to borrow a real ModelConfig without
+# enumerating every field by hand. Production builds don't compile it.
+larql-models = { path = "../larql-models", features = ["test-utils"] }

crates/larql-cli/ROADMAP.md

@@ -9,7 +9,12 @@ Primary verbs: `run`, `chat`, `pull`, `model`, `link`, `list`, `show`, `slice`,
 `compile`, `convert`, `verify`, `hf`, plus diagnostic verbs `diag` and `parity`.
 Legacy research commands gated under `larql dev <subcmd>` for backwards-compat.
 Dual cache (HuggingFace hub + `~/.cache/larql/local/`) with shorthand resolution
-(`larql run gemma3-4b-it-vindex`).
+(`larql run gemma3-4b-it-vindex`). Multi-modal: `--image` + `--mm-weights`
+flags on `larql run` for prefix-only vision captioning (Phase 1, Gemma 3 +
+SigLIP). Image decode/resize in `image_input.rs`, plan assembly in
+`run_cmd_image.rs`. Engine capability check (`supports_multimodal()`) fires
+before the encoder runs. Q4K vindex dispatch supported. 3-image regression
+test in `tests/multimodal_e2e.rs` (`#[ignore]`, NOT FOR CI).
 
 The `shannon` family was extended in 2026-05-16 with **`larql shannon
 verify`** — a cross-engine bits/char correctness check that orchestrates

crates/larql-cli/src/commands/primary/accuracy_cmd.rs

@@ -563,7 +563,10 @@ mod tests {
         assert_eq!(score_mark(ScoreOutcome::Served, Some(false)), "✗");
         assert_eq!(score_mark(ScoreOutcome::Served, None), "✗");
         assert_eq!(score_mark(ScoreOutcome::SkippedEmptyPrompt, None), "·");
-        assert_eq!(score_mark(ScoreOutcome::SkippedInternalError, None), "·");
+        // ScoreOutcome::SkippedInternalError → SkippedBackendFailure post
+        // kv-engine-retrieval-trait-split refactor; "internal error" generalised
+        // into the typed BackendFailure / InvariantViolation split.
+        assert_eq!(score_mark(ScoreOutcome::SkippedBackendFailure, None), "·");
     }
 
     #[test]

crates/larql-cli/src/commands/primary/mod.rs

@@ -15,6 +15,7 @@ pub mod publish_cmd;
 pub mod pull_cmd;
 pub mod rm_cmd;
 pub mod run_cmd;
+pub mod run_cmd_image;
 pub mod shannon_cmd;
 pub mod show_cmd;
 pub mod slice_cmd;

crates/larql-cli/src/commands/primary/run_cmd.rs

@@ -253,6 +253,34 @@ pub struct RunArgs {
     /// re-routing, at the cost of one extra remote round-trip per token.
     #[arg(long, default_value = "1", value_name = "N")]
     pub moe_predispatch_iters: usize,
+
+    /// Path to one or more image files to splice into the prompt prefix
+    /// (multi-modal Phase 1d, prefix-only). Repeat the flag to pass
+    /// multiple images:
+    ///
+    ///   larql run gemma-3-4b-it --image cat.jpg --image stop_sign.jpg "describe both"
+    ///
+    /// Currently supported on Gemma 3 multimodal checkpoints only.
+    /// Requires `--engine standard` (other engines lack
+    /// `prefill_from_hidden`; the CLI will fail fast with a clear
+    /// message if `--image` is combined with an MM-incapable engine —
+    /// see ADR-0023). Also requires `--mm-weights` to point at the
+    /// directory containing the SigLIP vision_tower and
+    /// multi_modal_projector safetensors shards (typically the HF
+    /// snapshot dir, e.g.
+    /// `~/.cache/huggingface/hub/models--google--gemma-3-4b-it/snapshots/<hash>`).
+    #[arg(long, value_name = "PATH", num_args = 0..)]
+    pub image: Vec<PathBuf>,
+
+    /// Directory containing the SigLIP and multi_modal_projector
+    /// safetensors shards. Required when `--image` is set. The vindex
+    /// itself only carries LM weights (FFN + attention + embeddings);
+    /// the vision tower and projector live in the original HF snapshot
+    /// alongside `config.json`. Both Phase 1b and Phase 1c loaders
+    /// scan this directory for `*.safetensors` files filtered by tensor
+    /// key prefix.
+    #[arg(long, value_name = "DIR")]
+    pub mm_weights: Option<PathBuf>,
 }
 
 pub fn run(args: RunArgs) -> Result<(), Box<dyn std::error::Error>> {
@@ -309,6 +337,30 @@ pub fn run(args: RunArgs) -> Result<(), Box<dyn std::error::Error>> {
         );
     }
 
+    if !args.image.is_empty() {
+        // Multi-modal Phase 1d, prefix-only. Mutually exclusive with
+        // the experts / ffn / moe-shards paths above (they're
+        // tokenizer-deep and don't compose with hidden-state prefill).
+        if args.ffn.is_some() {
+            return Err("--image is incompatible with --ffn (Phase 1d scope)".into());
+        }
+        if args.moe_shards.is_some() || args.moe_units_manifest.is_some() {
+            return Err(
+                "--image is incompatible with --moe-shards / --moe-units-manifest \
+                 (Phase 1d scope)"
+                    .into(),
+            );
+        }
+        if args.experts {
+            return Err("--image is incompatible with --experts (Phase 1d scope)".into());
+        }
+        let prompt = args
+            .prompt
+            .as_deref()
+            .ok_or("--image requires a prompt argument (chat mode with images is Phase 2+ work)")?;
+        return super::run_cmd_image::run_with_images(&vindex_path, prompt, &args);
+    }
+
     if let Some(prompt) = args.prompt.as_deref() {
         run_once(&vindex_path, prompt, &args)
     } else {