models.yaml

# DGX Spark Model Configuration
# ================================
# Single source of truth for all model configurations.
# This file replaces models.json and consolidates settings from serve.sh scripts.
#
# Usage:
#   - model-manager reads this file to start/stop models
#   - web-gui uses model metadata for display
#   - serve.sh scripts can be auto-generated from this config
#
# Schema version for future migrations
schema_version: "1.0"

# Default settings applied to all models (can be overridden per-model)
defaults:
  vllm:
    image: "nvcr.io/nvidia/vllm:25.11-py3"
    max_num_seqs: 8
    gpu_memory_utilization: 0.4
    dtype: "auto"
    swap_space: 16
    restart_policy: "unless-stopped"
    # Docker settings
    gpus: "all"
    ipc: "host"
    ulimit_memlock: -1
    ulimit_stack: 67108864
  ollama:
    image: "ollama/ollama"
    restart_policy: "unless-stopped"

# Model definitions
models:
  # ===========================================================================
  # vLLM Models (managed directly by model-manager)
  # ===========================================================================

  qwen3-coder-30b-awq:
    name: "Qwen3-Coder-30B-AWQ"
    description: "AWQ 4-bit quantized Qwen3 Coder - best balance of speed and quality"
    engine: vllm
    port: 8104
    container_name: "vllm-qwen3-coder-30b-awq"
    model_id: "cpatonn/Qwen3-Coder-30B-A3B-Instruct-AWQ-4bit"

    # Resource estimates (for GPU memory checking)
    estimated_memory_gb: 45

    # vLLM settings (override defaults)
    settings:
      max_model_len: 65536          # 64K context
      max_num_seqs: 8
      gpu_memory_utilization: 0.4
      swap_space: 16
      # Performance features
      enable_prefix_caching: true
      enable_chunked_prefill: true
      # Tool calling
      enable_auto_tool_choice: true
      tool_call_parser: "qwen3_coder"

  qwen3-coder-30b:
    name: "Qwen3-Coder-30B"
    description: "Full precision Qwen3 Coder model"
    engine: vllm
    port: 8100
    container_name: "vllm-qwen3-coder-30b"
    model_id: "Qwen/Qwen3-Coder-30B-A3B-Instruct"
    estimated_memory_gb: 65
    settings:
      max_model_len: 32768
      max_num_seqs: 8
      gpu_memory_utilization: 0.5
      # Tool calling
      enable_auto_tool_choice: true
      tool_call_parser: "qwen3_coder"

  qwen2-vl-7b:
    name: "Qwen2-VL-7B"
    description: "Vision-language model for image understanding"
    engine: vllm
    port: 8101
    container_name: "vllm-qwen2-vl-7b"
    model_id: "Qwen/Qwen2-VL-7B-Instruct"
    estimated_memory_gb: 20
    settings:
      max_model_len: 32768
      max_num_seqs: 8
      gpu_memory_utilization: 0.3
      # Tool calling (hermes format for Qwen2-VL)
      enable_auto_tool_choice: true
      tool_call_parser: "hermes"

  ministral3-14b:
    name: "Ministral-3-14B"
    description: "Mistral's efficient 14B parameter model"
    engine: vllm
    port: 8103
    container_name: "vllm-ministral3-14b"
    model_id: "mistralai/Ministral-3-14B-Instruct-2512"
    estimated_memory_gb: 30
    settings:
      max_model_len: 32768
      max_num_seqs: 8
      gpu_memory_utilization: 0.3
      # Mistral-specific
      tokenizer_mode: "mistral"
      config_format: "mistral"
      # Tool calling
      enable_auto_tool_choice: true
      tool_call_parser: "mistral"

  chandra-ocr:
    name: "Chandra OCR"
    description: "Document OCR with tables, handwriting, math equations, and full layout preservation (9B)"
    engine: vllm
    port: 8106
    container_name: "vllm-chandra-ocr"
    model_id: "datalab-to/chandra"
    estimated_memory_gb: 24
    # NOTE: Requires custom vLLM/Triton build for GB10 (SM 12.1) - see vllm-chandra-ocr/GB10_COMPATIBILITY_NOTES.md
    # Run manually: source ~/vllm-install/.vllm/bin/activate && python -m vllm.entrypoints.openai.api_server ...
    settings:
      max_model_len: 16384
      max_num_seqs: 8
      gpu_memory_utilization: 0.3
      # Performance features
      enable_prefix_caching: true
      enable_chunked_prefill: true
      enforce_eager: true
      # Chandra is a vision model - no tool calling

  # ===========================================================================
  # Ollama Models
  # ===========================================================================

  qwen3-vl-32b-ollama:
    name: "Qwen3-VL-32B (Ollama)"
    description: "Advanced vision model via Ollama"
    engine: ollama
    port: 11435
    container_name: "ollama-qwen3-vl-32b"
    model_id: "qwen3-vl:32b"
    estimated_memory_gb: 70
    settings: {}

  # ===========================================================================
  # Script-based Models (complex startup, managed by custom scripts)
  # ===========================================================================

  qwen3-235b-awq:
    name: "Qwen3-235B-A22B-AWQ (Distributed)"
    description: "235B parameter model distributed across 2 DGX Sparks via Ray"
    engine: script
    port: 8235
    script_dir: "vllm-qwen3-235b-awq"
    model_id: "QuantTrio/Qwen3-235B-A22B-Instruct-2507-AWQ"
    estimated_memory_gb: 116
    # Script-based models store config for reference but execute via serve.sh
    settings:
      tensor_parallel_size: 2
      max_model_len: 8192
      gpu_memory_utilization: 0.75
      swap_space: 16
      enforce_eager: true
      trust_remote_code: true
      enable_auto_tool_choice: true
      tool_call_parser: "qwen3_xml"

  nemotron-3-nano-30b-bf16:
    name: "Nemotron-3-Nano-30B-BF16"
    description: "NVIDIA Nemotron in BF16 precision"
    engine: script
    port: 8105
    script_dir: "vllm-nemotron-3-nano-30b-bf16"
    model_id: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B"
    estimated_memory_gb: 65
    settings:
      max_model_len: 32768
      # Tool calling
      enable_auto_tool_choice: true
      tool_call_parser: "qwen3_coder"