LLM fallback

.env.example

@@ -1,13 +1,28 @@
 # Example environment configuration
 
 ### LLM configuration
+# Output LLM interaction to a file.
+LLM_DEBUG_OUTPUT=false
+# Primary LLM
+#LLM_TIMEOUT_SECONDS=10
+#LLM_MODEL=ollama/gemma3
+#LLM_KEY=no-key-needed
+#LLM_BASE_URL=http://localhost:11434
 LLM_MODEL=openai/gpt-4o
 LLM_KEY=sk-proj-...
 # LLM_MODEL=anthropic/claude-3-5-sonnet-20240620
 # LLM_KEY=${ANTHROPIC_API_KEY}
 # LLM_MODEL=gemini/gemini-2.5-flash-preview-04-17
 # LLM_KEY=${GOOGLE_API_KEY}
 
+# Fallback LLM
+#LLM_FALLBACK_TIMEOUT_SECONDS=10
+#LLM_FALLBACK_MODEL=ollama/gemma3
+#LLM_FALLBACK_KEY=no-key-needed
+#LLM_FALLBACK_BASE_URL=http://localhost:11434
+LLM_FALLBACK_MODEL=openai/gpt-4o
+LLM_FALLBACK_KEY=sk-proj-...
+
 ### Tool API keys
 # RAPIDAPI_KEY=9df2cb5...                         # Optional - if unset flight search generates realistic mock data
 # RAPIDAPI_HOST_FLIGHTS=sky-scrapper.p.rapidapi.com # For real travel flight information (optional)

.gitignore

@@ -1,6 +1,9 @@
 # OS-specific files
 .DS_Store
 
+# Debug files
+debug_llm_calls
+
 # Python cache & compiled files
 __pycache__/
 *.py[cod]
@@ -30,9 +33,14 @@ coverage.xml
 
 # PyCharm / IntelliJ settings
 .idea/
+*.iml
 
 .env
 .env*
 
 # Cursor
-.cursor
+.cursor
+
+# Claude Code
+CLAUDE.md
+.claude

AGENTS.md

@@ -1,5 +1,20 @@
 # Temporal AI Agent Contribution Guide
 
+## Table of Contents
+- [Repository Layout](#repository-layout)
+- [Running the Application](#running-the-application)
+  - [Quick Start with Docker](#quick-start-with-docker)
+  - [Local Development Setup](#local-development-setup)
+  - [Environment Configuration](#environment-configuration)
+- [Testing](#testing)
+- [Linting and Code Quality](#linting-and-code-quality)
+- [Agent Customization](#agent-customization)
+  - [Adding New Goals and Tools](#adding-new-goals-and-tools)
+  - [Configuring Goals](#configuring-goals)
+- [Architecture](#architecture)
+- [Commit Messages and Pull Requests](#commit-messages-and-pull-requests)
+- [Additional Resources](#additional-resources)
+
 ## Repository Layout
 - `workflows/` - Temporal workflows including the main AgentGoalWorkflow for multi-turn AI conversations
 - `activities/` - Temporal activities for tool execution and LLM interactions  

Makefile

@@ -8,6 +8,9 @@ setup:
 run-worker:
 	uv run scripts/run_worker.py
 
+run-worker-debug:
+	LOGLEVEL=DEBUG PYTHONUNBUFFERED=1 uv run scripts/run_worker.py
+
 run-api:
 	uv run uvicorn api.main:app --reload
 
@@ -41,6 +44,7 @@ help:
 	@echo "Available commands:"
 	@echo "  make setup              - Install all dependencies"
 	@echo "  make run-worker         - Start the Temporal worker"
+	@echo "  make run-worker-debug   - Start the Temporal worker with DEBUG logging"
 	@echo "  make run-api            - Start the API server"
 	@echo "  make run-frontend       - Start the frontend development server"
 	@echo "  make run-train-api      - Start the train API server"

README.md

@@ -1,5 +1,21 @@
 # Temporal AI Agent
 
+## Table of Contents
+- [Overview](#overview)
+- [Demo Videos](#demo-videos)
+- [Why Temporal?](#why-temporal)
+- [What is "Agentic AI"?](#what-is-agentic-ai)
+- [MCP Tool Calling Support](#-mcp-tool-calling-support)
+- [Setup and Configuration](#setup-and-configuration)
+- [Customizing Interaction & Tools](#customizing-interaction--tools)
+- [Architecture](#architecture)
+- [Testing](#testing)
+- [Development](#development)
+- [Productionalization & Adding Features](#productionalization--adding-features)
+- [Enablement Guide](#enablement-guide-internal-resource-for-temporal-employees)
+
+## Overview
+
 This demo shows a multi-turn conversation with an AI agent running inside a Temporal workflow. The purpose of the agent is to collect information towards a goal, running tools along the way. The agent supports both native tools and Model Context Protocol (MCP) tools, allowing it to interact with external services.
 
 The agent operates in single-agent mode by default, focusing on one specific goal. It also supports experimental multi-agent/multi-goal mode where users can choose between different agent types and switch between them during conversations.
@@ -14,6 +30,8 @@ The AI will respond with clarifications and ask for any missing information to t
 - Ollama models (local)
 - And many more!
 
+## Demo Videos
+
 It's really helpful to [watch the demo (5 minute YouTube video)](https://www.youtube.com/watch?v=GEXllEH2XiQ) to understand how interaction works.
 
 [![Watch the demo](./assets/agent-youtube-screenshot.jpeg)](https://www.youtube.com/watch?v=GEXllEH2XiQ)

activities/tool_activities.py

@@ -19,6 +19,7 @@
     ValidationResult,
 )
 from models.tool_definitions import MCPServerDefinition
+from shared.llm_manager import LLMManager
 from shared.mcp_client_manager import MCPClientManager
 
 # Import MCP client libraries
@@ -36,20 +37,24 @@
 
 class ToolActivities:
     def __init__(self, mcp_client_manager: MCPClientManager = None):
-        """Initialize LLM client using LiteLLM and optional MCP client manager"""
+        """Initialize LLM client using LLMManager with fallback support and optional MCP client manager"""
+        # Use LLMManager for automatic fallback support
+        self.llm_manager = LLMManager()
+
+        # Keep legacy attributes for backward compatibility
         self.llm_model = os.environ.get("LLM_MODEL", "openai/gpt-4")
         self.llm_key = os.environ.get("LLM_KEY")
         self.llm_base_url = os.environ.get("LLM_BASE_URL")
+
         self.mcp_client_manager = mcp_client_manager
-        print(f"Initializing ToolActivities with LLM model: {self.llm_model}")
-        if self.llm_base_url:
-            print(f"Using custom base URL: {self.llm_base_url}")
+        print(f"Initializing ToolActivities with LLMManager")
         if self.mcp_client_manager:
             print("MCP client manager enabled for connection pooling")
 
+
     @activity.defn
-    async def agent_validatePrompt(
-        self, validation_input: ValidationInput
+    async def agent_validate_prompt(
+        self, validation_input: ValidationInput, fallback_mode: bool
     ) -> ValidationResult:
         """
         Validates the prompt in the context of the conversation history and agent goal.
@@ -101,15 +106,16 @@ async def agent_validatePrompt(
             prompt=validation_prompt, context_instructions=context_instructions
         )
 
-        result = await self.agent_toolPlanner(prompt_input)
+        result = await self.agent_tool_planner(prompt_input, fallback_mode)
 
         return ValidationResult(
             validationResult=result.get("validationResult", False),
             validationFailedReason=result.get("validationFailedReason", {}),
         )
 
+
     @activity.defn
-    async def agent_toolPlanner(self, input: ToolPromptInput) -> dict:
+    async def agent_tool_planner(self, input: ToolPromptInput, fallback_mode: bool) -> dict:
         messages = [
             {
                 "role": "system",
@@ -124,17 +130,7 @@ async def agent_toolPlanner(self, input: ToolPromptInput) -> dict:
         ]
 
         try:
-            completion_kwargs = {
-                "model": self.llm_model,
-                "messages": messages,
-                "api_key": self.llm_key,
-            }
-
-            # Add base_url if configured
-            if self.llm_base_url:
-                completion_kwargs["base_url"] = self.llm_base_url
-
-            response = completion(**completion_kwargs)
+            response = await self.llm_manager.call_llm(messages, fallback_mode)
 
             response_content = response.choices[0].message.content
             activity.logger.info(f"Raw LLM response: {repr(response_content)}")
@@ -205,6 +201,39 @@ async def get_wf_env_vars(self, input: EnvLookupInput) -> EnvLookupOutput:
 
         return output
 
+    def warm_up_ollama(self) -> bool:
+        """
+        Pre-load the Ollama model to avoid cold start latency.
+        Returns True if successful, False otherwise.
+        """
+        import time
+
+        try:
+            start_time = time.time()
+            print("Sending warm-up request to Ollama...")
+
+            # Make a simple completion request to load the model
+            response = completion(
+                model=self.llm_model,
+                messages=[{"role": "user", "content": "Hello"}],
+                api_key=self.llm_key,
+                base_url=self.llm_base_url,
+            )
+
+            end_time = time.time()
+            duration = end_time - start_time
+
+            if response and response.choices:
+                print(f"✅ Model loaded successfully in {duration:.1f} seconds")
+                return True
+            else:
+                print("❌ Model loading failed: No response received")
+                return False
+
+        except Exception as e:
+            print(f"❌ Model loading failed: {str(e)}")
+            return False
+
     @activity.defn
     async def mcp_tool_activity(
         self, tool_name: str, tool_args: Dict[str, Any]

dev-tools/README.md

@@ -0,0 +1,3 @@
+# Developer Tools
+
+This directory contains tools useful during development.

dev-tools/allow-anthropic.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+HOST="${HOST:-api.anthropic.com}"
+ANCHOR_NAME="anthropic"
+ANCHOR_FILE="/etc/pf.anchors/${ANCHOR_NAME}"
+PF_CONF="/etc/pf.conf"
+
+require_root() {
+  if [ "${EUID:-$(id -u)}" -ne 0 ]; then
+    echo "Please run with sudo." >&2
+    exit 1
+  fi
+}
+
+backup_once() {
+  local file="$1"
+  if [ -f "$file" ] && [ ! -f "${file}.bak" ]; then
+    cp -p "$file" "${file}.bak"
+  fi
+}
+
+ensure_anchors_dir() {
+  if [ ! -d "/etc/pf.anchors" ]; then
+    mkdir -p /etc/pf.anchors
+    chmod 755 /etc/pf.anchors
+  fi
+}
+
+ensure_anchor_hook() {
+  if ! grep -qE '^\s*anchor\s+"'"${ANCHOR_NAME}"'"' "$PF_CONF"; then
+    echo "Wiring anchor into ${PF_CONF}..."
+    backup_once "$PF_CONF"
+    {
+      echo ''
+      echo '# --- Begin anthropic anchor hook ---'
+      echo 'anchor "'"${ANCHOR_NAME}"'"'
+      echo 'load anchor "'"${ANCHOR_NAME}"'" from "/etc/pf.anchors/'"${ANCHOR_NAME}"'"'
+      echo '# --- End anthropic anchor hook ---'
+    } >> "$PF_CONF"
+  fi
+}
+
+default_iface() {
+  route -n get default 2>/dev/null | awk '/interface:/{print $2; exit}'
+}
+
+resolve_ips() {
+  (dig +short A "$HOST"; dig +short AAAA "$HOST") 2>/dev/null \
+    | awk 'NF' | sort -u
+}
+
+write_anchor_allow() {
+  local iface="$1"; shift
+  local ips=("$@")
+
+  local table_entries=""
+  if [ "${#ips[@]}" -gt 0 ]; then
+    for ip in "${ips[@]}"; do
+      if [ -n "$ip" ]; then
+        if [ -z "$table_entries" ]; then
+          table_entries="$ip"
+        else
+          table_entries="$table_entries, $ip"
+        fi
+      fi
+    done
+  fi
+
+  backup_once "$ANCHOR_FILE"
+  {
+    echo "# ${ANCHOR_FILE}"
+    echo "# Auto-generated: $(date)"
+    echo "# Host: ${HOST}"
+    echo "table <anthropic> persist { ${table_entries} }"
+    echo ""
+    echo "# Allow outbound traffic to Anthropic"
+    echo "pass out quick on ${iface} to <anthropic>"
+  } > "$ANCHOR_FILE"
+}
+
+enable_pf() {
+  pfctl -E >/dev/null 2>&1 || true
+}
+
+reload_pf() {
+  if ! pfctl -nf "$PF_CONF" >/dev/null 2>&1; then
+    echo "pf.conf validation failed. Aborting." >&2
+    exit 1
+  fi
+  pfctl -f "$PF_CONF" >/dev/null
+}
+
+main() {
+  require_root
+  ensure_anchors_dir
+
+  local iface
+  iface="$(default_iface || true)"
+  if [ -z "${iface:-}" ]; then
+    echo "Could not determine default network interface." >&2
+    exit 1
+  fi
+
+  ensure_anchor_hook
+
+  ips=()
+  while IFS= read -r ip; do
+    ips+=("$ip")
+  done < <(resolve_ips)
+
+  if [ "${#ips[@]}" -eq 0 ]; then
+    echo "Warning: No IPs resolved for ${HOST}. The table will be empty." >&2
+  fi
+
+  write_anchor_allow "$iface" "${ips[@]}"
+  enable_pf
+  reload_pf
+
+  echo "✅ Anthropic API is now ALLOWED via pf on interface ${iface}."
+  echo "Anchor file: ${ANCHOR_FILE}"
+}
+
+main "$@"