LlamaIndex: Evaluate Ollama LLM backend with smaller models

topic/machine-learning/llama-index/boot.py

@@ -8,52 +8,61 @@
 from llama_index.core.base.embeddings.base import BaseEmbedding
 from llama_index.core.llms import LLM
 from llama_index.llms.azure_openai import AzureOpenAI
+from llama_index.llms.ollama import Ollama
 from llama_index.llms.openai import OpenAI
 from llama_index.embeddings.langchain import LangchainEmbedding
 
 
-MODEL_NAME = "gpt-4.1"
-
-
 def configure_llm(debug: bool = False) -> Tuple[LLM, BaseEmbedding]:
     """
     Configure LLM. Use either vanilla Open AI, or Azure Open AI.
     """
 
-    openai.api_type = os.getenv("OPENAI_API_TYPE")
-    openai.azure_endpoint = os.getenv("OPENAI_AZURE_ENDPOINT")
-    openai.api_version = os.getenv("OPENAI_AZURE_API_VERSION")
-    openai.api_key = os.getenv("OPENAI_API_KEY")
+    llm_backend = os.getenv("LLM_BACKEND")
+    llm_model = os.getenv("LLM_MODEL")
+
+    if not llm_backend:
+        raise ValueError("LLM_BACKEND environment variable is required")
+    if not llm_model:
+        raise ValueError("LLM_MODEL environment variable is required")
 
     # https://docs.llamaindex.ai/en/stable/understanding/tracing_and_debugging/tracing_and_debugging/
     if debug:
         llama_index.core.set_global_handler("simple")
 
-    if openai.api_type == "openai":
+    if llm_backend == "openai":
         llm = OpenAI(
-            model=MODEL_NAME,
+            model=llm_model,
             temperature=0.0,
             api_key=os.getenv("OPENAI_API_KEY"),
         )
-    elif openai.api_type == "azure":
+    elif llm_backend == "azure":
         llm = AzureOpenAI(
-            model=MODEL_NAME,
+            model=llm_model,
             temperature=0.0,
             engine=os.getenv("LLM_INSTANCE"),
             azure_endpoint=os.getenv("OPENAI_AZURE_ENDPOINT"),
             api_key = os.getenv("OPENAI_API_KEY"),
             api_version = os.getenv("OPENAI_AZURE_API_VERSION"),
         )
+    elif llm_backend == "ollama":
+        # https://docs.llamaindex.ai/en/stable/api_reference/llms/ollama/
+        llm = Ollama(
+            model=llm_model,
+            temperature=0.0,
+            request_timeout=120.0,
+            keep_alive=-1,
+        )
     else:
-        raise ValueError(f"Open AI API type not defined or invalid: {openai.api_type}")
+        raise ValueError(f"LLM backend not defined or invalid: {llm_backend}")
 
-    if openai.api_type == "openai":
-        embed_model = LangchainEmbedding(OpenAIEmbeddings(model=MODEL_NAME))
-    elif openai.api_type == "azure":
+    if llm_backend == "openai":
+        embed_model = LangchainEmbedding(OpenAIEmbeddings(model="text-embedding-3-large"))
+    elif llm_backend == "azure":
         embed_model = LangchainEmbedding(
             AzureOpenAIEmbeddings(
                 azure_endpoint=os.getenv("OPENAI_AZURE_ENDPOINT"),
-                model=os.getenv("EMBEDDING_MODEL_INSTANCE")
+                model="text-embedding-3-large",
             )
         )
     else:

topic/machine-learning/llama-index/demo_nlsql.py

@@ -31,7 +31,7 @@ def main():
     sql_database = SQLDatabase(engine_crate, include_tables=[os.getenv("CRATEDB_TABLE_NAME")])
     query_engine = NLSQLTableQueryEngine(
         sql_database=sql_database,
-        tables=[os.getenv("CRATEDB_TABLE_NAME")],
+        # tables=[os.getenv("CRATEDB_TABLE_NAME")],
         llm=llm,
         embed_model=embed_model,
     )

topic/machine-learning/llama-index/env.azure

@@ -1,3 +1,6 @@
+export LLM_BACKEND=azure
+export LLM_MODEL=gpt-4.1
+
 export OPENAI_API_KEY=TODO
 export OPENAI_API_TYPE=azure
 export OPENAI_AZURE_ENDPOINT=https://TODO.openai.azure.com

topic/machine-learning/llama-index/env.standalone

@@ -1,4 +1,16 @@
-# OPENAI_API_KEY=sk-XJZ7pfog5Gp8Kus8D--invalid--0CJ5lyAKSefZLaV1Y9S1
-export OPENAI_API_TYPE=openai
+# Backend: Ollama
+# export LLM_BACKEND=ollama
+# export LLM_MODEL=phi4-mini:latest
+# export LLM_MODEL=phi4-mini:3.8b-fp16
+# export LLM_MODEL=deepseek-r1:7b
+# export LLM_MODEL=llama3.2:latest
+# export LLM_MODEL=qwen3:0.6b
+# export LLM_MODEL=gemma3:1b
+
+# Backend: OpenAI
+export LLM_BACKEND=openai
+export LLM_MODEL=gpt-4.1
+# export OPENAI_API_KEY=sk-XJZ7pfog5Gp8Kus8D--invalid--0CJ5lyAKSefZLaV1Y9S1
+
 export CRATEDB_SQLALCHEMY_URL="crate://crate@localhost:4200/"
 export CRATEDB_TABLE_NAME=time_series_data

topic/machine-learning/llama-index/requirements.txt

@@ -3,6 +3,7 @@ langchain-openai<0.4
 llama-index-embeddings-langchain<0.4
 llama-index-embeddings-openai<0.4
 llama-index-llms-azure-openai<0.4
+llama-index-llms-ollama<0.7
 llama-index-llms-openai<0.5
 llama-index-tools-mcp<0.3
 python-dotenv

topic/machine-learning/llama-index/test.py

@@ -24,7 +24,7 @@ def init_database(cratedb):
 
 def test_nlsql(cratedb, capsys):
     """
-    Execute `demo_nlsql.py` and verify outcome.
+    Execute `demo_nlsql.py` and verify the outcome.
     """
 
     # Load the standalone configuration also for software testing.
@@ -42,7 +42,7 @@ def test_nlsql(cratedb, capsys):
 
 def test_mcp(cratedb, capsys):
     """
-    Execute `demo_mcp.py` and verify outcome.
+    Execute `demo_mcp.py` and verify the outcome.
     """
 
     # Load the standalone configuration also for software testing.