[Bug]: Extra LLM parameters are ignored (temperature, n, top_p) caused vLLM to crash!

[thusinh1969]

Describe the bug

GraphRAG parsing parameters missed completely the controlled parameters for LLM, such as temperature, n, top_p. Although these are in the settings.yaml file (described below) but using the verbose mode ưe found that these parameters werre somehow missed parsing.

Steps to reproduce

The verbose mode showing that 3 parameters those should be included (temperature, n, top_p) are completely mising:

🚀 Reading settings from ragtest/settings.yaml
Using default configuration: {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"root_dir": "./ragtest",
"reporting": {
"type": "file",
"base_dir": "output/${timestamp}/reports",
"storage_account_blob_url": null
},
"storage": {
"type": "file",
"base_dir": "output/${timestamp}/artifacts",
"storage_account_blob_url": null
},
"cache": {
"type": "file",
"base_dir": "cache",
"storage_account_blob_url": null
},
"input": {
"type": "file",
"file_type": "text",
"base_dir": "input",
"storage_account_blob_url": null,
"encoding": "utf-8",
"file_pattern": ".*\.txt$",
"file_filter": null,
"source_column": null,
"timestamp_column": null,
"timestamp_format": null,
"text_column": "text",
"title_column": null,
"document_attribute_columns": []
},
"embed_graph": {
"enabled": false,
"num_walks": 10,
"walk_length": 40,
"window_size": 2,
"iterations": 3,
"random_seed": 597832,
"strategy": null
},
"embeddings": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_embedding",
"model": "intfloat/multilingual-e5-large",
"max_tokens": 4000,
"request_timeout": 180.0,
"api_base": "http://localhost:8000/v1",
"api_version": "gemma2-9b-it",
"organization": "REDACTED, length 4",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": null,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 10,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"batch_size": 16,
"batch_max_tokens": 8191,
"target": "required",
"skip": [],
"vector_store": null,
"strategy": null
},
"chunks": {
"size": 512,
"overlap": 100,
"group_by_columns": [
"id"
],
"strategy": null
},
"snapshots": {
"graphml": false,
"raw_entities": false,
"top_level_nodes": false
},
"entity_extraction": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"prompt": "prompts/entity_extraction.txt",
"entity_types": [
"person",
"partnership",
"owner",
"friend",
"role",
"technology",
"equipment",
"organization",
"tax code",
"event",
"location",
"date",
"factory",
"farm",
"tower",
"resort",
"hotel",
"real estate",
"concept",
"decision",
"article",
"creditor",
"debtor",
"stock owner",
"bond owner",
"fund raiser",
"issuance",
"guarantor",
"investigator",
"convicted",
"arrested"
],
"max_gleanings": 0,
"strategy": null
},
"summarize_descriptions": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"prompt": "prompts/summarize_descriptions.txt",
"max_length": 500,
"strategy": null
},
"community_reports": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"prompt": null,
"max_length": 2000,
"max_input_length": 8000,
"strategy": null
},
"claim_extraction": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"enabled": false,
"prompt": "prompts/claim_extraction.txt",
"description": "c\u00e1c kh\u1eb3ng \u0111\u1ecbnh c\u00f3 minh ch\u1ee9ng li\u00ean quan \u0111\u1ebfn c\u00e1c th\u1ef1c th\u1ec3 (entities),
t\u1ed5 ch\u1ee9c (orgnizations) hay c\u00e1c quan h\u1ec7 (relationships).",
"max_gleanings": 0,
"strategy": null
},
"cluster_graph": {
"max_cluster_size": 10,
"strategy": null
},
"umap": {
"enabled": false
},
"local_search": {
"text_unit_prop": 0.5,
"community_prop": 0.1,
"conversation_history_max_turns": 5,
"top_k_entities": 10,
"top_k_relationships": 10,
"max_tokens": 12000,
"llm_max_tokens": 2000
},
"global_search": {
"max_tokens": 12000,
"data_max_tokens": 12000,
"map_max_tokens": 1000,
"reduce_max_tokens": 2000,
"concurrency": 32
},
"encoding_model": "cl100k_base",
"skip_workflows": []
}

Expected Behavior

GraphRAG should use correctly all parameters that is described in the docs here:
https://microsoft.github.io/graphrag/posts/config/json_yaml/

GraphRAG Config Used

encoding_model: cl100k_base
skip_workflows: []
llm:
api_key: ABC
type: openai_chat # or azure_openai_chat
model: Gemma2_9b_it
model_supports_json: true # recommended if this is available for your model.
max_tokens: 1500
request_timeout: 180.0
api_base: http://localhost:8900/v1
api_version: gemma2-9b-it
organization: EraX
deployment_name: gemma2
max_retries: 1
concurrent_requests: 1 # the number of parallel inflight requests that may be made
temperature: 1.0
top_p: 0.95
n: 1
max_retry_wait: 10.0
sleep_on_rate_limit_recommendation: true # whether to sleep when azure suggests wait-times
tokens_per_minute: 150_000 # set a leaky bucket throttle
requests_per_minute: 10_000 # set a leaky bucket throttle

parallelization:
stagger: 0.3
num_threads: 1 # the number of threads to use for parallel processing

async_mode: threaded # or asyncio

embeddings:

parallelization: override the global parallelization settings for embeddings

async_mode: threaded # or asyncio
llm:
api_key: ABC
type: openai_embedding # or azure_openai_embedding
model: intfloat/multilingual-e5-large
api_base: http://localhost:8000/v1
api_version: gemma2-9b-it
organization: EraX
deployment_name: gemma2
temperature: 1.0
top_p: 0.95
n: 1
# tokens_per_minute: 150_000 # set a leaky bucket throttle
# requests_per_minute: 10_000 # set a leaky bucket throttle
max_retries: 10
# max_retry_wait: 10.0
# sleep_on_rate_limit_recommendation: true # whether to sleep when azure suggests wait-times
concurrent_requests: 1 # the number of parallel inflight requests that may be made
# batch_size: 16 # the number of documents to send in a single request
# batch_max_tokens: 8191 # the maximum number of tokens to send in a single request
# target: required # or optional

chunks:
size: 512
overlap: 100
group_by_columns: [id] # by default, we don't allow chunks to cross documents

input:
type: file # or blob
file_type: text # or csv
base_dir: "input"
file_encoding: utf-8
file_pattern: ".*\.txt$"

cache:
type: file # or blob
base_dir: "cache"

connection_string: <azure_blob_storage_connection_string>

container_name: <azure_blob_storage_container_name>

storage:
type: file # or blob
base_dir: "output/${timestamp}/artifacts"

connection_string: <azure_blob_storage_connection_string>

container_name: <azure_blob_storage_container_name>

reporting:
type: file # or console, blob
base_dir: "output/${timestamp}/reports"

connection_string: <azure_blob_storage_connection_string>

container_name: <azure_blob_storage_container_name>

entity_extraction:

llm: override the global llm settings for this task

parallelization: override the global parallelization settings for this task

async_mode: override the global async_mode settings for this task

llm:
n: 1
temperature: 1.0
frequency_penalty: 1.0
top_p: 0.95
prompt: "prompts/entity_extraction.txt"
entity_types: [person, partnership, owner, friend, role, technology, equipment, organization, tax code, event, location, date, factory, farm, tower, resort, hotel, real estate, concept, decision, article, creditor, debtor, stock owner, bond owner, fund raiser, issuance, guarantor, investigator, convicted, arrested]
max_gleanings: 0

summarize_descriptions:

llm: override the global llm settings for this task

parallelization: override the global parallelization settings for this task

async_mode: override the global async_mode settings for this task

llm:
n: 1
temperature: 1.0
frequency_penalty: 1.0
top_p: 0.95
prompt: "prompts/summarize_descriptions.txt"
max_length: 500

claim_extraction:

llm: override the global llm settings for this task

llm:
n: 1
temperature: 1.0
frequency_penalty: 1.0
top_p: 0.95

parallelization: override the global parallelization settings for this task

async_mode: override the global async_mode settings for this task

enabled: true

prompt: "prompts/claim_extraction.txt"
description: "các khẳng định có minh chứng liên quan đến các thực thể (entities), tổ chức (orgnizations) hay các quan hệ (relationships)."
max_gleanings: 0

community_report:

llm: override the global llm settings for this task

parallelization: override the global parallelization settings for this task

async_mode: override the global async_mode settings for this task

llm:
n: 1
temperature: 1.0
frequency_penalty: 1.0
top_p: 0.95
prompt: "prompts/community_report.txt"
max_length: 2000
max_input_length: 8000

cluster_graph:
max_cluster_size: 10

embed_graph:
enabled: false # if true, will generate node2vec embeddings for nodes

num_walks: 10

walk_length: 40

window_size: 2

iterations: 3

random_seed: 597832

umap:
enabled: false # if true, will generate UMAP embeddings for nodes

snapshots:
graphml: false
raw_entities: false
top_level_nodes: false

local_search:

text_unit_prop: 0.5

community_prop: 0.1

conversation_history_max_turns: 5

top_k_mapped_entities: 10

top_k_relationships: 10

max_tokens: 12000

global_search:

max_tokens: 12000

data_max_tokens: 12000

map_max_tokens: 1000

reduce_max_tokens: 2000

concurrency: 32

Logs and screenshots

🚀 Reading settings from ragtest/settings.yaml
Using default configuration: {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"root_dir": "./ragtest",
"reporting": {
"type": "file",
"base_dir": "output/${timestamp}/reports",
"storage_account_blob_url": null
},
"storage": {
"type": "file",
"base_dir": "output/${timestamp}/artifacts",
"storage_account_blob_url": null
},
"cache": {
"type": "file",
"base_dir": "cache",
"storage_account_blob_url": null
},
"input": {
"type": "file",
"file_type": "text",
"base_dir": "input",
"storage_account_blob_url": null,
"encoding": "utf-8",
"file_pattern": ".*\.txt$",
"file_filter": null,
"source_column": null,
"timestamp_column": null,
"timestamp_format": null,
"text_column": "text",
"title_column": null,
"document_attribute_columns": []
},
"embed_graph": {
"enabled": false,
"num_walks": 10,
"walk_length": 40,
"window_size": 2,
"iterations": 3,
"random_seed": 597832,
"strategy": null
},
"embeddings": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_embedding",
"model": "intfloat/multilingual-e5-large",
"max_tokens": 4000,
"request_timeout": 180.0,
"api_base": "http://localhost:8000/v1",
"api_version": "gemma2-9b-it",
"organization": "REDACTED, length 4",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": null,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 10,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"batch_size": 16,
"batch_max_tokens": 8191,
"target": "required",
"skip": [],
"vector_store": null,
"strategy": null
},
"chunks": {
"size": 512,
"overlap": 100,
"group_by_columns": [
"id"
],
"strategy": null
},
"snapshots": {
"graphml": false,
"raw_entities": false,
"top_level_nodes": false
},
"entity_extraction": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"prompt": "prompts/entity_extraction.txt",
"entity_types": [
"person",
"partnership",
"owner",
"friend",
"role",
"technology",
"equipment",
"organization",
"tax code",
"event",
"location",
"date",
"factory",
"farm",
"tower",
"resort",
"hotel",
"real estate",
"concept",
"decision",
"article",
"creditor",
"debtor",
"stock owner",
"bond owner",
"fund raiser",
"issuance",
"guarantor",
"investigator",
"convicted",
"arrested"
],
"max_gleanings": 0,
"strategy": null
},
"summarize_descriptions": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"prompt": "prompts/summarize_descriptions.txt",
"max_length": 500,
"strategy": null
},
"community_reports": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"prompt": null,
"max_length": 2000,
"max_input_length": 8000,
"strategy": null
},
"claim_extraction": {
"llm": {
"api_key": "REDACTED, length 3",
"type": "openai_chat",
"model": "Gemma2_9b_it",
"max_tokens": 1500,
"request_timeout": 180.0,
"api_base": "http://localhost:8900/v1",
"api_version": "gemma2-9b-it",
"proxy": null,
"cognitive_services_endpoint": null,
"deployment_name": "gemma2",
"model_supports_json": true,
"tokens_per_minute": 0,
"requests_per_minute": 0,
"max_retries": 1,
"max_retry_wait": 10.0,
"sleep_on_rate_limit_recommendation": true,
"concurrent_requests": 1
},
"parallelization": {
"stagger": 0.3,
"num_threads": 1
},
"async_mode": "threaded",
"enabled": false,
"prompt": "prompts/claim_extraction.txt",
"description": "c\u00e1c kh\u1eb3ng \u0111\u1ecbnh c\u00f3 minh ch\u1ee9ng li\u00ean quan \u0111\u1ebfn c\u00e1c th\u1ef1c th\u1ec3 (entities),
t\u1ed5 ch\u1ee9c (orgnizations) hay c\u00e1c quan h\u1ec7 (relationships).",
"max_gleanings": 0,
"strategy": null
},
"cluster_graph": {
"max_cluster_size": 10,
"strategy": null
},
"umap": {
"enabled": false
},
"local_search": {
"text_unit_prop": 0.5,
"community_prop": 0.1,
"conversation_history_max_turns": 5,
"top_k_entities": 10,
"top_k_relationships": 10,
"max_tokens": 12000,
"llm_max_tokens": 2000
},
"global_search": {
"max_tokens": 12000,
"data_max_tokens": 12000,
"map_max_tokens": 1000,
"reduce_max_tokens": 2000,
"concurrency": 32
},
"encoding_model": "cl100k_base",
"skip_workflows": []
}

Additional Information

All latest version installed today 14 July 2024.

Thanks,
Steve

[zanderjiang]

These parameters were not included in the configuration initialization. It will be fixed in the next release. For the mean time, try running from source code using poetry if you need to adjust the parameters. https://microsoft.github.io/graphrag/posts/developing

[natoverse]

Consolidating alternate model issues here: #657