diff --git a/docs/modules/llm.md b/docs/modules/llm.md index 72947b28..0a4c0c49 100644 --- a/docs/modules/llm.md +++ b/docs/modules/llm.md @@ -94,6 +94,35 @@ response = llm.generate( ) ``` +### NovitaLLM + +NovitaLLM is an adapter for models hosted on the [Novita AI platform](https://novita.ai/), which offers access to open-source and proprietary LLMs via an OpenAI-compatible API. + +Thanks to Novita's OpenAI-compatible interface, the `NovitaLLM` model class in EvoAgentX allows seamless switching between models hosted on Novita using the same API format. + +**Basic Usage:** + +```python +from evoagentx.models import NovitaConfig, NovitaLLM + +# Configure the model +config = NovitaConfig( + model="deepseek/deepseek-v4-pro", + novita_key="your-novita-api-key", + temperature=0.7, + max_tokens=1000 +) + +# Initialize the model +llm = NovitaLLM(config=config) + +# Generate text +response = llm.generate( + prompt="Write a poem about artificial intelligence.", + system_message="You are a creative poet." +) +``` + ### OpenRouterLLM OpenRouterLLM is an adapter for the [OpenRouter platform](https://openrouter.ai/), which provides access to a wide range of language models from various providers through a unified API. It supports models from providers like Anthropic, Google, Meta, Mistral AI, and more, all accessible through a single interface. diff --git a/docs/zh/modules/llm.md b/docs/zh/modules/llm.md index d7e94b85..be1adb2f 100644 --- a/docs/zh/modules/llm.md +++ b/docs/zh/modules/llm.md @@ -93,6 +93,35 @@ response = llm.generate( ) ``` +### NovitaLLM + +NovitaLLM 是 [Novita AI 平台](https://novita.ai/) 上托管模型的适配器，该平台通过 OpenAI 兼容的 API 提供对开源和专有模型的访问。 + +得益于 Novita 兼容 OpenAI 的接口，EvoAgentX 中的 `NovitaLLM` 模型类允许使用相同的 API 格式在 Novita 上托管的不同模型之间无缝切换。 + +**基本用法：** + +```python +from evoagentx.models import NovitaConfig, NovitaLLM + +# Configure the model +config = NovitaConfig( + model="deepseek/deepseek-v4-pro", + novita_key="your-novita-api-key", + temperature=0.7, + max_tokens=1000 +) + +# Initialize the model +llm = NovitaLLM(config=config) + +# Generate text +response = llm.generate( + prompt="Write a poem about artificial intelligence.", + system_message="You are a creative poet." +) +``` + ### OpenRouterLLM OpenRouterLLM 是 [OpenRouter 平台](https://openrouter.ai/) 的适配器，该平台通过统一的 API 提供对各种提供商的语言模型的访问。它支持来自 Anthropic、Google、Meta、Mistral AI 等提供商的模型，所有这些都可以通过单一接口访问。 diff --git a/evoagentx/models/__init__.py b/evoagentx/models/__init__.py index 232d9cb8..1e37a69f 100644 --- a/evoagentx/models/__init__.py +++ b/evoagentx/models/__init__.py @@ -7,3 +7,4 @@ from .siliconflow_model import * from .openrouter_model import * from .aliyun_model import * +from .novita_model import * diff --git a/evoagentx/models/model_configs.py b/evoagentx/models/model_configs.py index 60b2ca4b..ed8a545c 100644 --- a/evoagentx/models/model_configs.py +++ b/evoagentx/models/model_configs.py @@ -140,6 +140,38 @@ def __str__(self): return self.model +class NovitaConfig(LLMConfig): + + # LLM keys + llm_type: str = "NovitaLLM" + novita_key: Optional[str] = Field(default=None, description="the API key used to authenticate Novita AI requests") + + # generation parameters + temperature: Optional[float] = Field(default=None, description="the temperature used to scaling logits") + max_tokens : Optional[int] = Field(default=None, description="maximum number of generated tokens") + max_completion_tokens: Optional[int] = Field(default=None, description="An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. Commonly used in OpenAI's o1 series models.") + top_p: Optional[float] = Field(default=None, description="Only sample from tokens with cumulative probability greater than top_p when generating text.") + n: Optional[int] = Field(default=None, description="How many chat completion choices to generate for each input message.") + stream: Optional[bool] = Field(default=None, description=" If set to true, it sends partial message deltas. Tokens will be sent as they become available, with the stream terminated by a [DONE] message.") + stream_options: Optional[dict] = Field(default=None, description="Options for streaming response. Only set this when you set stream: true") + timeout: Optional[Union[float, int]] = Field(default=None, description="Timeout in seconds for completion requests (Defaults to 600 seconds)") + + # tools + tools: Optional[List] = Field(default=None, description="A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for.") + tool_choice: Optional[Union[str, dict]] = Field(default=None, description="Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. required forces the model to call a tool. Specifying a particular function via {\"type\": \"function\", \"function\": {\"name\": \"my_function\"}} forces the model to call that function.") + parallel_tool_calls: Optional[bool] = Field(default=None, description="Whether to enable parallel function calling during tool use. OpenAI default is true.") + + # token probabilities + logprobs: Optional[bool] = Field(default=None, description="Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message.") + top_logprobs: Optional[int] = Field(default=None, description="An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used.") + + # output format + response_format: Optional[Union[BaseModel, dict]] = Field(default=None, description=" An object specifying the format that the model must output.") + + def __str__(self): + return self.model + + # def get_default_device(): # return "cuda" if torch.cuda.is_available() else "cpu" diff --git a/evoagentx/models/novita_model.py b/evoagentx/models/novita_model.py new file mode 100644 index 00000000..aecb0208 --- /dev/null +++ b/evoagentx/models/novita_model.py @@ -0,0 +1,61 @@ +from typing import Union + +from openai import AsyncOpenAI, OpenAI +from openai.types.chat import ChatCompletion, ChatCompletionChunk + +from .openai_model import OpenAILLM +from .model_configs import NovitaConfig +from ..core.logging import logger +from ..core.registry import register_model +from .model_utils import Cost, cost_manager + +# Novita AI exposes an OpenAI-compatible API at this base URL. +NOVITA_BASE_URL = "https://api.novita.ai/openai" + + +@register_model(config_cls=NovitaConfig, alias=["novita"]) +class NovitaLLM(OpenAILLM): + """Novita AI LLM client. + + Novita speaks the OpenAI chat-completions protocol, so this reuses all of + ``OpenAILLM``'s generation/streaming/tool-call logic and only overrides client + construction and cost handling. + + LiteLLM has no pricing data for Novita-hosted models, so the dollar cost + cannot be computed or approximated. Token counts are still tracked; the + per-model cost is recorded as 0. A warning is emitted once at init time. + """ + + def init_model(self): + self._client = None + self._async_client = None + # parameters in NovitaConfig that are not Novita models' input parameters + self._default_ignore_fields = ["llm_type", "novita_key", "output_response"] + logger.warning( + "[NovitaLLM] LiteLLM has no pricing data for Novita models, so dollar " + "cost cannot be computed. Token usage will be tracked, but cost will " + "be recorded as 0." + ) + + def _init_client(self, config: NovitaConfig): + return OpenAI(api_key=config.novita_key, base_url=NOVITA_BASE_URL) + + def _init_async_client(self, config: NovitaConfig): + return AsyncOpenAI(api_key=config.novita_key, base_url=NOVITA_BASE_URL) + + def _update_cost(self, response: Union[ChatCompletion, ChatCompletionChunk]): + # Override OpenAILLM's LiteLLM-based cost computation: only record token + # counts and leave cost at 0 (see class docstring). + usage = getattr(response, "usage", None) + if usage is None: + logger.warning( + f"[NovitaLLM] usage is None in response (id={getattr(response, 'id', '?')}); " + "tokens will not be recorded." + ) + return + cost = Cost( + input_tokens=usage.prompt_tokens, + output_tokens=usage.completion_tokens, + cost=0.0, + ) + cost_manager.update_cost(cost=cost, model=self.config.model)