diff --git a/semantic-kernel/Frameworks/agent/TOC.yml b/semantic-kernel/Frameworks/agent/TOC.yml
index c85787ea..2d6709d8 100644
--- a/semantic-kernel/Frameworks/agent/TOC.yml
+++ b/semantic-kernel/Frameworks/agent/TOC.yml
@@ -6,6 +6,8 @@
href: chat-completion-agent.md
- name: OpenAI Assistant Agent
href: assistant-agent.md
+- name: Azure AI Agent
+ href: azure-ai-agent.md
- name: Agent Collaboration
href: agent-chat.md
- name: Create an Agent from a Template
diff --git a/semantic-kernel/Frameworks/agent/agent-templates.md b/semantic-kernel/Frameworks/agent/agent-templates.md
index 6813d6f9..b8d0a4ca 100644
--- a/semantic-kernel/Frameworks/agent/agent-templates.md
+++ b/semantic-kernel/Frameworks/agent/agent-templates.md
@@ -76,10 +76,8 @@ ChatCompletionAgent agent =
::: zone pivot="programming-language-python"
```python
-kernel = Kernel()
-
agent = ChatCompletionAgent(
- kernel=kernel,
+ service=AzureChatCompletion(), # or other supported AI Services
name="StoryTeller",
instructions="Tell a story about {{$topic}} that is {{$length}} sentences long.",
arguments=KernelArguments(topic="Dog", length="2"),
@@ -100,24 +98,34 @@ Templated instructions are especially powerful when working with an [`OpenAIAssi
::: zone pivot="programming-language-csharp"
```csharp
// Retrieve an existing assistant definition by identifier
-OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.RetrieveAsync(
- this.GetClientProvider(),
- "<stored agent-identifier>",
- new Kernel(),
- new KernelArguments()
- {
- { "topic", "Dog" },
- { "length", "3" },
- });
+AzureOpenAIClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(new AzureCliCredential(), new Uri("<your endpoint>"));
+AssistantClient assistantClient = client.GetAssistantClient();
+Assistant assistant = await client.GetAssistantAsync();
+OpenAIAssistantAgent agent = new(assistant, assistantClient, new KernelPromptTemplateFactory(), PromptTemplateConfig.SemanticKernelTemplateFormat)
+{
+ Arguments = new KernelArguments()
+ {
+ { "topic", "Dog" },
+ { "length", "3" },
+ }
+}
```
::: zone-end
::: zone pivot="programming-language-python"
```python
-agent = await OpenAIAssistantAgent.retrieve(
- id=<assistant_id>,
- kernel=Kernel(),
+# Create the client using Azure OpenAI resources and configuration
+client, model = AzureAssistantAgent.setup_resources()
+
+# Retrieve the assistant definition from the server based on the assistant ID
+definition = await client.beta.assistants.retrieve(
+ assistant_id="your-assistant-id",
+)
+
+# Create the AzureAssistantAgent instance using the client and the assistant definition
+agent = AzureAssistantAgent(
+ client=client,
+ definition=definition,
arguments=KernelArguments(topic="Dog", length="3"),
)
```
@@ -130,10 +138,9 @@ agent = await OpenAIAssistantAgent.retrieve(
::: zone-end
-## Agent Definition from a _Prompt Template_
-
-The same _Prompt Template Config_ used to create a _Kernel Prompt Function_ can also be leveraged to define an agent. This allows for a unified approach in managing both prompts and agents, promoting consistency and reuse across different components. By externalizing agent definitions from the codebase, this method simplifies the management of multiple agents, making them easier to update and maintain without requiring changes to the underlying logic. This separation also enhances flexibility, enabling developers to modify agent behavior or introduce new agents by simply updating the configuration, rather than adjusting the code itself.
+## Agent Definition from a Prompt Template
+The same Prompt Template Config used to create a Kernel Prompt Function can also be leveraged to define an agent. This allows for a unified approach in managing both prompts and agents, promoting consistency and reuse across different components. By externalizing agent definitions from the codebase, this method simplifies the management of multiple agents, making them easier to update and maintain without requiring changes to the underlying logic. This separation also enhances flexibility, enabling developers to modify agent behavior or introduce new agents by simply updating the configuration, rather than adjusting the code itself.
#### YAML Template
```yaml
@@ -192,7 +199,7 @@ data = yaml.safe_load(generate_story_yaml)
prompt_template_config = PromptTemplateConfig(**data)
agent = ChatCompletionAgent(
- kernel=_create_kernel_with_chat_completion(),
+ service=AzureChatCompletion(), # or other supported AI services
prompt_template_config=prompt_template_config,
arguments=KernelArguments(topic="Dog", length="3"),
)
@@ -249,10 +256,8 @@ await foreach (ChatMessageContent response in agent.InvokeAsync(chat, overrideAr
::: zone pivot="programming-language-python"
```python
-kernel = Kernel()
-
agent = ChatCompletionAgent(
- kernel=kernel,
+ service=AzureChatCompletion(),
name="StoryTeller",
instructions="Tell a story about {{$topic}} that is {{$length}} sentences long.",
arguments=KernelArguments(topic="Dog", length="2"),
diff --git a/semantic-kernel/Frameworks/agent/assistant-agent.md b/semantic-kernel/Frameworks/agent/assistant-agent.md
index 17ee3199..aa888610 100644
--- a/semantic-kernel/Frameworks/agent/assistant-agent.md
+++ b/semantic-kernel/Frameworks/agent/assistant-agent.md
@@ -8,7 +8,7 @@ ms.author: crickman
ms.date: 09/13/2024
ms.service: semantic-kernel
---
-# Exploring the _Semantic Kernel_ `OpenAIAssistantAgent`
+# Exploring the Semantic Kernel `OpenAIAssistantAgent`
> [!IMPORTANT]
> This feature is in the release candidate stage. Features at this stage are nearly complete and generally stable, though they may undergo minor refinements or optimizations before reaching full general availability.
@@ -17,8 +17,6 @@ Detailed API documentation related to this discussion is available at:
::: zone pivot="programming-language-csharp"
- [`OpenAIAssistantAgent`](/dotnet/api/microsoft.semantickernel.agents.openai.openaiassistantagent)
-- [`OpenAIAssistantDefinition`](/dotnet/api/microsoft.semantickernel.agents.openai.openaiassistantdefinition)
-- [`OpenAIClientProvider`](/dotnet/api/microsoft.semantickernel.agents.openai.openaiclientprovider)
::: zone-end
@@ -45,21 +43,55 @@ The _OpenAI Assistant API_ is a specialized interface designed for more advanced
- [Assistant API in Azure](/azure/ai-services/openai/assistants-quickstart)
+## Preparing Your Development Environment
+
+To proceed with developing an `OpenAIAIAssistantAgent`, configure your development environment with the appropriate packages.
+
+::: zone pivot="programming-language-csharp"
+
+Add the `Microsoft.SemanticKernel.Agents.OpenAI` package to your project:
+
+```pwsh
+dotnet add package Microsoft.SemanticKernel.Agents.AzureAI --prerelease
+```
+
+You may also want to include the `Azure.Identity` package:
+
+```pwsh
+dotnet add package Azure.Identity
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+Install the `semantic-kernel` package with the optional _Azure_ dependencies:
+
+```bash
+pip install semantic-kernel[azure]
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+
## Creating an `OpenAIAssistantAgent`
Creating an `OpenAIAssistant` requires invoking a remote service, which is handled asynchronously. To manage this, the `OpenAIAssistantAgent` is instantiated through a static factory method, ensuring the process occurs in a non-blocking manner. This method abstracts the complexity of the asynchronous call, returning a promise or future once the assistant is fully initialized and ready for use.
::: zone pivot="programming-language-csharp"
```csharp
-OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.CreateAsync(
- OpenAIClientProvider.ForAzureOpenAI(/*<...service configuration>*/),
- new OpenAIAssistantDefinition("<model name>")
- {
- Name = "<agent name>",
- Instructions = "<agent instructions>",
- },
- new Kernel());
+AssistantClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(...).GetAssistantClient();
+Assistant assistant =
+ await this.AssistantClient.CreateAssistantAsync(
+ "<model name>",
+ "<agent name>",
+ instructions: "<agent instructions>");
+OpenAIAssistantAgent agent = new(assistant, client);
```
::: zone-end
@@ -119,11 +151,9 @@ Once created, the identifier of the assistant may be access via its identifier.
For .NET, the agent identifier is exposed as a `string` via the property defined by any agent.
```csharp
-OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.RetrieveAsync(
- OpenAIClientProvider.ForAzureOpenAI(/*<...service configuration>*/),
- "<your agent id>",
- new Kernel());
+AssistantClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(...).GetAssistantClient();
+Assistant assistant = await this.AssistantClient.GetAssistantAsync("<assistant id>");
+OpenAIAssistantAgent agent = new(assistant, client);
```
::: zone-end
@@ -220,20 +250,18 @@ await agent.delete_thread(thread_id)
## Deleting an `OpenAIAssistantAgent`
-Since the assistant's definition is stored remotely, it supports the capability to self-delete. This enables the agent to be removed from the system when it is no longer needed.
+Since the assistant's definition is stored remotely, it will persist if not deleted.
+Deleting an assistant definition may be performed directly with the `AssistantClient`.
-> Note: Attempting to use an agent instance after being deleted results in an exception.
+> Note: Attempting to use an agent instance after being deleted will result in a service exception.
::: zone pivot="programming-language-csharp"
For .NET, the agent identifier is exposed as a `string` via the [`Agent.Id`](/dotnet/api/microsoft.semantickernel.agents.agent.id) property defined by any agent.
```csharp
-// Perform the deletion
-await agent.DeleteAsync();
-
-// Inspect whether an agent has been deleted
-bool isDeleted = agent.IsDeleted();
+AssistantClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(...).GetAssistantClient();
+Assistant assistant = await this.AssistantClient.DeleteAssistantAsync("<assistant id>");
```
::: zone-end
@@ -261,5 +289,5 @@ For an end-to-end example for a `OpenAIAssistantAgent`, see:
> [!div class="nextstepaction"]
-> [Agent Collaboration in `AgentChat`](./agent-chat.md)
+> [Exploring the Azure AI Agent](./azure-ai-agent.md)
diff --git a/semantic-kernel/Frameworks/agent/azure-ai-agent.md b/semantic-kernel/Frameworks/agent/azure-ai-agent.md
new file mode 100644
index 00000000..8e9ebf3f
--- /dev/null
+++ b/semantic-kernel/Frameworks/agent/azure-ai-agent.md
@@ -0,0 +1,671 @@
+---
+title: Exploring the Semantic Kernel Azure AI Agent Agent
+description: An exploration of the definition, behaviors, and usage patterns for an Azure AI Agent
+zone_pivot_groups: programming-languages
+author: moonbox3
+ms.topic: tutorial
+ms.author: evmattso
+ms.date: 03/05/2025
+ms.service: semantic-kernel
+---
+# Exploring the Semantic Kernel `AzureAIAgent`
+
+> [!IMPORTANT]
+> This feature is in the experimental stage. Features at this stage are still under development and subject to change before advancing to the preview or release candidate stage.
+
+Detailed API documentation related to this discussion is available at:
+
+::: zone pivot="programming-language-csharp"
+
+- [`OpenAIAssistantAgent`](/dotnet/api/microsoft.semantickernel.agents.azureai)
+
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+> Updated Semantic Kernel Python API Docs are coming soon.
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+## What is an `AzureAIAgent`?
+
+An `AzureAIAgent` is a specialized agent within the Semantic Kernel framework, designed to provide advanced conversational capabilities with seamless tool integration. It automates tool calling, eliminating the need for manual parsing and invocation. The agent also securely manages conversation history using threads, reducing the overhead of maintaining state. Additionally, the `AzureAIAgent` supports a variety of built-in tools, including file retrieval, code execution, and data interaction via Bing, Azure AI Search, Azure Functions, and OpenAPI.
+
+To use an `AzureAIAgent`, an Azure AI Foundry Project must be utilized. The following articles provide an overview of the Azure AI Foundry, how to create and configure a project, and the agent service:
+
+- [What is Azure AI Foundry?](/azure/ai-foundry/what-is-ai-foundry)
+- [The Azure AI Foundry SDK](/azure/ai-foundry/how-to/develop/sdk-overview)
+- [What is Azure AI Agent Service](/azure/ai-services/agents/overview)
+- [Quickstart: Create a new agent](/azure/ai-services/agents/quickstart)
+
+
+## Preparing Your Development Environment
+
+To proceed with developing an `AzureAIAgent`, configure your development environment with the appropriate packages.
+
+::: zone pivot="programming-language-csharp"
+
+Add the `Microsoft.SemanticKernel.Agents.AzureAI` package to your project:
+
+```pwsh
+dotnet add package Microsoft.SemanticKernel.Agents.AzureAI --prerelease
+```
+
+You may also want to include the `Azure.Identity` package:
+
+```pwsh
+dotnet add package Azure.Identity
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+Install the `semantic-kernel` package with the optional Azure dependencies:
+
+```bash
+pip install semantic-kernel[azure]
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+
+## Configuring the AI Project Client
+
+Accessing an `AzureAIAgent` first requires the creation of a project client that is configured for a specific Foundry Project, most commonly by providing a connection string ([The Azure AI Foundry SDK: Getting Started with Projects](/azure/ai-foundry/how-to/develop/sdk-overview#get-started-with-projects)).
+
+::: zone pivot="programming-language-csharp"
+
+```c#
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+```
+
+The `AgentsClient` may be accessed from the `AIProjectClient`:
+
+```c#
+AgentsClient agentsClient = client.GetAgentsClient();
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+Modify your the `.env` file in the root directory to include:
+
+```bash
+AZURE_AI_AGENT_PROJECT_CONNECTION_STRING = "<example-connection-string>"
+AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME = "<example-model-deployment-name>"
+```
+
+or
+
+```bash
+AZURE_AI_AGENT_ENDPOINT = "<example-endpoint>"
+AZURE_AI_AGENT_SUBSCRIPTION_ID = "<example-subscription-id>"
+AZURE_AI_AGENT_RESOURCE_GROUP_NAME = "<example-resource-group-name>"
+AZURE_AI_AGENT_PROJECT_NAME = "<example-project-name>"
+AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME = "<example-model-deployment-name>"
+```
+
+Once the configuration is defined, the client may be created:
+
+```python
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+):
+ # Your operational code here
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+## Creating an `AzureAIAgent`
+
+To create an `AzureAIAgent`, you start by configuring and initializing the agent project through the Azure AI service and then integrate it with Semantic Kernel:
+
+::: zone pivot="programming-language-csharp"
+
+```c#
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+AgentsClient agentsClient = client.GetAgentsClient();
+
+// 1. Define an agent on the Azure AI agent service
+Agent definition = agentsClient.CreateAgentAsync(
+ "<name of the the model used by the agent>",
+ name: "<agent name>",
+ description: "<agent description>",
+ instructions: "<agent instructions>");
+
+// 2. Create a Semantic Kernel agent based on the agent definition
+AzureAIAgent agent = new(definition, agentsClient);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+```python
+from azure.identity.aio import DefaultAzureCredential
+from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings
+
+ai_agent_settings = AzureAIAgentSettings.create()
+
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+):
+ # 1. Define an agent on the Azure AI agent service
+ agent_definition = await client.agents.create_agent(
+ model=ai_agent_settings.model_deployment_name,
+ name="<name>",
+ instructions="<instructions>",
+ )
+
+ # 2. Create a Semantic Kernel agent based on the agent definition
+ agent = AzureAIAgent(
+ client=client,
+ definition=agent_definition,
+ )
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+## Interacting with an `AzureAIAgent`
+
+Interaction with the `AzureAIAgent` is straightforward. The agent maintains the conversation history automatically using a thread:
+
+::: zone pivot="programming-language-csharp"
+```c#
+AgentThread thread = await agentsClient.CreateThreadAsync();
+try
+{
+ ChatMessageContent message = new(AuthorRole.User, "<your user input>");
+ await agent.AddChatMessageAsync(threadId, message);
+ await foreach (ChatMessageContent response in agent.InvokeAsync(thread.Id))
+ {
+ Console.WriteLine(response.Content);
+ }
+}
+finally
+{
+ await this.AgentsClient.DeleteThreadAsync(thread.Id);
+ await this.AgentsClient.DeleteAgentAsync(agent.Id);
+}
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+USER_INPUTS = ["Hello", "What's your name?"]
+
+thread = await client.agents.create_thread()
+
+try:
+ for user_input in USER_INPUTS:
+ await agent.add_chat_message(thread_id=thread.id, message=user_input)
+ response = await agent.get_response(thread_id=thread.id)
+ print(response)
+finally:
+ await client.agents.delete_thread(thread.id)
+```
+
+Optionally, an agent may be invoked as:
+
+```python
+for user_input in USER_INPUTS:
+ await agent.add_chat_message(thread_id=thread.id, message=user_input)
+ async for content in agent.invoke(thread_id=thread.id):
+ print(content.content)
+```
+
+::: zone-end
+
+An agent may also produce a streamed response:
+
+::: zone pivot="programming-language-csharp"
+```c#
+ChatMessageContent message = new(AuthorRole.User, "<your user input>");
+await agent.AddChatMessageAsync(threadId, message);
+await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(thread.Id))
+{
+ Console.Write(response.Content);
+}
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+for user_input in USER_INPUTS:
+ await agent.add_chat_message(thread_id=thread.id, message=user_input)
+ async for content in agent.invoke_stream(thread_id=thread.id):
+ print(content.content, end="", flush=True)
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+## Using Plugins with an `AzureAIAgent`
+
+Semantic Kernel supports extending an `AzureAIAgent` with custom plugins for enhanced functionality:
+
+::: zone pivot="programming-language-csharp"
+```c#
+Plugin plugin = KernelPluginFactory.CreateFromType<YourPlugin>();
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+AgentsClient agentsClient = client.GetAgentsClient();
+
+Agent definition = agentsClient.CreateAgentAsync(
+ "<name of the the model used by the agent>",
+ name: "<agent name>",
+ description: "<agent description>",
+ instructions: "<agent instructions>");
+
+AzureAIAgent agent = new(definition, agentsClient, plugins: [plugin]);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+from semantic_kernel.functions import kernel_function
+
+class SamplePlugin:
+ @kernel_function(description="Provides sample data.")
+ def get_data(self) -> str:
+ return "Sample data"
+
+ai_agent_settings = AzureAIAgentSettings.create()
+
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+ ):
+ agent_definition = await client.agents.create_agent(
+ model=ai_agent_settings.model_deployment_name,
+ )
+
+ agent = AzureAIAgent(
+ client=client,
+ definition=agent_definition,
+ plugins=[SamplePlugin()]
+ )
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+## Advanced Features
+
+An `AzureAIAgent` can leverage advanced tools such as:
+
+- [Code Interpreter](#code-interpreter)
+- [File Search](#file-search)
+- [OpenAPI integration](#openapi-integration)
+- [Azure AI Search integration](#azureai-search-integration)
+
+### Code Interpreter
+
+Code Interpreter allows the agents to write and run Python code in a sandboxed execution environment ([Azure AI Agent Service Code Interpreter](/azure/ai-services/agents/how-to/tools/code-interpreter)).
+
+::: zone pivot="programming-language-csharp"
+```c#
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+AgentsClient agentsClient = client.GetAgentsClient();
+
+Agent definition = agentsClient.CreateAgentAsync(
+ "<name of the the model used by the agent>",
+ name: "<agent name>",
+ description: "<agent description>",
+ instructions: "<agent instructions>",
+ tools: [new CodeInterpreterToolDefinition()],
+ toolResources:
+ new()
+ {
+ CodeInterpreter = new()
+ {
+ FileIds = { ... },
+ }
+ }));
+
+AzureAIAgent agent = new(definition, agentsClient);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+from azure.ai.projects.models import CodeInterpreterTool
+
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+ ):
+ code_interpreter = CodeInterpreterTool()
+ agent_definition = await client.agents.create_agent(
+ model=ai_agent_settings.model_deployment_name,
+ tools=code_interpreter.definitions,
+ tool_resources=code_interpreter.resources,
+ )
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+### File Search
+
+File search augments agents with knowledge from outside its model ([Azure AI Agent Service File Search Tool](/azure/ai-services/agents/how-to/tools/file-search)).
+
+::: zone pivot="programming-language-csharp"
+
+```c#
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+AgentsClient agentsClient = client.GetAgentsClient();
+
+Agent definition = agentsClient.CreateAgentAsync(
+ "<name of the the model used by the agent>",
+ name: "<agent name>",
+ description: "<agent description>",
+ instructions: "<agent instructions>",
+ tools: [new FileSearchToolDefinition()],
+ toolResources:
+ new()
+ {
+ FileSearch = new()
+ {
+ VectorStoreIds = { ... },
+ }
+ }));
+
+AzureAIAgent agent = new(definition, agentsClient);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+from azure.ai.projects.models import FileSearchTool
+
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+ ):
+ file_search = FileSearchTool(vector_store_ids=[vector_store.id])
+ agent_definition = await client.agents.create_agent(
+ model=ai_agent_settings.model_deployment_name,
+ tools=file_search.definitions,
+ tool_resources=file_search.resources,
+ )
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+### OpenAPI Integration
+
+Connects your agent to an external API ([How to use Azure AI Agent Service with OpenAPI Specified Tools](/azure/ai-services/agents/how-to/tools/openapi-spec)).
+
+::: zone pivot="programming-language-csharp"
+```c#
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+AgentsClient agentsClient = client.GetAgentsClient();
+
+string apiJsonSpecification = ...; // An Open API JSON specification
+
+Agent definition = agentsClient.CreateAgentAsync(
+ "<name of the the model used by the agent>",
+ name: "<agent name>",
+ description: "<agent description>",
+ instructions: "<agent instructions>",
+ tools: [
+ new OpenApiToolDefinition(
+ "<api name>",
+ "<api description>",
+ BinaryData.FromString(apiJsonSpecification),
+ new OpenApiAnonymousAuthDetails())
+ ],
+);
+
+AzureAIAgent agent = new(definition, agentsClient);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+from azure.ai.projects.models import OpenApiTool, OpenApiAnonymousAuthDetails
+
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+ ):
+ openapi_spec_file_path = "sample/filepath/..."
+ with open(os.path.join(openapi_spec_file_path, "spec_one.json")) as file_one:
+ openapi_spec_one = json.loads(file_one.read())
+ with open(os.path.join(openapi_spec_file_path, "spec_two.json")) as file_two:
+ openapi_spec_two = json.loads(file_two.read())
+
+ # Note that connection or managed identity auth setup requires additional setup in Azure
+ auth = OpenApiAnonymousAuthDetails()
+ openapi_tool_one = OpenApiTool(
+ name="<name>",
+ spec=openapi_spec_one,
+ description="<description>",
+ auth=auth,
+ )
+ openapi_tool_two = OpenApiTool(
+ name="<name>",
+ spec=openapi_spec_two,
+ description="<description>",
+ auth=auth,
+ )
+
+ agent_definition = await client.agents.create_agent(
+ model=ai_agent_settings.model_deployment_name,
+ tools=openapi_tool_one.definitions + openapi_tool_two.definitions,
+ )
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+### AzureAI Search Integration
+
+Use an existing Azure AI Search index with with your agent ([Use an existing AI Search index](/azure/ai-services/agents/how-to/tools/azure-ai-search)).
+
+::: zone pivot="programming-language-csharp"
+```c#
+AIProjectClient client = AzureAIAgent.CreateAzureAIClient("<your connection-string>", new AzureCliCredential());
+AgentsClient agentsClient = client.GetAgentsClient();
+
+ConnectionsClient cxnClient = client.GetConnectionsClient();
+ListConnectionsResponse searchConnections = await cxnClient.GetConnectionsAsync(AzureAIP.ConnectionType.AzureAISearch);
+ConnectionResponse searchConnection = searchConnections.Value[0];
+
+Agent definition = agentsClient.CreateAgentAsync(
+ "<name of the the model used by the agent>",
+ name: "<agent name>",
+ description: "<agent description>",
+ instructions: "<agent instructions>",
+ tools: [new AzureAIP.AzureAISearchToolDefinition()],
+ toolResources: new()
+ {
+ AzureAISearch = new()
+ {
+ IndexList = { new AzureAIP.IndexResource(searchConnection.Id, "<your index name>") }
+ }
+ });
+
+AzureAIAgent agent = new(definition, agentsClient);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+from azure.ai.projects.models import AzureAISearchTool, ConnectionType
+
+async with (
+ DefaultAzureCredential() as creds,
+ AzureAIAgent.create_client(credential=creds) as client,
+ ):
+ conn_list = await client.connections.list()
+
+ ai_search_conn_id = ""
+ for conn in conn_list:
+ if conn.connection_type == ConnectionType.AZURE_AI_SEARCH:
+ ai_search_conn_id = conn.id
+ break
+
+ ai_search = AzureAISearchTool(
+ index_connection_id=ai_search_conn_id,
+ index_name=AZURE_AI_SEARCH_INDEX_NAME,
+ )
+
+ agent_definition = await client.agents.create_agent(
+ model=ai_agent_settings.model_deployment_name,
+ instructions="Answer questions using your index.",
+ tools=ai_search.definitions,
+ tool_resources=ai_search.resources,
+ headers={"x-ms-enable-preview": "true"},
+ )
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+### Retrieving an Existing `AzureAIAgent`
+
+An existing agent can be retrieved and reused by specifying its assistant ID:
+
+::: zone pivot="programming-language-csharp"
+
+```c#
+Agent definition = agentsClient.GetAgentAsync("<your agent id>");
+AzureAIAgent agent = new(definition, agentsClient);
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+agent_definition = await client.agents.get_agent(assistant_id="your-agent-id")
+agent = AzureAIAgent(client=client, definition=agent_definition)
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+## Deleting an `AzureAIAgent`
+
+Agents and their associated threads can be deleted when no longer needed:
+
+::: zone pivot="programming-language-csharp"
+
+```c#
+await agentsClient.DeleteThreadAsync(thread.Id);
+await agentsClient.DeleteAgentAsync(agent.Id);
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+await client.agents.delete_thread(thread.id)
+await client.agents.delete_agent(agent.id)
+```
+::: zone-end
+
+If working with a vector store or files, they may be deleted as well:
+
+::: zone pivot="programming-language-csharp"
+```c#
+await agentsClient.DeleteVectorStoreAsync("<your store id>");
+await agentsClient.DeleteFileAsync("<your file id>");
+```
+::: zone-end
+
+::: zone pivot="programming-language-python"
+```python
+await client.agents.delete_file(file_id=file.id)
+await client.agents.delete_vector_store(vector_store_id=vector_store.id)
+```
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+> More information on the _file search_ tool is described in the [Azure AI Agent Service file search tool](/azure/ai-services/agents/how-to/tools/file-search) article.
+
+## How-To
+
+For practical examples of using an `AzureAIAgent`, see our code samples on GitHub:
+
+::: zone pivot="programming-language-csharp"
+
+- [Getting Started with Azure AI Agents](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent)
+- [Advanced Azure AI Agent Code Samples](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/Concepts/Agents)
+
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+- [Getting Started with Azure AI Agents](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/getting_started_with_agents/azure_ai_agent)
+- [Advanced Azure AI Agent Code Samples](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/agents/azure_ai_agent)
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+> [!div class="nextstepaction"]
+> [Agent Collaboration in Agent Chat](./agent-chat.md)
\ No newline at end of file
diff --git a/semantic-kernel/Frameworks/agent/chat-completion-agent.md b/semantic-kernel/Frameworks/agent/chat-completion-agent.md
index 0257948f..c8d3b6d7 100644
--- a/semantic-kernel/Frameworks/agent/chat-completion-agent.md
+++ b/semantic-kernel/Frameworks/agent/chat-completion-agent.md
@@ -1,5 +1,5 @@
---
-title: Exploring the Semantic Kernel Chat Completion Agent
+title: Exploring the Semantic Kernel ChatCompletionAgent
description: An exploration of the definition, behaviors, and usage patterns for a Chat Completion Agent
zone_pivot_groups: programming-languages
author: crickman
@@ -8,7 +8,7 @@ ms.author: crickman
ms.date: 09/13/2024
ms.service: semantic-kernel
---
-# Exploring the _Semantic Kernel_ Chat Completion Agent
+# Exploring the Semantic Kernel `ChatCompletionAgent`
> [!IMPORTANT]
> This feature is in the release candidate stage. Features at this stage are nearly complete and generally stable, though they may undergo minor refinements or optimizations before reaching full general availability.
@@ -74,7 +74,38 @@ Onnx|[`Microsoft.SemanticKernel.Connectors.Onnx`](/dotnet/api/microsoft.semantic
::: zone-end
-## Creating a Chat Completion Agent
+## Preparing Your Development Environment
+
+To proceed with developing an `AzureAIAgent`, configure your development environment with the appropriate packages.
+
+::: zone pivot="programming-language-csharp"
+
+Add the `Microsoft.SemanticKernel.Agents.Core` package to your project:
+
+```pwsh
+dotnet add package Microsoft.SemanticKernel.Agents.Core --prerelease
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+Install the `semantic-kernel` package:
+
+```bash
+pip install semantic-kernel
+```
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+> Agents are currently unavailable in Java.
+
+::: zone-end
+
+
+## Creating a `ChatCompletionAgent`
A _chat completion agent_ is fundamentally based on an [AI services](../../concepts/ai-services/index.md). As such, creating an _chat completion agent_ starts with creating a [`Kernel`](../../concepts/kernel.md) instance that contains one or more chat-completion services and then instantiating the agent with a reference to that [`Kernel`](../../concepts/kernel.md) instance.
@@ -224,8 +255,7 @@ agent = ChatCompletionAgent(...)
chat = ChatHistory()
# Add the user message
-chat.add_message(ChatMessageContent(role=AuthorRole.USER, content=input))
-
+chat.add_user_message(user_input)
# Generate the agent response
response = await agent.get_response(chat)
# response is a `ChatMessageContent` object
@@ -240,7 +270,7 @@ agent = ChatCompletionAgent(...)
chat = ChatHistory()
# Add the user message
-chat.add_user_message(ChatMessageContent(role=AuthorRole.USER, content=input))
+chat.add_user_message(user_input)
# Generate the agent response(s)
async for response in agent.invoke(chat):
@@ -281,4 +311,4 @@ For an end-to-end example for a `ChatCompletionAgent`, see:
> [!div class="nextstepaction"]
-> [Exploring `OpenAIAssistantAgent`](./assistant-agent.md)
+> [Exploring the OpenAI Assistant Agent](./assistant-agent.md)
diff --git a/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md b/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md
index 4329e493..083d3841 100644
--- a/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md
+++ b/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md
@@ -862,8 +862,6 @@ Try using these suggested inputs:
8. its good, but is it ready for my college professor?
```csharp
-// Copyright (c) Microsoft. All rights reserved.
-
using System;
using System.ComponentModel;
using System.Diagnostics;
@@ -1131,8 +1129,6 @@ You can try using one of the suggested inputs. As the agent chat begins, the age
> You can reference any file by providing `@<file_path_to_file>`. To reference the "WomensSuffrage" text from above, download it [here](https://github.com/microsoft/semantic-kernel/blob/main/python/samples/learn_resources/resources/WomensSuffrage.txt) and place it in your current working directory. You can then reference it with `@WomensSuffrage.txt`.
```python
-# Copyright (c) Microsoft. All rights reserved.
-
import asyncio
import os
diff --git a/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md b/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md
index be6fdc4d..0ce0dfcc 100644
--- a/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md
+++ b/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md
@@ -208,13 +208,12 @@ The full example code is provided in the [Final](#final) section. Refer to that
Prior to creating an `OpenAIAssistantAgent`, ensure the configuration settings are available and prepare the file resources.
-Instantiate the `Settings` class referenced in the previous [Configuration](#configuration) section. Use the settings to also create an `OpenAIClientProvider` that will be used for the [Agent Definition](#agent-definition) as well as file-upload.
+Instantiate the `Settings` class referenced in the previous [Configuration](#configuration) section. Use the settings to also create an `AzureOpenAIClient` that will be used for the [Agent Definition](#agent-definition) as well as file-upload.
```csharp
Settings settings = new();
-OpenAIClientProvider clientProvider =
- OpenAIClientProvider.ForAzureOpenAI(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint));
+AzureOpenAIClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint));
```
::: zone-end
@@ -226,11 +225,11 @@ OpenAIClientProvider clientProvider =
::: zone pivot="programming-language-csharp"
-Use the `OpenAIClientProvider` to access an `OpenAIFileClient` and upload the two data-files described in the previous [Configuration](#configuration) section, preserving the _File Reference_ for final clean-up.
+Use the `AzureOpenAIClient` to access an `OpenAIFileClient` and upload the two data-files described in the previous [Configuration](#configuration) section, preserving the _File Reference_ for final clean-up.
```csharp
Console.WriteLine("Uploading files...");
-OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient();
+OpenAIFileClient fileClient = client.GetOpenAIFileClient();
OpenAIFile fileDataCountryDetail = await fileClient.UploadFileAsync("PopulationByAdmin1.csv", FileUploadPurpose.Assistants);
OpenAIFile fileDataCountryList = await fileClient.UploadFileAsync("PopulationByCountry.csv", FileUploadPurpose.Assistants);
```
@@ -303,27 +302,27 @@ We first set up the Azure OpenAI resources to obtain the client and model. Next,
::: zone pivot="programming-language-csharp"
-We are now ready to instantiate an `OpenAIAssistantAgent`. The agent is configured with its target model, _Instructions_, and the _Code Interpreter_ tool enabled. Additionally, we explicitly associate the two data files with the _Code Interpreter_ tool.
+We are now ready to instantiate an `OpenAIAssistantAgent` by first creating an assistant definition. The assistant is configured with its target model, _Instructions_, and the _Code Interpreter_ tool enabled. Additionally, we explicitly associate the two data files with the _Code Interpreter_ tool.
```csharp
Console.WriteLine("Defining agent...");
-OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.CreateAsync(
- clientProvider,
- new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment)
- {
- Name = "SampleAssistantAgent",
- Instructions =
- """
- Analyze the available data to provide an answer to the user's question.
- Always format response using markdown.
- Always include a numerical index that starts at 1 for any lists or tables.
- Always sort lists in ascending order.
- """,
- EnableCodeInterpreter = true,
- CodeInterpreterFileIds = [fileDataCountryList.Id, fileDataCountryDetail.Id],
- },
- new Kernel());
+AssistantClient assistantClient = client.GetAssistantClient();
+ Assistant assistant =
+ await assistantClient.CreateAssistantAsync(
+ settings.AzureOpenAI.ChatModelDeployment,
+ name: "SampleAssistantAgent",
+ instructions:
+ """
+ Analyze the available data to provide an answer to the user's question.
+ Always format response using markdown.
+ Always include a numerical index that starts at 1 for any lists or tables.
+ Always sort lists in ascending order.
+ """,
+ enableCodeInterpreter: true,
+ codeInterpreterFileIds: [fileDataCountryList.Id, fileDataCountryDetail.Id]);
+
+// Create agent
+OpenAIAssistantAgent agent = new(assistant, assistantClient);
```
::: zone-end
@@ -355,7 +354,7 @@ Let's also ensure the resources are removed at the end of execution to minimize
::: zone pivot="programming-language-csharp"
```csharp
Console.WriteLine("Creating thread...");
-string threadId = await agent.CreateThreadAsync();
+AssistantThread thread = await assistantClient.CreateThreadAsync();
Console.WriteLine("Ready!");
@@ -374,8 +373,8 @@ finally
Console.WriteLine("Cleaning-up...");
await Task.WhenAll(
[
- agent.DeleteThreadAsync(threadId),
- agent.DeleteAsync(),
+ assistantClient.DeleteThreadAsync(thread.Id),
+ assistantClient.DeleteAssistantAsync(assistant.Id),
fileClient.DeleteFileAsync(fileDataCountryList.Id),
fileClient.DeleteFileAsync(fileDataCountryDetail.Id),
]);
@@ -424,7 +423,7 @@ if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase))
break;
}
-await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input));
+await agent.AddChatMessageAsync(thread.Id, new ChatMessageContent(AuthorRole.User, input));
Console.WriteLine();
```
@@ -542,7 +541,7 @@ To generate an `Agent` response to user input, invoke the agent by specifying th
::: zone pivot="programming-language-csharp"
```csharp
bool isCode = false;
-await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(threadId))
+await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(thread.Id))
{
if (isCode != (response.Metadata?.ContainsKey(OpenAIAssistantAgent.CodeInterpreterMetadataKey) ?? false))
{
@@ -604,17 +603,19 @@ Try using these suggested inputs:
::: zone pivot="programming-language-csharp"
```csharp
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Agents.OpenAI;
+using Microsoft.SemanticKernel.ChatCompletion;
+using OpenAI.Assistants;
+using OpenAI.Files;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
-using Azure.Identity;
-using Microsoft.SemanticKernel;
-using Microsoft.SemanticKernel.Agents.OpenAI;
-using Microsoft.SemanticKernel.ChatCompletion;
-using OpenAI.Files;
namespace AgentsSample;
@@ -625,35 +626,39 @@ public static class Program
// Load configuration from environment variables or user secrets.
Settings settings = new();
- OpenAIClientProvider clientProvider =
- OpenAIClientProvider.ForAzureOpenAI(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint));
+ // Initialize the clients
+ AzureOpenAIClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint));
+ //OpenAIClient client = OpenAIAssistantAgent.CreateOpenAIClient(new ApiKeyCredential(settings.OpenAI.ApiKey)));
+ AssistantClient assistantClient = client.GetAssistantClient();
+ OpenAIFileClient fileClient = client.GetOpenAIFileClient();
+ // Upload files
Console.WriteLine("Uploading files...");
- OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient();
OpenAIFile fileDataCountryDetail = await fileClient.UploadFileAsync("PopulationByAdmin1.csv", FileUploadPurpose.Assistants);
OpenAIFile fileDataCountryList = await fileClient.UploadFileAsync("PopulationByCountry.csv", FileUploadPurpose.Assistants);
- Console.WriteLine("Defining agent...");
- OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.CreateAsync(
- clientProvider,
- new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment)
- {
- Name = "SampleAssistantAgent",
- Instructions =
+ // Define assistant
+ Console.WriteLine("Defining assistant...");
+ Assistant assistant =
+ await assistantClient.CreateAssistantAsync(
+ settings.AzureOpenAI.ChatModelDeployment,
+ name: "SampleAssistantAgent",
+ instructions:
"""
Analyze the available data to provide an answer to the user's question.
Always format response using markdown.
Always include a numerical index that starts at 1 for any lists or tables.
Always sort lists in ascending order.
""",
- EnableCodeInterpreter = true,
- CodeInterpreterFileIds = [fileDataCountryList.Id, fileDataCountryDetail.Id],
- },
- new Kernel());
+ enableCodeInterpreter: true,
+ codeInterpreterFileIds: [fileDataCountryList.Id, fileDataCountryDetail.Id]);
+ // Create agent
+ OpenAIAssistantAgent agent = new(assistant, assistantClient);
+
+ // Create the conversation thread
Console.WriteLine("Creating thread...");
- string threadId = await agent.CreateThreadAsync();
+ AssistantThread thread = await assistantClient.CreateThreadAsync();
Console.WriteLine("Ready!");
@@ -676,12 +681,12 @@ public static class Program
break;
}
- await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input));
+ await agent.AddChatMessageAsync(thread.Id, new ChatMessageContent(AuthorRole.User, input));
Console.WriteLine();
bool isCode = false;
- await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(threadId))
+ await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(thread.Id))
{
if (isCode != (response.Metadata?.ContainsKey(OpenAIAssistantAgent.CodeInterpreterMetadataKey) ?? false))
{
@@ -709,8 +714,8 @@ public static class Program
Console.WriteLine("Cleaning-up...");
await Task.WhenAll(
[
- agent.DeleteThreadAsync(threadId),
- agent.DeleteAsync(),
+ assistantClient.DeleteThreadAsync(thread.Id),
+ assistantClient.DeleteAssistantAsync(assistant.Id),
fileClient.DeleteFileAsync(fileDataCountryList.Id),
fileClient.DeleteFileAsync(fileDataCountryDetail.Id),
]);
@@ -761,8 +766,6 @@ public static class Program
::: zone pivot="programming-language-python"
```python
-# Copyright (c) Microsoft. All rights reserved.
-
import asyncio
import logging
import os
diff --git a/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md b/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md
index 001395a6..81bcc550 100644
--- a/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md
+++ b/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md
@@ -209,16 +209,12 @@ Prior to creating an `OpenAIAssistantAgent`, ensure the configuration settings a
::: zone pivot="programming-language-csharp"
-Instantiate the `Settings` class referenced in the previous [Configuration](#configuration) section. Use the settings to also create an `OpenAIClientProvider` that will be used for the [Agent Definition](#agent-definition) as well as file-upload and the creation of a `VectorStore`.
+Instantiate the `Settings` class referenced in the previous [Configuration](#configuration) section. Use the settings to also create an `AzureOpenAIClient` that will be used for the [Agent Definition](#agent-definition) as well as file-upload and the creation of a `VectorStore`.
```csharp
-
Settings settings = new();
-OpenAIClientProvider clientProvider =
- OpenAIClientProvider.ForAzureOpenAI(
- new AzureCliCredential(),
- new Uri(settings.AzureOpenAI.Endpoint));
+AzureOpenAIClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint));
```
::: zone-end
@@ -242,11 +238,11 @@ Now create an empty _Vector Store for use with the _File Search_ tool:
::: zone pivot="programming-language-csharp"
-Use the `OpenAIClientProvider` to access a `VectorStoreClient` and create a `VectorStore`.
+Use the `AzureOpenAIClient` to access a `VectorStoreClient` and create a `VectorStore`.
```csharp
Console.WriteLine("Creating store...");
-VectorStoreClient storeClient = clientProvider.Client.GetVectorStoreClient();
+VectorStoreClient storeClient = client.GetVectorStoreClient();
CreateVectorStoreOperation operation = await storeClient.CreateVectorStoreAsync(waitUntilCompleted: true);
string storeId = operation.VectorStoreId;
```
@@ -317,7 +313,7 @@ Now upload those files and add them to the _Vector Store_ by using the previousl
Dictionary<string, OpenAIFile> fileReferences = [];
Console.WriteLine("Uploading files...");
-OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient();
+OpenAIFileClient fileClient = client.GetOpenAIFileClient();
foreach (string fileName in _fileNames)
{
OpenAIFile fileInfo = await fileClient.UploadFileAsync(fileName, FileUploadPurpose.Assistants);
@@ -339,27 +335,26 @@ We are now ready to instantiate an `OpenAIAssistantAgent`. The agent is configur
::: zone pivot="programming-language-csharp"
-We will utilize the `OpenAIClientProvider` again as part of creating the `OpenAIAssistantAgent`:
+We will utilize the `AzureOpenAIClient` again as part of creating the `OpenAIAssistantAgent`:
```csharp
-Console.WriteLine("Defining agent...");
-OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.CreateAsync(
- clientProvider,
- new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment)
- {
- Name = "SampleAssistantAgent",
- Instructions =
+Console.WriteLine("Defining assistant...");
+Assistant assistant =
+ await assistantClient.CreateAssistantAsync(
+ settings.AzureOpenAI.ChatModelDeployment,
+ name: "SampleAssistantAgent",
+ instructions:
"""
The document store contains the text of fictional stories.
Always analyze the document store to provide an answer to the user's question.
Never rely on your knowledge of stories not included in the document store.
Always format response using markdown.
""",
- EnableFileSearch = true,
- VectorStoreId = storeId,
- },
- new Kernel());
+ enableFileSearch: true,
+ vectorStoreId: storeId);
+
+// Create agent
+OpenAIAssistantAgent agent = new(assistant, assistantClient);
```
::: zone-end
@@ -402,7 +397,7 @@ Let's also ensure the resources are removed at the end of execution to minimize
::: zone pivot="programming-language-csharp"
```csharp
Console.WriteLine("Creating thread...");
-string threadId = await agent.CreateThreadAsync();
+AssistantThread thread = await assistantClient.CreateThreadAsync();
Console.WriteLine("Ready!");
@@ -420,8 +415,8 @@ finally
Console.WriteLine("Cleaning-up...");
await Task.WhenAll(
[
- agent.DeleteThreadAsync(threadId),
- agent.DeleteAsync(),
+ assistantClient.DeleteThreadAsync(thread.Id),
+ assistantClient.DeleteAssistantAsync(assistant.Id),
storeClient.DeleteVectorStoreAsync(storeId),
..fileReferences.Select(fileReference => fileClient.DeleteFileAsync(fileReference.Key))
]);
@@ -471,7 +466,7 @@ if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase))
break;
}
-await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input));
+await agent.AddChatMessageAsync(thread.Id, new ChatMessageContent(AuthorRole.User, input));
Console.WriteLine();
```
::: zone-end
@@ -524,7 +519,7 @@ To generate an `Agent` response to user input, invoke the agent by specifying th
::: zone pivot="programming-language-csharp"
```csharp
List<StreamingAnnotationContent> footnotes = [];
-await foreach (StreamingChatMessageContent chunk in agent.InvokeStreamingAsync(threadId))
+await foreach (StreamingChatMessageContent chunk in agent.InvokeStreamingAsync(thread.Id))
{
// Capture annotations for footnotes
footnotes.AddRange(chunk.Items.OfType<StreamingAnnotationContent>());
@@ -585,16 +580,19 @@ Try using these suggested inputs:
::: zone pivot="programming-language-csharp"
```csharp
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
+using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Agents;
using Microsoft.SemanticKernel.Agents.OpenAI;
using Microsoft.SemanticKernel.ChatCompletion;
+using OpenAI.Assistants;
using OpenAI.Files;
using OpenAI.VectorStores;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
namespace AgentsSample;
@@ -616,21 +614,21 @@ public static class Program
// Load configuration from environment variables or user secrets.
Settings settings = new();
- OpenAIClientProvider clientProvider =
- OpenAIClientProvider.ForAzureOpenAI(
- new AzureCliCredential(),
- new Uri(settings.AzureOpenAI.Endpoint));
+ // Initialize the clients
+ AzureOpenAIClient client = OpenAIAssistantAgent.CreateAzureOpenAIClient(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint));
+ //OpenAIClient client = OpenAIAssistantAgent.CreateOpenAIClient(new ApiKeyCredential(settings.OpenAI.ApiKey)));
+ AssistantClient assistantClient = client.GetAssistantClient();
+ OpenAIFileClient fileClient = client.GetOpenAIFileClient();
+ VectorStoreClient storeClient = client.GetVectorStoreClient();
+ // Create the vector store
Console.WriteLine("Creating store...");
- VectorStoreClient storeClient = clientProvider.Client.GetVectorStoreClient();
CreateVectorStoreOperation operation = await storeClient.CreateVectorStoreAsync(waitUntilCompleted: true);
string storeId = operation.VectorStoreId;
- // Retain file references.
- Dictionary<string, OpenAIFile> fileReferences = [];
-
+ // Upload files and retain file references.
Console.WriteLine("Uploading files...");
- OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient();
+ Dictionary<string, OpenAIFile> fileReferences = [];
foreach (string fileName in _fileNames)
{
OpenAIFile fileInfo = await fileClient.UploadFileAsync(fileName, FileUploadPurpose.Assistants);
@@ -638,28 +636,28 @@ public static class Program
fileReferences.Add(fileInfo.Id, fileInfo);
}
-
- Console.WriteLine("Defining agent...");
- OpenAIAssistantAgent agent =
- await OpenAIAssistantAgent.CreateAsync(
- clientProvider,
- new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment)
- {
- Name = "SampleAssistantAgent",
- Instructions =
+ // Define assistant
+ Console.WriteLine("Defining assistant...");
+ Assistant assistant =
+ await assistantClient.CreateAssistantAsync(
+ settings.AzureOpenAI.ChatModelDeployment,
+ name: "SampleAssistantAgent",
+ instructions:
"""
The document store contains the text of fictional stories.
Always analyze the document store to provide an answer to the user's question.
Never rely on your knowledge of stories not included in the document store.
Always format response using markdown.
""",
- EnableFileSearch = true,
- VectorStoreId = storeId,
- },
- new Kernel());
+ enableFileSearch: true,
+ vectorStoreId: storeId);
+ // Create agent
+ OpenAIAssistantAgent agent = new(assistant, assistantClient);
+
+ // Create the conversation thread
Console.WriteLine("Creating thread...");
- string threadId = await agent.CreateThreadAsync();
+ AssistantThread thread = await assistantClient.CreateThreadAsync();
Console.WriteLine("Ready!");
@@ -681,11 +679,11 @@ public static class Program
break;
}
- await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input));
+ await agent.AddChatMessageAsync(thread.Id, new ChatMessageContent(AuthorRole.User, input));
Console.WriteLine();
List<StreamingAnnotationContent> footnotes = [];
- await foreach (StreamingChatMessageContent chunk in agent.InvokeStreamingAsync(threadId))
+ await foreach (StreamingChatMessageContent chunk in agent.InvokeStreamingAsync(thread.Id))
{
// Capture annotations for footnotes
footnotes.AddRange(chunk.Items.OfType<StreamingAnnotationContent>());
@@ -713,8 +711,8 @@ public static class Program
Console.WriteLine("Cleaning-up...");
await Task.WhenAll(
[
- agent.DeleteThreadAsync(threadId),
- agent.DeleteAsync(),
+ assistantClient.DeleteThreadAsync(thread.Id),
+ assistantClient.DeleteAssistantAsync(assistant.Id),
storeClient.DeleteVectorStoreAsync(storeId),
..fileReferences.Select(fileReference => fileClient.DeleteFileAsync(fileReference.Key))
]);
@@ -729,8 +727,6 @@ public static class Program
::: zone pivot="programming-language-python"
```python
-# Copyright (c) Microsoft. All rights reserved.
-
import asyncio
import os
diff --git a/semantic-kernel/concepts/ai-services/TOC.yml b/semantic-kernel/concepts/ai-services/TOC.yml
index 8d24a54b..fd35b379 100644
--- a/semantic-kernel/concepts/ai-services/TOC.yml
+++ b/semantic-kernel/concepts/ai-services/TOC.yml
@@ -6,4 +6,6 @@
- name: Embedding generation
href: embedding-generation/TOC.yml
- name: AI Integrations
- href: integrations.md
\ No newline at end of file
+ href: integrations.md
+- name: Realtime
+ href: realtime.md
\ No newline at end of file
diff --git a/semantic-kernel/concepts/ai-services/index.md b/semantic-kernel/concepts/ai-services/index.md
index 3e473aec..9ce5d2fd 100644
--- a/semantic-kernel/concepts/ai-services/index.md
+++ b/semantic-kernel/concepts/ai-services/index.md
@@ -14,21 +14,23 @@ One of the main features of Semantic Kernel is its ability to add different AI s
Within Semantic Kernel, there are interfaces for the most popular AI tasks. In the table below, you can see the services that are supported by each of the SDKs.
-| Services | C# | Python | Java | Notes |
-|-----------------------------------|:----:|:------:|:----:|-------|
-| [Chat completion](./chat-completion/index.md) | ✅ | ✅ | ✅ |
-| Text generation | ✅ | ✅ | ✅ |
-| Embedding generation (Experimental) | ✅ | ✅ | ✅ |
-| Text-to-image (Experimental) | ✅ | ✅ | ❌ |
-| Image-to-text (Experimental) | ✅ | ❌ | ❌ |
-| Text-to-audio (Experimental) | ✅ | ✅ | ❌ |
-| Audio-to-text (Experimental) | ✅ | ✅ | ❌ |
+| Services | C# | Python | Java | Notes |
+| --------------------------------------------- | :---: | :----: | :---: | ----- |
+| [Chat completion](./chat-completion/index.md) | ✅ | ✅ | ✅ |
+| Text generation | ✅ | ✅ | ✅ |
+| Embedding generation (Experimental) | ✅ | ✅ | ✅ |
+| Text-to-image (Experimental) | ✅ | ✅ | ❌ |
+| Image-to-text (Experimental) | ✅ | ❌ | ❌ |
+| Text-to-audio (Experimental) | ✅ | ✅ | ❌ |
+| Audio-to-text (Experimental) | ✅ | ✅ | ❌ |
+| [Realtime](./realtime.md) (Experimental) | ❌ | ✅ | ❌ |
> [!TIP]
> In most scenarios, you will only need to add chat completion to your kernel, but to support multi-modal AI, you can add any of the above services to your kernel.
## Next steps
+
To learn more about each of the services, please refer to the specific articles for each service type. In each of the articles we provide sample code for adding the service to the kernel across multiple AI service providers.
> [!div class="nextstepaction"]
-> [Learn about chat completion](./chat-completion/index.md)
\ No newline at end of file
+> [Learn about chat completion](./chat-completion/index.md)
diff --git a/semantic-kernel/concepts/ai-services/integrations.md b/semantic-kernel/concepts/ai-services/integrations.md
index a8d34ec5..d30a6984 100644
--- a/semantic-kernel/concepts/ai-services/integrations.md
+++ b/semantic-kernel/concepts/ai-services/integrations.md
@@ -18,21 +18,22 @@ With the available AI connectors, developers can easily build AI agents with swa
### AI Services
-| Services | C# | Python | Java | Notes |
-|-----------------------------------|:----:|:------:|:----:|-------|
-| Text Generation | ✅ | ✅ | ✅ | Example: Text-Davinci-003 |
-| Chat Completion | ✅ | ✅ | ✅ | Example: GPT4, Chat-GPT |
-| Text Embeddings (Experimental) | ✅ | ✅ | ✅ | Example: Text-Embeddings-Ada-002 |
-| Text to Image (Experimental) | ✅ | ✅ | ❌ | Example: Dall-E |
-| Image to Text (Experimental) | ✅ | ❌ | ❌ | Example: Pix2Struct |
-| Text to Audio (Experimental) | ✅ | ✅ | ❌ | Example: Text-to-speech |
-| Audio to Text (Experimental) | ✅ | ✅ | ❌ | Example: Whisper |
+| Services | C# | Python | Java | Notes |
+| ------------------------------ | :---: | :----: | :---: | -------------------------------- |
+| Text Generation | ✅ | ✅ | ✅ | Example: Text-Davinci-003 |
+| Chat Completion | ✅ | ✅ | ✅ | Example: GPT4, Chat-GPT |
+| Text Embeddings (Experimental) | ✅ | ✅ | ✅ | Example: Text-Embeddings-Ada-002 |
+| Text to Image (Experimental) | ✅ | ✅ | ❌ | Example: Dall-E |
+| Image to Text (Experimental) | ✅ | ❌ | ❌ | Example: Pix2Struct |
+| Text to Audio (Experimental) | ✅ | ✅ | ❌ | Example: Text-to-speech |
+| Audio to Text (Experimental) | ✅ | ✅ | ❌ | Example: Whisper |
+| Realtime (Experimental) | ❌ | ✅ | ❌ | Example: gpt-4o-realtime-preview |
## Additional plugins
If you want to extend the functionality of your AI agent, you can use plugins to integrate with other Microsoft services. Here are some of the plugins that are available for Semantic Kernel:
-| Plugin | C# | Python | Java | Description |
-| ---------- | :-: | :----: | :--: | ----------- |
-| Logic Apps | ✅ | ✅ | ✅ | Build workflows within Logic Apps using its available connectors and import them as plugins in Semantic Kernel. [Learn more](../plugins/adding-logic-apps-as-plugins.md). |
-| Azure Container Apps Dynamic Sessions | ✅ | ✅ | ❌ | With dynamic sessions, you can recreate the Code Interpreter experience from the Assistants API by effortlessly spinning up Python containers where AI agents can execute Python code. [Learn more](/azure/container-apps/sessions). |
+| Plugin | C# | Python | Java | Description |
+| ------------------------------------- | :---: | :----: | :---: | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Logic Apps | ✅ | ✅ | ✅ | Build workflows within Logic Apps using its available connectors and import them as plugins in Semantic Kernel. [Learn more](../plugins/adding-logic-apps-as-plugins.md). |
+| Azure Container Apps Dynamic Sessions | ✅ | ✅ | ❌ | With dynamic sessions, you can recreate the Code Interpreter experience from the Assistants API by effortlessly spinning up Python containers where AI agents can execute Python code. [Learn more](/azure/container-apps/sessions). |
diff --git a/semantic-kernel/concepts/ai-services/realtime.md b/semantic-kernel/concepts/ai-services/realtime.md
new file mode 100644
index 00000000..dd559e91
--- /dev/null
+++ b/semantic-kernel/concepts/ai-services/realtime.md
@@ -0,0 +1,189 @@
+---
+title: Realtime AI Integrations for Semantic Kernel
+description: Learn about realtime multi-modal AI integrations available in Semantic Kernel.
+author: eavanvalkenburg
+ms.topic: conceptual
+ms.author: edvan
+ms.date: 02/26/2025
+ms.service: semantic-kernel
+---
+
+# Realtime Multi-modal APIs
+
+The first realtime API integration for Semantic Kernel has been added, it is currently only available in Python and considered experimental. This is because the underlying services are still being developed and are subject to changes and we might need to make breaking changes to the API in Semantic Kernel as we learn from customers how to use this and as we add other providers of these kinds of models and APIs.
+
+## Realtime Client abstraction
+
+To support different realtime APIs from different vendors, using different protocols, a new client abstraction has been added to the kernel. This client is used to connect to the realtime service and send and receive messages.
+The client is responsible for handling the connection to the service, sending messages, and receiving messages. The client is also responsible for handling any errors that occur during the connection or message sending/receiving process. Considering the way these models work, they can be considered agents more than regular chat completions, therefore they also take instructions, rather than a system message, they keep their own internal state and can be invoked to do work on our behalf.
+### Realtime API
+
+Any realtime client implements the following methods:
+
+| Method | Description |
+| ---------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `create_session` | Creates a new session |
+| `update_session` | Updates an existing session |
+| `delete_session` | Deletes an existing session |
+| `receive` | This is a asynchronous generator method that listens for messages from the service and yields them as they arrive. |
+| `send` | Sends a message to the service |
+
+### Python implementations
+
+The python version of Semantic Kernel currently supports the following realtime clients:
+
+| Client | Protocol | Modalities | Function calling enabled | Description |
+| ------ | --------- | ------------ | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| OpenAI | Websocket | Text & Audio | Yes | The OpenAI Realtime API is a websocket based api that allows you to send and receive messages in realtime, this connector uses the OpenAI Python package to connect and receive and send messages. |
+| OpenAI | WebRTC | Text & Audio | Yes | The OpenAI Realtime API is a WebRTC based api that allows you to send and receive messages in realtime, it needs a webRTC compatible audio track at session creation time. |
+| Azure | Websocket | Text & Audio | Yes | The Azure Realtime API is a websocket based api that allows you to send and receive messages in realtime, this uses the same package as the OpenAI websocket connector. |
+
+## Getting started
+
+To get started with the Realtime API, you need to install the `semantic-kernel` package with the `realtime` extra.
+
+```bash
+pip install semantic-kernel[realtime]
+```
+
+Depending on how you want to handle audio, you might need additional packages to interface with speakers and microphones, like `pyaudio` or `sounddevice`.
+
+### Websocket clients
+
+Then you can create a kernel and add the realtime client to it, this shows how to do that with a AzureRealtimeWebsocket connection, you can replace AzureRealtimeWebsocket with OpenAIRealtimeWebsocket without any further changes.
+
+```python
+from semantic_kernel.connectors.ai.open_ai import (
+ AzureRealtimeWebsocket,
+ AzureRealtimeExecutionSettings,
+ ListenEvents,
+)
+from semantic_kernel.contents import RealtimeAudioEvent, RealtimeTextEvent
+
+# this will use environment variables to get the api key, endpoint, api version and deployment name.
+realtime_client = AzureRealtimeWebsocket()
+settings = AzureRealtimeExecutionSettings(voice='alloy')
+async with realtime_client(settings=settings, create_response=True):
+ async for event in realtime_client.receive():
+ match event:
+ # receiving a piece of audio (and send it to a undefined audio player)
+ case RealtimeAudioEvent():
+ await audio_player.add_audio(event.audio)
+ # receiving a piece of audio transcript
+ case RealtimeTextEvent():
+ # Semantic Kernel parses the transcript to a TextContent object captured in a RealtimeTextEvent
+ print(event.text.text, end="")
+ case _:
+ # OpenAI Specific events
+ if event.service_type == ListenEvents.SESSION_UPDATED:
+ print("Session updated")
+ if event.service_type == ListenEvents.RESPONSE_CREATED:
+ print("\nMosscap (transcript): ", end="")
+```
+
+There are two important things to note, the first is that the `realtime_client` is an async context manager, this means that you can use it in an async function and use `async with` to create the session.
+The second is that the `receive` method is an async generator, this means that you can use it in a for loop to receive messages as they arrive.
+
+### WebRTC client
+
+The setup of a WebRTC connection is a bit more complex and so we need a extra parameter when creating the client. This parameter, `audio_track` needs to be a object that implements the `MediaStreamTrack` protocol of the `aiortc` package, this is also demonstrated in the samples that are linked below.
+
+To create a client that uses WebRTC, you would do the following:
+
+```python
+from semantic_kernel.connectors.ai.open_ai import (
+ ListenEvents,
+ OpenAIRealtimeExecutionSettings,
+ OpenAIRealtimeWebRTC,
+)
+from aiortc.mediastreams import MediaStreamTrack
+
+class AudioRecorderWebRTC(MediaStreamTrack):
+ # implement the MediaStreamTrack methods.
+
+realtime_client = OpenAIRealtimeWebRTC(audio_track=AudioRecorderWebRTC())
+# Create the settings for the session
+settings = OpenAIRealtimeExecutionSettings(
+ instructions="""
+You are a chat bot. Your name is Mosscap and
+you have one goal: figure out what people need.
+Your full name, should you need to know it, is
+Splendid Speckled Mosscap. You communicate
+effectively, but you tend to answer with long
+flowery prose.
+""",
+ voice="shimmer",
+)
+audio_player = AudioPlayer
+async with realtime_client(settings=settings, create_response=True):
+ async for event in realtime_client.receive():
+ match event.event_type:
+ # receiving a piece of audio (and send it to a undefined audio player)
+ case "audio":
+ await audio_player.add_audio(event.audio)
+ case "text":
+ # the model returns both audio and transcript of the audio, which we will print
+ print(event.text.text, end="")
+ case "service":
+ # OpenAI Specific events
+ if event.service_type == ListenEvents.SESSION_UPDATED:
+ print("Session updated")
+ if event.service_type == ListenEvents.RESPONSE_CREATED:
+ print("\nMosscap (transcript): ", end="")
+```
+
+Both of these samples receive the audio as RealtimeAudioEvent and then they pass that to a unspecified audio_player object.
+
+### Audio output callback
+
+Next to this we have a parameter called `audio_output_callback` on the `receive` method and on the class creation. This callback will be called first before any further handling of the audio and gets a `numpy` array of the audio data, instead of it being parsed into AudioContent and returned as a RealtimeAudioEvent that you can then handle, which is what happens above. This has shown to give smoother audio output because there is less overhead between the audio data coming in and it being given to the player.
+
+This example shows how to define and use the `audio_output_callback`:
+
+```python
+from semantic_kernel.connectors.ai.open_ai import (
+ ListenEvents,
+ OpenAIRealtimeExecutionSettings,
+ OpenAIRealtimeWebRTC,
+)
+from aiortc.mediastreams import MediaStreamTrack
+
+class AudioRecorderWebRTC(MediaStreamTrack):
+ # implement the MediaStreamTrack methods.
+
+class AudioPlayer:
+ async def play_audio(self, content: np.ndarray):
+ # implement the audio player
+
+realtime_client = OpenAIRealtimeWebRTC(audio_track=AudioRecorderWebRTC())
+# Create the settings for the session
+settings = OpenAIRealtimeExecutionSettings(
+ instructions="""
+You are a chat bot. Your name is Mosscap and
+you have one goal: figure out what people need.
+Your full name, should you need to know it, is
+Splendid Speckled Mosscap. You communicate
+effectively, but you tend to answer with long
+flowery prose.
+""",
+ voice="shimmer",
+)
+audio_player = AudioPlayer
+async with realtime_client(settings=settings, create_response=True):
+ async for event in realtime_client.receive(audio_output_callback=audio_player.play_audio):
+ match event.event_type:
+ # no need to handle case: "audio"
+ case "text":
+ # the model returns both audio and transcript of the audio, which we will print
+ print(event.text.text, end="")
+ case "service":
+ # OpenAI Specific events
+ if event.service_type == ListenEvents.SESSION_UPDATED:
+ print("Session updated")
+ if event.service_type == ListenEvents.RESPONSE_CREATED:
+ print("\nMosscap (transcript): ", end="")
+```
+
+### Samples
+
+There are four samples in [our repo](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/realtime), they cover both the basics using both websockets and WebRTC, as well as a more complex setup including function calling. Finally there is a more [complex demo](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/demos/call_automation) that uses [Azure Communication Services](/azure/communication-services/) to allow you to call your Semantic Kernel enhanced realtime API.
diff --git a/semantic-kernel/concepts/plugins/adding-native-plugins.md b/semantic-kernel/concepts/plugins/adding-native-plugins.md
index 68d3dcba..f2f8be4e 100644
--- a/semantic-kernel/concepts/plugins/adding-native-plugins.md
+++ b/semantic-kernel/concepts/plugins/adding-native-plugins.md
@@ -18,6 +18,7 @@ Behind the scenes, Semantic Kernel will then use the descriptions you provide, a
## Providing the LLM with the right information
When authoring a plugin, you need to provide the AI agent with the right information to understand the capabilities of the plugin and its functions. This includes:
+
- The name of the plugin
- The names of the functions
- The descriptions of the functions
@@ -34,6 +35,7 @@ Below, we'll walk through the two different ways of providing your AI agent with
The easiest way to create a native plugin is to start with a class and then add methods annotated with the `KernelFunction` attribute. It is also recommended to liberally use the `Description` annotation to provide the AI agent with the necessary information to understand the function.
::: zone pivot="programming-language-csharp"
+
```csharp
public class LightsPlugin
{
@@ -73,9 +75,11 @@ public class LightsPlugin
}
}
```
+
::: zone-end
::: zone pivot="programming-language-python"
+
```python
from typing import List, Optional, Annotated
@@ -102,6 +106,7 @@ class LightsPlugin:
return light
return None
```
+
::: zone-end
::: zone pivot="programming-language-java"
@@ -110,10 +115,10 @@ class LightsPlugin:
::: zone-end
+::: zone pivot="programming-language-csharp"
> [!TIP]
> Because the LLMs are predominantly trained on Python code, it is recommended to use snake_case for function names and parameters (even if you're using C# or Java). This will help the AI agent better understand the function and its parameters.
-::: zone pivot="programming-language-csharp"
> [!TIP]
> Your functions can specify `Kernel`, `KernelArguments`, `ILoggerFactory`, `ILogger`, `IAIServiceSelector`, `CultureInfo`, `IFormatProvider`, `CancellationToken` as parameters and these will not be advertised to the LLM and will be automatically set when the function is called.
> If you rely on `KernelArguments` instead of explicit input arguments then your code will be responsible for performing type conversions.
@@ -122,6 +127,7 @@ class LightsPlugin:
If your function has a complex object as an input variable, Semantic Kernel will also generate a schema for that object and pass it to the AI agent. Similar to functions, you should provide `Description` annotations for properties that are non-obvious to the AI. Below is the definition for the `LightState` class and the `Brightness` enum.
::: zone pivot="programming-language-csharp"
+
```csharp
using System.Text.Json.Serialization;
@@ -152,19 +158,28 @@ public enum Brightness
High
}
```
+
::: zone-end
::: zone pivot="programming-language-python"
+
```python
+from enum import Enum
from typing import TypedDict
+class Brightness(Enum):
+ LOW = "low"
+ MEDIUM = "medium"
+ HIGH = "high"
+
class LightModel(TypedDict):
id: int
name: str
is_on: bool | None
- brightness: int | None
- hex: str | None
+ brightness: Brightness | None
+ color: Annotated[str | None, "The color of the light with a hex code (ensure you include the # symbol)"]
```
+
::: zone-end
::: zone pivot="programming-language-java"
@@ -176,12 +191,29 @@ class LightModel(TypedDict):
> [!NOTE]
> While this is a "fun" example, it does a good job showing just how complex a plugin's parameters can be. In this single case, we have a complex object with _four_ different types of properties: an integer, string, boolean, and enum. Semantic Kernel's value is that it can automatically generate the schema for this object and pass it to the AI agent and marshal the parameters generated by the AI agent into the correct object.
+::: zone pivot="programming-language-csharp"
+
Once you're done authoring your plugin class, you can add it to the kernel using the `AddFromType<>` or `AddFromObject` methods.
+::: zone-end
+
+::: zone pivot="programming-language-python"
+
+Once you're done authoring your plugin class, you can add it to the kernel using the `add_plugin` method.
+
+::: zone-end
+
+::: zone pivot="programming-language-java"
+
+Once you're done authoring your plugin class, you can add it to the kernel using the `AddFromType<>` or `AddFromObject` methods.
+
+::: zone-end
+
> [!TIP]
> When creating a function, always ask yourself "how can I give the AI additional help to use this function?" This can include using specific input types (avoid strings where possible), providing descriptions, and examples.
::: zone pivot="programming-language-csharp"
+
#### Adding a plugin using the `AddFromObject` method
The `AddFromObject` method allows you to add an instance of the plugin class directly to the plugin collection in case you want to directly control how the plugin is constructed.
@@ -325,9 +357,11 @@ builder.Services.AddTransient((serviceProvider)=> {
return new Kernel(serviceProvider, pluginCollection);
});
```
+
::: zone-end
::: zone pivot="programming-language-python"
+
#### Adding a plugin using the `add_plugin` method
The `add_plugin` method allows you to add a plugin instance to the kernel. Below is an example of how you can construct the `LightsPlugin` class and add it to the kernel.
@@ -349,8 +383,8 @@ lights_plugin = LightsPlugin(lights)
# Add the plugin to the kernel
kernel.add_plugin(lights_plugin)
```
-::: zone-end
+::: zone-end
::: zone pivot="programming-language-java"
@@ -497,7 +531,47 @@ This approach eliminates the need to manually provide and update the return type
::: zone-end
+::: zone pivot="programming-language-python"
+
+### Providing more details about the functions
+
+When creating a plugin in Python, you can provide additional information about the functions in the `kernel_function` decorator. This information will be used by the AI agent to understand the functions better.
+
+```python
+from typing import List, Optional, Annotated
+
+class LightsPlugin:
+ def __init__(self, lights: List[LightModel]):
+ self._lights = lights
+
+ @kernel_function(name="GetLights", description="Gets a list of lights and their current state")
+ async def get_lights(self) -> List[LightModel]:
+ """Gets a list of lights and their current state."""
+ return self._lights
+
+ @kernel_function(name="ChangeState", description="Changes the state of the light")
+ async def change_state(
+ self,
+ change_state: LightModel
+ ) -> Optional[LightModel]:
+ """Changes the state of the light."""
+ for light in self._lights:
+ if light["id"] == change_state["id"]:
+ light["is_on"] = change_state.get("is_on", light["is_on"])
+ light["brightness"] = change_state.get("brightness", light["brightness"])
+ light["hex"] = change_state.get("hex", light["hex"])
+ return light
+ return None
+```
+
+The sample above shows how to override the function name and provide a description for the function. By default, the function name is the name of the function and the description is empty. If the function name is descriptive enough, you won't need a description, which will save you tokens. However, if the function behavior is not obvious from the name, you should provide a description for the AI.
+
+Because the LLMs are predominantly trained on Python code, it is recommended to use function names that follow the [Python naming conventions](https://peps.python.org/pep-0008/#function-and-variable-names), which means you rarely need to override the function names if you follow the conventions in your Python code.
+
+::: zone-end
+
## Next steps
+
Now that you know how to create a plugin, you can now learn how to use them with your AI agent. Depending on the type of functions you've added to your plugins, there are different patterns you should follow. For retrieval functions, refer to the [using retrieval functions](./using-data-retrieval-functions-for-rag.md) article. For task automation functions, refer to the [using task automation functions](./using-task-automation-functions.md) article.
> [!div class="nextstepaction"]
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml
index b4c46306..de9c5f57 100644
--- a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml
@@ -6,6 +6,8 @@
href: azure-cosmosdb-mongodb-connector.md
- name: Azure CosmosDB NoSQL connector
href: azure-cosmosdb-nosql-connector.md
+- name: Chroma connector
+ href: chroma-connector.md
- name: Couchbase connector
href: couchbase-connector.md
- name: Elasticsearch connector
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md
index 305ce851..a85ee0d4 100644
--- a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md
@@ -17,27 +17,59 @@ ms.service: semantic-kernel
The Azure AI Search Vector Store connector can be used to access and manage data in Azure AI Search. The connector has the following characteristics.
-| Feature Area | Support |
-|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------|
-| Collection maps to | Azure AI Search Index |
-| Supported key property types | string |
-| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>bool</li><li>DateTimeOffset</li><li>*and enumerables of each of these types*</li></ul> |
-| Supported vector property types | ReadOnlyMemory\<float\> |
-| Supported index types | <ul><li>Hnsw</li><li>Flat</li></ul> |
-| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
-| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
-| Supports multiple vectors in a record | Yes |
-| IsFilterable supported? | Yes |
-| IsFullTextSearchable supported? | Yes |
-| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) |
+::: zone pivot="programming-language-csharp"
+
+| Feature Area | Support |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Azure AI Search Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>bool</li><li>DateTimeOffset</li><li>*and enumerables of each of these types*</li></ul> |
+| Supported vector property types | ReadOnlyMemory\<float\> |
+| Supported index types | <ul><li>Hnsw</li><li>Flat</li></ul> |
+| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
+| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | Yes |
+| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) |
+::: zone-end
+::: zone pivot="programming-language-python"
+| Feature Area | Support |
+| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Azure AI Search Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>bool</li><li>DateTimeOffset</li><li>*and iterables of each of these types*</li></ul> |
+| Supported vector property types | list[float], list[int], ndarray |
+| Supported index types | <ul><li>Hnsw</li><li>Flat</li></ul> |
+| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanDistance</li><li>Hamming</li></ul> |
+| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | Yes |
+::: zone-end
+::: zone pivot="programming-language-java"
+| Feature Area | Support |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Azure AI Search Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>bool</li><li>DateTimeOffset</li><li>*and enumerables of each of these types*</li></ul> |
+| Supported vector property types | ReadOnlyMemory\<float\> |
+| Supported index types | <ul><li>Hnsw</li><li>Flat</li></ul> |
+| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
+| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | Yes |
+| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) |
+::: zone-end
## Limitations
Notable Azure AI Search connector functionality limitations.
-| Feature Area | Workaround |
-|--------------------------------------------------------------------------------------| -----------------------------------------------------------------------------------------------|
-| Configuring full text search analyzers during collection creation is not supported. | Use the Azure AI Search Client SDK directly for collection creation |
+| Feature Area | Workaround |
+| ----------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
+| Configuring full text search analyzers during collection creation is not supported. | Use the Azure AI Search Client SDK directly for collection creation |
::: zone pivot="programming-language-csharp"
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md
index 46fbbf96..c50ed06f 100644
--- a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md
@@ -13,30 +13,50 @@ ms.service: semantic-kernel
> [!WARNING]
> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release.
-::: zone pivot="programming-language-csharp"
-
## Overview
The Azure CosmosDB MongoDB Vector Store connector can be used to access and manage data in Azure CosmosDB MongoDB (vCore). The connector has the following characteristics.
-| Feature Area | Support |
-|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------|
-| Collection maps to | Azure Cosmos DB MongoDB (vCore) Collection + Index |
-| Supported key property types | string |
-| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>*and enumerables of each of these types*</li></ul> |
-| Supported vector property types | <ul><li>ReadOnlyMemory\<float\></li><li>ReadOnlyMemory\<double\></li></ul> |
-| Supported index types | <ul><li>Hnsw</li><li>IvfFlat</li></ul> |
-| Supported distance functions | <ul><li>CosineDistance</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
-| Supported filter clauses | <ul><li>EqualTo</li></ul> |
-| Supports multiple vectors in a record | Yes |
-| IsFilterable supported? | Yes |
-| IsFullTextSearchable supported? | No |
-| StoragePropertyName supported? | No, use BsonElementAttribute instead. [See here for more info.](#data-mapping) |
+::: zone pivot="programming-language-csharp"
+
+| Feature Area | Support |
+| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Azure Cosmos DB MongoDB (vCore) Collection + Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>*and enumerables of each of these types*</li></ul> |
+| Supported vector property types | <ul><li>ReadOnlyMemory\<float\></li><li>ReadOnlyMemory\<double\></li></ul> |
+| Supported index types | <ul><li>Hnsw</li><li>IvfFlat</li></ul> |
+| Supported distance functions | <ul><li>CosineDistance</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
+| Supported filter clauses | <ul><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | No |
+| StoragePropertyName supported? | No, use BsonElementAttribute instead. [See here for more info.](#data-mapping) |
+::: zone-end
+::: zone pivot="programming-language-python"
+| Feature Area | Support |
+| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Azure Cosmos DB MongoDB (vCore) Collection + Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>*and iterables of each of these types*</li></ul> |
+| Supported vector property types | <ul><li>list[float]</li><li>list[int]</li></ul><li>ndarray</li></ul> |
+| Supported index types | <ul><li>Hnsw</li><li>IvfFlat</li></ul> |
+| Supported distance functions | <ul><li>CosineDistance</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
+| Supported filter clauses | <ul><li>EqualTo</li></ul><ul><li>AnyTagsEqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | No |
+::: zone-end
+::: zone pivot="programming-language-java"
+More info coming soon.
+::: zone-end
## Limitations
This connector is compatible with Azure Cosmos DB MongoDB (vCore) and is *not* designed to be compatible with Azure Cosmos DB MongoDB (RU).
+::: zone pivot="programming-language-csharp"
+
## Getting started
Add the Azure CosmosDB MongoDB Vector Store connector NuGet package to your project.
@@ -165,9 +185,52 @@ public class Hotel
::: zone-end
::: zone pivot="programming-language-python"
-## Coming soon
+## Getting started
-More info coming soon.
+Add the Azure CosmosDB MongoDB Vector Store dependencies to your environment. Because the Azure CosmosDB MongoDB connector is built on the MongoDB Atlas connector and uses the same client as that one, you need to install with these extras:
+
+```bash
+pip install semantic-kernel[azure, mongo]
+```
+
+You can then create the vector store.
+
+```python
+from semantic_kernel.connectors.memory.azure_cosmos_db import AzureCosmosDBforMongoDBStore
+
+# If the right environment settings are set, namely AZURE_COSMOS_DB_MONGODB_CONNECTION_STRING and optionally AZURE_COSMOS_DB_MONGODB_DATABASE_NAME, this is enough to create the Store:
+store = AzureCosmosDBforMongoDBStore()
+```
+
+Alternatively, you can also pass in your own mongodb client if you want to have more control over the client construction:
+
+```python
+from pymongo import AsyncMongoClient
+from semantic_kernel.connectors.memory.azure_cosmos_db import AzureCosmosDBforMongoDBStore
+
+client = AsyncMongoClient(...)
+store = AzureCosmosDBforMongoDBStore(mongo_client=client)
+```
+
+When a client is passed in, Semantic Kernel will not close the connection for you, so you need to ensure to close it, for instance with a `async with` statement.
+
+You can also create a collection directly, without the store.
+
+```python
+from semantic_kernel.connectors.memory.azure_cosmos_db import AzureCosmosDBforMongoDBCollection
+
+# `hotel` is a class created with the @vectorstoremodel decorator
+collection = AzureCosmosDBforMongoDBCollection(
+ collection_name="my_collection",
+ data_model_type=hotel
+)
+```
+
+## Serialization
+
+Since the Azure CosmosDB for MongoDB connector needs a simple dict with the fields corresponding to the index as the input, the serialization is quite easy, it only uses a predetermined key `_id`, so we replace the key of the data model with that if it is not already `_id`.
+
+For more details on this concept see the [serialization documentation](./../serialization.md).
::: zone-end
::: zone pivot="programming-language-java"
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/chroma-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/chroma-connector.md
new file mode 100644
index 00000000..17dc15cc
--- /dev/null
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/chroma-connector.md
@@ -0,0 +1,104 @@
+---
+title: Using the Semantic Kernel Chroma Vector Store connector (Preview)
+description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in ChromaDB.
+zone_pivot_groups: programming-languages
+author: eavanvalkenburg
+ms.topic: conceptual
+ms.author: edvan
+ms.date: 02/26/2025
+ms.service: semantic-kernel
+---
+
+# Using the Chroma connector (Preview)
+
+> [!WARNING]
+> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release.
+
+::: zone pivot="programming-language-csharp"
+
+## Not supported
+
+Not supported.
+
+::: zone-end
+::: zone pivot="programming-language-python"
+
+## Overview
+
+The Chroma Vector Store connector can be used to access and manage data in Chroma. The connector has the
+following characteristics.
+
+| Feature Area | Support |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------- |
+| Collection maps to | Chroma collection |
+| Supported key property types | string |
+| Supported data property types | All types that are supported by System.Text.Json (either built-in or by using a custom converter) |
+| Supported vector property types | <ul><li>list[float]</li><li>list[int]</li><li>ndarray</li></ul> |
+| Supported index types | <ul><li>HNSW</li></ul> |
+| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanSquaredDistance</li></ul> |
+| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | No |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | Yes |
+
+## Limitations
+
+Notable Chroma connector functionality limitations.
+
+| Feature Area | Workaround |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------- |
+| Client-server mode | Use the client.HttpClient and pass the result to the `client` parameter, we do not support a AsyncHttpClient at this time |
+| Chroma Cloud | Unclear at this time, as Chroma Cloud is still in private preview |
+
+## Getting Started
+
+Add the Chroma Vector Store connector dependencies to your project.
+
+```bash
+pip install semantic-kernel[chroma]
+```
+
+You can then create the vector store.
+
+```python
+from semantic_kernel.connectors.memory.chroma import ChromaStore
+
+store = ChromaStore()
+```
+
+Alternatively, you can also pass in your own mongodb client if you want to have more control over the client construction:
+
+```python
+from chromadb import Client
+from semantic_kernel.connectors.memory.chroma import ChromaStore
+
+client = Client(...)
+store = ChromaStore(client=client)
+```
+
+You can also create a collection directly, without the store.
+
+```python
+from semantic_kernel.connectors.memory.chroma import ChromaCollection
+
+# `hotel` is a class created with the @vectorstoremodel decorator
+collection = ChromaCollection(
+ collection_name="my_collection",
+ data_model_type=hotel
+)
+```
+
+## Serialization
+
+The Chroma client returns both `get` and `search` results in tabular form, this means that there are between 3 and 5 lists being returned in a dict, the lists are 'keys', 'documents', 'embeddings', and optionally 'metadatas' and 'distances'. The Semantic Kernel Chroma connector will automatically convert this into a list of `dict` objects, which are then parsed back to your data model.
+
+It could be very interesting performance wise to do straight serialization from this format into a dataframe-like structure as that saves a lot of rebuilding of the data structure. This is not done for you, even when using container mode, you would have to specify this yourself, for more details on this concept see the [serialization documentation](./../serialization.md).
+
+::: zone-end
+::: zone pivot="programming-language-java"
+
+## Not supported
+
+Not supported.
+
+::: zone-end
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md
index 15b30d1d..ef0c44cc 100644
--- a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md
@@ -22,60 +22,59 @@ Semantic Kernel provides a number of out-of-the-box Vector Store integrations ma
::: zone pivot="programming-language-csharp"
-| Vector Store Connectors | C# | Uses officially supported SDK | Maintainer / Vendor |
-|------------------------------------------------------------|:---------------:|:---------------------------------:|:----------------------------------:|
-| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Cosmos DB MongoDB (vCore)](./azure-cosmosdb-mongodb-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Cosmos DB No SQL](./azure-cosmosdb-nosql-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Couchbase](./couchbase-connector.md) | ✅ | ✅ | Couchbase |
-| [Elasticsearch](./elasticsearch-connector.md) | ✅ | ✅ | Elastic |
-| Chroma | Planned | | |
-| [In-Memory](./inmemory-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
-| Milvus | Planned | | |
-| [MongoDB](./mongodb-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Pinecone](./pinecone-connector.md) | ✅ | ❌ | Microsoft Semantic Kernel Project |
-| [Postgres](./postgres-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Qdrant](./qdrant-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Redis](./redis-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| Sql Server | Planned | | |
-| [SQLite](./sqlite-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Volatile (In-Memory)](./volatile-connector.md) | Deprecated (use In-Memory) | N/A | Microsoft Semantic Kernel Project |
-| [Weaviate](./weaviate-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| Vector Store Connectors | C# | Uses officially supported SDK | Maintainer / Vendor |
+| ------------------------------------------------------------------ | :------------------------: | :---------------------------: | :-------------------------------: |
+| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Cosmos DB MongoDB (vCore)](./azure-cosmosdb-mongodb-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Cosmos DB No SQL](./azure-cosmosdb-nosql-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Couchbase](./couchbase-connector.md) | ✅ | ✅ | Couchbase |
+| [Elasticsearch](./elasticsearch-connector.md) | ✅ | ✅ | Elastic |
+| Chroma | Planned | | |
+| [In-Memory](./inmemory-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
+| Milvus | Planned | | |
+| [MongoDB](./mongodb-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Pinecone](./pinecone-connector.md) | ✅ | ❌ | Microsoft Semantic Kernel Project |
+| [Postgres](./postgres-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Qdrant](./qdrant-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Redis](./redis-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| Sql Server | Planned | | |
+| [SQLite](./sqlite-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Volatile (In-Memory)](./volatile-connector.md) | Deprecated (use In-Memory) | N/A | Microsoft Semantic Kernel Project |
+| [Weaviate](./weaviate-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
::: zone-end
::: zone pivot="programming-language-python"
-| Vector Store Connectors | Python | Uses officially supported SDK | Maintainer / Vendor |
-|------------------------------------------------------------|:---------------:|:----------------------------------:|:----------------------------------:|
-| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Cosmos DB MongoDB (vCore)](./azure-cosmosdb-mongodb-connector.md) | In Development | ✅ | Microsoft Semantic Kernel Project |
-| [Cosmos DB No SQL](./azure-cosmosdb-nosql-connector.md) | In Development | ✅ | Microsoft Semantic Kernel Project |
-| [Elasticsearch](./elasticsearch-connector.md) | Planned | | |
-| Chroma | Planned | | |
-| [In-Memory](./inmemory-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
-| Milvus | Planned | | |
-| [MongoDB](./mongodb-connector.md) | In Development | ✅ | Microsoft Semantic Kernel Project |
-| [Pinecone](./pinecone-connector.md) | In Development | ✅ | Microsoft Semantic Kernel Project |
-| [Postgres](./postgres-connector.md) | ✅ | | Microsoft Semantic Kernel Project |
-| [Qdrant](./qdrant-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| [Redis](./redis-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| Sql Server | Planned | | |
-| [SQLite](./sqlite-connector.md) | In Development | ✅ | Microsoft Semantic Kernel Project |
-| [Volatile (In-Memory)](./volatile-connector.md) | Deprecated (use In-Memory) | N/A | Microsoft Semantic Kernel Project |
-| [Weaviate](./weaviate-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
+| Vector Store Connectors | Python | Uses officially supported SDK | Maintainer / Vendor |
+| ------------------------------------------------------------------ | :-----: | :---------------------------: | :-------------------------------: |
+| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Cosmos DB MongoDB (vCore)](./azure-cosmosdb-mongodb-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Cosmos DB No SQL](./azure-cosmosdb-nosql-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Chroma](./chroma-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Elasticsearch](./elasticsearch-connector.md) | Planned | | |
+| Faiss | Planned | | |
+| [In-Memory](./inmemory-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
+| [MongoDB](./mongodb-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Pinecone](./pinecone-connector.md) | Planned | ✅ | Microsoft Semantic Kernel Project |
+| [Postgres](./postgres-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Qdrant](./qdrant-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| [Redis](./redis-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| SQL Server | Planned | ✅ | Microsoft Semantic Kernel Project |
+| SQLite | Planned | ✅ | Microsoft Semantic Kernel Project |
+| [Weaviate](./weaviate-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
::: zone-end
::: zone pivot="programming-language-java"
-| Vector Store Connectors | Java | Uses officially supported SDK | Maintainer / Vendor |
-|------------------------------------------------------------|:--------------:|:----------------------------------:|:----------------------------------:|
-| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| HSQLDB | Use [JDBC](./jdbc-connector.md) | ✅ | Microsoft Semantic Kernel Project |
-| [JDBC](./jdbc-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| MySQL | Use [JDBC](./jdbc-connector.md) | ✅ | Microsoft Semantic Kernel Project |
-| Postgres | Use [JDBC](./jdbc-connector.md) | | Microsoft Semantic Kernel Project |
-| [Redis](./redis-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
-| SQLite | Use [JDBC](./jdbc-connector.md) | ✅ | Microsoft Semantic Kernel Project |
-| [Volatile (In-Memory)](./volatile-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
+| Vector Store Connectors | Java | Uses officially supported SDK | Maintainer / Vendor |
+| ------------------------------------------------- | :-----------------------------: | :---------------------------: | :-------------------------------: |
+| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| HSQLDB | Use [JDBC](./jdbc-connector.md) | ✅ | Microsoft Semantic Kernel Project |
+| [JDBC](./jdbc-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| MySQL | Use [JDBC](./jdbc-connector.md) | ✅ | Microsoft Semantic Kernel Project |
+| Postgres | Use [JDBC](./jdbc-connector.md) | | Microsoft Semantic Kernel Project |
+| [Redis](./redis-connector.md) | ✅ | ✅ | Microsoft Semantic Kernel Project |
+| SQLite | Use [JDBC](./jdbc-connector.md) | ✅ | Microsoft Semantic Kernel Project |
+| [Volatile (In-Memory)](./volatile-connector.md) | ✅ | N/A | Microsoft Semantic Kernel Project |
::: zone-end
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/mongodb-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/mongodb-connector.md
index 6c406610..8671e1a3 100644
--- a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/mongodb-connector.md
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/mongodb-connector.md
@@ -13,25 +13,45 @@ ms.service: semantic-kernel
> [!WARNING]
> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release.
-::: zone pivot="programming-language-csharp"
-
## Overview
The MongoDB Vector Store connector can be used to access and manage data in MongoDB. The connector has the following characteristics.
-| Feature Area | Support |
-|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------|
-| Collection maps to | MongoDB Collection + Index |
-| Supported key property types | string |
-| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>*and enumerables of each of these types*</li></ul> |
-| Supported vector property types | <ul><li>ReadOnlyMemory\<float\></li><li>ReadOnlyMemory\<double\></li></ul> |
-| Supported index types | N/A |
-| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
-| Supported filter clauses | <ul><li>EqualTo</li></ul> |
-| Supports multiple vectors in a record | Yes |
-| IsFilterable supported? | Yes |
-| IsFullTextSearchable supported? | No |
-| StoragePropertyName supported? | No, use BsonElementAttribute instead. [See here for more info.](#data-mapping) |
+::: zone pivot="programming-language-csharp"
+
+| Feature Area | Support |
+| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | MongoDB Collection + Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>*and enumerables of each of these types*</li></ul> |
+| Supported vector property types | <ul><li>ReadOnlyMemory\<float\></li><li>ReadOnlyMemory\<double\></li></ul> |
+| Supported index types | N/A |
+| Supported distance functions | <ul><li>CosineSimilarity</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
+| Supported filter clauses | <ul><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | No |
+| StoragePropertyName supported? | No, use BsonElementAttribute instead. [See here for more info.](#data-mapping) |
+::: zone-end
+::: zone pivot="programming-language-python"
+
+| Feature Area | Support |
+| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | MongoDB Collection + Index |
+| Supported key property types | string |
+| Supported data property types | <ul><li>string</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>*and iterables of each of these types*</li></ul> |
+| Supported vector property types | <ul><li>list[float]</li><li>list[int]</li><li>ndarray</li></ul> |
+| Supported index types | <ul><li>Hnsw</li><li>IvfFlat</li></ul> |
+| Supported distance functions | <ul><li>CosineDistance</li><li>DotProductSimilarity</li><li>EuclideanDistance</li></ul> |
+| Supported filter clauses | <ul><li>EqualTo</li></ul><ul><li>AnyTagsEqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | No |
+::: zone-end
+::: zone pivot="programming-language-java"
+More info coming soon.
+::: zone-end
+::: zone pivot="programming-language-csharp"
## Getting started
@@ -136,9 +156,52 @@ public class Hotel
::: zone-end
::: zone pivot="programming-language-python"
-## Coming soon
+## Getting started
-More info coming soon.
+Add the MongoDB Atlas Vector Store dependencies to your environment. It needs the `pymongo` package which is included in the mongo extra: , you need to install with these extras:
+
+```bash
+pip install semantic-kernel[mongo]
+```
+
+You can then create the vector store.
+
+```python
+from semantic_kernel.connectors.memory.mongodb_atlas import MongoDBAtlasStore
+
+# If the right environment settings are set, namely MONGODB_ATLAS_CONNECTION_STRING and optionally MONGODB_ATLAS_DATABASE_NAME and MONGODB_ATLAS_INDEX_NAME, this is enough to create the Store:
+store = MongoDBAtlasStore()
+```
+
+Alternatively, you can also pass in your own mongodb client if you want to have more control over the client construction:
+
+```python
+from pymongo import AsyncMongoClient
+from semantic_kernel.connectors.memory.mongodb_atlas import MongoDBAtlasStore
+
+client = AsyncMongoClient(...)
+store = MongoDBAtlasStore(mongo_client=client)
+```
+
+When a client is passed in, Semantic Kernel will not close the connection for you, so you need to ensure to close it, for instance with a `async with` statement.
+
+You can also create a collection directly, without the store.
+
+```python
+from semantic_kernel.connectors.memory.mongodb_atlas import MongoDBAtlasCollection
+
+# `hotel` is a class created with the @vectorstoremodel decorator
+collection = MongoDBAtlasCollection(
+ collection_name="my_collection",
+ data_model_type=hotel
+)
+```
+
+## Serialization
+
+Since the MongoDB Atlas connector needs a simple dict with the fields corresponding to the index as the input, the serialization is quite easy, it only uses a predetermined key `_id`, so we replace the key of the data model with that if it is not already `_id`.
+
+For more details on this concept see the [serialization documentation](./../serialization.md).
::: zone-end
::: zone pivot="programming-language-java"
diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md
index 52a0178a..b3efc390 100644
--- a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md
+++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md
@@ -13,33 +13,51 @@ ms.service: semantic-kernel
> [!WARNING]
> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release.
-::: zone pivot="programming-language-csharp"
-
## Overview
The Weaviate Vector Store connector can be used to access and manage data in Weaviate. The connector has the following characteristics.
-| Feature Area | Support |
-|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------|
-| Collection maps to | Weaviate Collection |
-| Supported key property types | Guid |
-| Supported data property types | <ul><li>string</li><li>byte</li><li>short</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>DateTimeOffset</li><li>Guid</li><li>*and enumerables of each of these types*</li></ul> |
-| Supported vector property types | <ul><li>ReadOnlyMemory\<float\></li><li>ReadOnlyMemory\<double\></li></ul> |
-| Supported index types | <ul><li>Hnsw</li><li>Flat</li><li>Dynamic</li></ul> |
-| Supported distance functions | <ul><li>CosineDistance</li><li>NegativeDotProductSimilarity</li><li>EuclideanSquaredDistance</li><li>Hamming</li><li>ManhattanDistance</li></ul>|
-| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
-| Supports multiple vectors in a record | Yes |
-| IsFilterable supported? | Yes |
-| IsFullTextSearchable supported? | Yes |
-| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) |
+::: zone pivot="programming-language-csharp"
+
+| Feature Area | Support |
+| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Weaviate Collection |
+| Supported key property types | Guid |
+| Supported data property types | <ul><li>string</li><li>byte</li><li>short</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>DateTime</li><li>DateTimeOffset</li><li>Guid</li><li>*and enumerables of each of these types*</li></ul> |
+| Supported vector property types | <ul><li>ReadOnlyMemory\<float\></li><li>ReadOnlyMemory\<double\></li></ul> |
+| Supported index types | <ul><li>Hnsw</li><li>Flat</li><li>Dynamic</li></ul> |
+| Supported distance functions | <ul><li>CosineDistance</li><li>NegativeDotProductSimilarity</li><li>EuclideanSquaredDistance</li><li>Hamming</li><li>ManhattanDistance</li></ul> |
+| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | Yes |
+| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) |
+::: zone-end
+::: zone pivot="programming-language-python"
+| Feature Area | Support |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Collection maps to | Weaviate Collection |
+| Supported key property types | Guid |
+| Supported data property types | <ul><li>string</li><li>byte</li><li>short</li><li>int</li><li>long</li><li>double</li><li>float</li><li>decimal</li><li>bool</li><li>*and iterables of each of these types*</li></ul> |
+| Supported vector property types | <ul><li>list[float]</li><li>list[int]</li><li>ndarray</li></ul> |
+| Supported index types | <ul><li>Hnsw</li><li>Flat</li><li>Dynamic</li></ul> |
+| Supported distance functions | <ul><li>CosineDistance</li><li>NegativeDotProductSimilarity</li><li>EuclideanSquaredDistance</li><li>Hamming</li><li>ManhattanDistance</li></ul> |
+| Supported filter clauses | <ul><li>AnyTagEqualTo</li><li>EqualTo</li></ul> |
+| Supports multiple vectors in a record | Yes |
+| IsFilterable supported? | Yes |
+| IsFullTextSearchable supported? | Yes |
+::: zone-end
+::: zone pivot="programming-language-java"
+Coming soon.
+::: zone-end
## Limitations
Notable Weaviate connector functionality limitations.
-| Feature Area | Workaround |
-|------------------------------------------------------------------------| -----------------------------------------------------------------------------------------------|
-| Using the 'vector' property for single vector objects is not supported | Use of the 'vectors' property is supported instead. |
+| Feature Area | Workaround |
+| ---------------------------------------------------------------------- | --------------------------------------------------- |
+| Using the 'vector' property for single vector objects is not supported | Use of the 'vectors' property is supported instead. |
> [!WARNING]
> Weaviate requires collection names to start with an upper case letter. If you do not provide a collection name with an upper case letter, Weaviate will return an error when you try and create your collection. The error that you will see is `Cannot query field "mycollection" on type "GetObjectsObj". Did you mean "Mycollection"?` where `mycollection` is your collection name. In this example, if you change your collection name to `Mycollection` instead, this will fix the error.
@@ -181,9 +199,64 @@ public class Hotel
::: zone-end
::: zone pivot="programming-language-python"
-## Coming soon
+## Getting Started
-More info coming soon.
+Add the Weaviate Vector Store connector dependencies to your project.
+
+```bash
+pip install semantic-kernel[weaviate]
+```
+
+You can then create the vector store, it uses environment settings to connect:
+
+For using Weaviate Cloud:
+
+- url: WEAVIATE_URL
+- api_key: WEAVIATE_API_KEY
+
+For using Weaviate Local (i.e. Weaviate in a Docker container):
+
+- local_host: WEAVIATE_LOCAL_HOST
+- local_port: WEAVIATE_LOCAL_PORT
+- local_grpc_port: WEAVIATE_LOCAL_GRPC_PORT
+
+If you want to use embedded:
+
+- use_embed: WEAVIATE_USE_EMBED
+
+These should be set exclusively, so only one set of the above is present, otherwise it will raise an exception.
+
+```python
+from semantic_kernel.connectors.memory.weaviate import WeaviateStore
+
+store = WeaviateStore()
+```
+
+Alternatively, you can also pass in your own mongodb client if you want to have more control over the client construction:
+
+```python
+import weaviate
+from semantic_kernel.connectors.memory.weaviate import WeaviateStore
+
+client = weaviate.WeaviateAsyncClient(...)
+store = WeaviateStore(async_client=client)
+```
+
+You can also create a collection directly, without the store.
+
+```python
+from semantic_kernel.connectors.memory.weaviate import WeaviateCollection
+
+# `hotel` is a class created with the @vectorstoremodel decorator
+collection = WeaviateCollection(
+ collection_name="my_collection",
+ data_model_type=hotel
+)
+```
+
+## Serialization
+
+The Weaviate client returns it's own objects which are parsed and turned into dicts in the regular flow, for more details on this concept see the [serialization documentation](./../serialization.md).
::: zone-end
::: zone pivot="programming-language-java"