feat: init success

agents/examples/default/property.json

@@ -729,6 +729,13 @@
             "extension_group": "transcriber",
             "property": {}
           },
+          {
+            "type": "extension",
+            "name": "message_collector2",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
           {
             "type": "extension",
             "name": "weatherapi_tool_python",
@@ -843,7 +850,7 @@
                 "name": "raw_text_data",
                 "dest": [
                   {
-                    "extension": "message_collector"
+                    "extension": "message_collector2"
                   }
                 ]
               }
@@ -862,6 +869,19 @@
               }
             ]
           },
+          {
+            "extension": "message_collector2",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
           {
             "extension": "tts",
             "cmd": [

agents/ten_packages/extension/openai_image_generate_tool/extension.py

@@ -64,6 +64,6 @@ async def run_tool(self, ten_env: AsyncTenEnv, name: str, args: dict) -> LLMTool
                 ten_env.log_info(f"Generated image: {response_url}")
                 result = LLMToolResultDirectRawResponse(
                     type="direct_raw_response",
-                    content=json.dumps({"data":{"image_url": response_url}, "type": "image_url"}),
+                    content={"data":{"image_url": response_url}, "type": "image_url"},
                 )
                 return result

playground/src/components/Chat/MessageList.tsx

@@ -8,7 +8,7 @@ import {
   isRagGraph,
 } from "@/common"
 import { Bot } from "lucide-react"
-import { EMessageType, type IChatItem } from "@/types"
+import { EMessageDataType, EMessageType, type IChatItem } from "@/types"
 import { Avatar, AvatarFallback } from "@/components/ui/avatar"
 import { cn } from "@/lib/utils"
 
@@ -50,7 +50,11 @@ export function MessageItem(props: { data: IChatItem }) {
         </Avatar>
         }
         <div className="max-w-[80%] rounded-lg bg-secondary p-2 text-secondary-foreground">
-          <p>{data.text}</p>
+          {data.data_type === EMessageDataType.IMAGE ? (
+            <img src={data.text} alt="chat" className="w-full" />
+          ) : (
+            <p>{data.text}</p>
+          )}
         </div>
       </div>
     </>

playground/src/components/Dynamic/RTCCard.tsx

@@ -4,7 +4,7 @@ import * as React from "react"
 import { cn } from "@/lib/utils"
 import { ICameraVideoTrack, ILocalVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng"
 import { useAppSelector, useAppDispatch, VOICE_OPTIONS, VideoSourceType } from "@/common"
-import { ITextItem, EMessageType } from "@/types"
+import { ITextItem, EMessageType, IChatItem } from "@/types"
 import { rtcManager, IUserTracks, IRtcUser } from "@/manager"
 import {
   setRoomConnected,
@@ -98,20 +98,11 @@ export default function RTCCard(props: { className?: string }) {
     }
   }
 
-  const onTextChanged = (text: ITextItem) => {
+  const onTextChanged = (text: IChatItem) => {
     console.log("[rtc] onTextChanged", text)
-    if (text.dataType == "transcribe") {
-      const isAgent = Number(text.uid) != Number(userId)
-      dispatch(
-        addChatItem({
-          userId: text.uid,
-          text: text.text,
-          type: isAgent ? EMessageType.AGENT : EMessageType.USER,
-          isFinal: text.isFinal,
-          time: text.time,
-        }),
-      )
-    }
+    dispatch(
+      addChatItem(text),
+    )
   }
 
   const onVoiceChange = (value: any) => {
@@ -138,7 +129,7 @@ export default function RTCCard(props: { className?: string }) {
           {/* -- You */}
           <div className="w-full space-y-2 px-2">
             <MicrophoneBlock audioTrack={audioTrack} />
-            <VideoBlock 
+            <VideoBlock
               cameraTrack={videoTrack}
               screenTrack={screenTrack}
               videoSourceType={videoSourceType}

playground/src/manager/rtc/rtc.ts

@@ -6,7 +6,7 @@ import AgoraRTC, {
   IRemoteAudioTrack,
   UID,
 } from "agora-rtc-sdk-ng";
-import { ITextItem } from "@/types";
+import { EMessageDataType, EMessageType, IChatItem, ITextItem } from "@/types";
 import { AGEventEmitter } from "../events";
 import { RtcEvents, IUserTracks } from "./types";
 import { apiGenAgoraData, VideoSourceType } from "@/common";
@@ -26,6 +26,7 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
   localTracks: IUserTracks;
   appId: string | null = null;
   token: string | null = null;
+  userId: number | null = null;
 
   constructor() {
     super();
@@ -45,6 +46,7 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
       const { appId, token } = data;
       this.appId = appId;
       this.token = token;
+      this.userId = userId;
       await this.client?.join(appId, channel, token, userId);
       this._joined = true;
     }
@@ -231,27 +233,28 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
         const { stream_id, is_final, text, text_ts, data_type } = JSON.parse(
           atob(completeMessage)
         );
-        let textItem: ITextItem;
+        const isAgent = Number(stream_id) != Number(this.userId)
+        let textItem: IChatItem = {
+          type: isAgent ? EMessageType.AGENT : EMessageType.USER,
+          time: text_ts,
+          text: text,
+          data_type: EMessageDataType.TEXT,
+          userId: stream_id,
+          isFinal: is_final,
+        };;
 
         if (data_type === "raw") {
-          textItem = {
-            uid: `${stream_id}`,
-            time: text_ts,
-            dataType: "image_url",
-            text: text,
-            isFinal: is_final,
+          let { data, type } = JSON.parse(text);
+          if (type === "image_url") {
+            textItem = {
+              ...textItem,
+              data_type: EMessageDataType.IMAGE,
+              text: data.image_url,
+            };
           }
-        } else {
-          textItem = {
-            uid: `${stream_id}`,
-            time: text_ts,
-            dataType: "transcribe",
-            text: text,
-            isFinal: is_final,
-          };
         }
 
-        if (text.trim().length > 0 && textItem) {
+        if (text.trim().length > 0) {
           this.emit("textChanged", textItem);
         }
 

playground/src/manager/rtc/types.ts

@@ -7,7 +7,7 @@ import {
   NetworkQuality,
   ILocalVideoTrack,
 } from "agora-rtc-sdk-ng"
-import { ITextItem } from "@/types"
+import { IChatItem, ITextItem } from "@/types"
 
 export interface IRtcUser extends IUserTracks {
   userId: UID
@@ -17,7 +17,7 @@ export interface RtcEvents {
   remoteUserChanged: (user: IRtcUser) => void
   localTracksChanged: (tracks: IUserTracks) => void
   networkQuality: (quality: NetworkQuality) => void
-  textChanged: (text: ITextItem) => void
+  textChanged: (text: IChatItem) => void
 }
 
 export interface IUserTracks {

playground/src/types/index.ts

@@ -24,10 +24,16 @@ export enum EMessageType {
   USER = "user",
 }
 
+export enum EMessageDataType {
+  TEXT = "text",
+  IMAGE = "image",
+}
+
 export interface IChatItem {
   userId: number | string;
   userName?: string;
   text: string;
+  data_type: EMessageDataType;
   type: EMessageType;
   isFinal?: boolean;
   time: number;