feat: initial version

TEN-framework · Dec 31, 2024 · ef24ca2 · ef24ca2
1 parent 4e6aae7
commit ef24ca2
Show file tree

Hide file tree

Showing 30 changed files with 584 additions and 153 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -10,4 +10,8 @@
     "editor.defaultFormatter": "ms-python.black-formatter"
   },
   "git.ignoreLimitWarning": true,
+  "pylint.ignorePatterns": [
+    "*/ten_runtime_python/**/*",
+    "/usr/lib/**/*"
+  ],
 }
diff --git a/agents/examples/default/manifest.json b/agents/examples/default/manifest.json
@@ -137,6 +137,11 @@
       "type": "extension",
       "name": "coze_python_async",
       "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_image_generate_tool",
+      "version": "=0.1.0"
     }
   ]
 }
diff --git a/agents/examples/default/property.json b/agents/examples/default/property.json
@@ -651,6 +651,282 @@
             ]
           }
         ]
+      },
+      {
+        "name": "story_teller",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": false
+            }
+          },
+          {
+            "type": "extension",
+            "name": "stt",
+            "addon": "deepgram_asr_python",
+            "extension_group": "stt",
+            "property": {
+              "api_key": "${env:DEEPGRAM_API_KEY}",
+              "language": "en-US",
+              "model": "nova-2",
+              "sample_rate": 16000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "fish_audio_tts",
+            "extension_group": "tts",
+            "property": {
+              "api_key": "${env:FISH_AUDIO_TTS_KEY}",
+              "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5",
+              "optimize_streaming_latency": true,
+              "request_timeout_seconds": 30,
+              "base_url": "https://api.fish.audio"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "openai_image_generate_tool",
+            "addon": "openai_image_generate_tool",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "stt"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "stt",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  },
+                  {
+                    "extension": "openai_image_generate_tool"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "openai_image_generate_tool",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
       }
     ],
     "log_level": 3

diff --git a/agents/ten_packages/extension/coze_python_async/extension.py b/agents/ten_packages/extension/coze_python_async/extension.py
@@ -168,9 +168,6 @@ async def on_call_chat_completion(
     ) -> any:
         raise RuntimeError("Not implemented")
 
-    async def on_generate_image(self, async_ten_env, prompt)->str:
-        raise RuntimeError("Not implemented")
-
     async def on_data_chat_completion(
         self, ten_env: AsyncTenEnv, **kargs: LLMDataCompletionArgs
     ) -> None:

diff --git a/agents/ten_packages/extension/dify_python/extension.py b/agents/ten_packages/extension/dify_python/extension.py
@@ -171,9 +171,6 @@ async def on_video_frame(
 
     async def on_call_chat_completion(self, async_ten_env, **kargs):
         raise NotImplementedError
-
-    async def on_generate_image(self, async_ten_env, prompt)->str:
-        return NotImplementedError
 
     async def on_tools_update(self, async_ten_env, tool):
         raise NotImplementedError

diff --git a/agents/ten_packages/extension/gemini_v2v_python/extension.py b/agents/ten_packages/extension/gemini_v2v_python/extension.py
@@ -24,6 +24,7 @@
     StatusCode,
     CmdResult,
     Data,
+    TenError,
 )
 from ten.audio_frame import AudioFrameDataFmt
 from ten_ai_base.const import CMD_PROPERTY_RESULT, CMD_TOOL_CALL
@@ -599,7 +600,7 @@ async def _handle_tool_call(self, func_calls: list[FunctionCall]) -> None:
             cmd: Cmd = Cmd.create(CMD_TOOL_CALL)
             cmd.set_property_string("name", name)
             cmd.set_property_from_json("arguments", json.dumps(arguments))
-            result: CmdResult = await self.ten_env.send_cmd(cmd)
+            [result, _] = await self.ten_env.send_cmd(cmd)
 
             func_response = FunctionResponse(
                 id=tool_call_id, name=name, response={"error": "Failed to call tool"}
@@ -739,8 +740,5 @@ async def _update_usage(self, usage: dict) -> None:
     async def on_call_chat_completion(self, async_ten_env, **kargs):
         raise NotImplementedError
 
-    async def on_generate_image(self, async_ten_env, prompt)->str:
-        return NotImplementedError
-
     async def on_data_chat_completion(self, async_ten_env, **kargs):
         raise NotImplementedError
diff --git a/agents/ten_packages/extension/glue_python_async/extension.py b/agents/ten_packages/extension/glue_python_async/extension.py
@@ -166,7 +166,7 @@ async def on_start(self, ten_env: AsyncTenEnv) -> None:
         self.memory = ChatMemory(self.config.max_history)
 
         if self.config.enable_storage:
-            result = await ten_env.send_cmd(Cmd.create("retrieve"))
+            [result, _] = await ten_env.send_cmd(Cmd.create("retrieve"))
             if result.get_status_code() == StatusCode.OK:
                 try:
                     history = json.loads(result.get_property_string("response"))
@@ -224,9 +224,6 @@ async def on_call_chat_completion(
     ) -> any:
         raise RuntimeError("Not implemented")
 
-    async def on_generate_image(self, async_ten_env, prompt) -> str:
-        return RuntimeError("Not implemented")
-
     async def on_data_chat_completion(
         self, ten_env: AsyncTenEnv, **kargs: LLMDataCompletionArgs
     ) -> None:
@@ -385,7 +382,7 @@ async def handle_tool_call(self, call: ToolCall) -> ToolCallResponse:
         cmd.set_property_from_json("arguments", call.function.arguments)
 
         # Send the command and handle the result through the future
-        result: CmdResult = await self.ten_env.send_cmd(cmd)
+        [result, _] = await self.ten_env.send_cmd(cmd)
         if result.get_status_code() == StatusCode.OK:
             tool_result: LLMToolResult = json.loads(
                 result.get_property_to_json(CMD_PROPERTY_RESULT)