trying to get SmallWebRTC PCC working

aconchillo · aconchillo · commit b1a563423904 · 2025-09-29T16:49:21.000-07:00
diff --git a/esp32-s3-box-3/CMakeLists.txt b/esp32-s3-box-3/CMakeLists.txt
@@ -9,15 +9,16 @@ if(NOT IDF_TARGET STREQUAL linux)
   add_compile_definitions(WIFI_PASSWORD="$ENV{WIFI_PASSWORD}")
 endif()
 
-if(NOT DEFINED ENV{PIPECAT_SMALLWEBRTC_URL})
-  message(FATAL_ERROR "Env variable PIPECAT_SMALLWEBRTC_URL must be set")
+if(NOT DEFINED ENV{PCC_AGENT_URL})
+  message(FATAL_ERROR "Env variable PCC_AGENT_URL must be set")
 endif()
 
 if(DEFINED ENV{LOG_DATACHANNEL_MESSAGES})
   add_compile_definitions(LOG_DATACHANNEL_MESSAGES="1")
 endif()
 
-add_compile_definitions(PIPECAT_SMALLWEBRTC_URL="$ENV{PIPECAT_SMALLWEBRTC_URL}")
+add_compile_definitions(PCC_AGENT_URL="$ENV{PCC_AGENT_URL}")
+add_compile_definitions(PCC_PUBLIC_API_KEY="$ENV{PCC_PUBLIC_API_KEY}")
 
 set(COMPONENTS src)
 set(EXTRA_COMPONENT_DIRS "src" "components/srtp" "components/peer" "components/esp-libopus")
diff --git a/esp32-s3-box-3/dependencies.lock b/esp32-s3-box-3/dependencies.lock
@@ -1,168 +1,10 @@
 dependencies:
-  espressif/button:
-    component_hash: f53face2ab21fa0ffaf4cf0f6e513d393f56df6586bb2ad1146120f03f19ee05
-    dependencies:
-    - name: espressif/cmake_utilities
-      registry_url: https://components.espressif.com
-      require: private
-      version: '*'
-    - name: idf
-      require: private
-      version: '>=4.0'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 4.1.3
-  espressif/cmake_utilities:
-    component_hash: 351350613ceafba240b761b4ea991e0f231ac7a9f59a9ee901f751bddc0bb18f
-    dependencies:
-    - name: idf
-      require: private
-      version: '>=4.1'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 0.5.3
-  espressif/esp-box-3:
-    component_hash: e064b82b6866f3d7c9d2165403bb1d0d426403b3f45e4be95c361f855f8843b4
-    dependencies:
-    - name: espressif/button
-      registry_url: https://components.espressif.com
-      require: public
-      version: ^4
-    - name: espressif/esp_codec_dev
-      registry_url: https://components.espressif.com
-      require: public
-      version: ~1.3.1
-    - name: espressif/esp_lcd_ili9341
-      registry_url: https://components.espressif.com
-      require: private
-      version: ^1
-    - name: espressif/esp_lcd_touch_gt911
-      registry_url: https://components.espressif.com
-      require: private
-      version: ^1
-    - name: espressif/esp_lcd_touch_tt21100
-      registry_url: https://components.espressif.com
-      require: private
-      version: ^1
-    - name: espressif/esp_lvgl_port
-      registry_url: https://components.espressif.com
-      require: public
-      version: ^2
-    - name: espressif/icm42670
-      registry_url: https://components.espressif.com
-      require: public
-      version: ^2.0.1
-    - name: idf
-      require: private
-      version: '>=5.3'
-    source:
-      registry_url: https://components.espressif.com/
-      type: service
-    targets:
-    - esp32s3
-    version: 3.0.0~1
-  espressif/esp_codec_dev:
-    component_hash: c71e2d13dad6fc41561590dd88dbc45c79e3f4ef48d5ee3575c60e8b6c8e79d5
-    dependencies:
-    - name: idf
-      require: private
-      version: '>=4.0'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 1.3.5
-  espressif/esp_lcd_ili9341:
-    component_hash: 31f1b793aa2110dd2ae071c21ccbff0a4eb20d9a4ee40b6294c0dc0ad9552c4e
-    dependencies:
-    - name: idf
-      require: private
-      version: '>=4.4'
-    - name: espressif/cmake_utilities
-      registry_url: https://components.espressif.com
-      require: private
-      version: 0.*
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 1.2.0
-  espressif/esp_lcd_touch:
-    component_hash: 779b4ba2464a3ae85681e4b860caa5fdc35801458c23f3039ee761bae7f442a4
-    dependencies:
-    - name: idf
-      require: private
-      version: '>=4.4.2'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 1.1.2
-  espressif/esp_lcd_touch_gt911:
-    component_hash: acc1c184358aa29ef72506f618c9c76a8cc2bf12af38a2bff3d44d84f3a08857
-    dependencies:
-    - name: espressif/esp_lcd_touch
-      registry_url: https://components.espressif.com
-      require: public
-      version: ^1.1.0
-    - name: idf
-      require: private
-      version: '>=4.4.2'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 1.1.3
-  espressif/esp_lcd_touch_tt21100:
-    component_hash: 31894c5572927cb2dc65e5087572c4bd71d572fda3ef7e3320cb735af228c423
-    dependencies:
-    - name: espressif/esp_lcd_touch
-      registry_url: https://components.espressif.com
-      require: public
-      version: ^1.1.0
-    - name: idf
-      require: private
-      version: '>=4.4.2'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 1.1.1
-  espressif/esp_lvgl_port:
-    component_hash: e720c95cf0667554a204591bb5fade4655fb2990465557041200fa44b5bc7556
-    dependencies:
-    - name: idf
-      require: private
-      version: '>=4.4'
-    - name: lvgl/lvgl
-      registry_url: https://components.espressif.com
-      require: public
-      version: '>=8,<10'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 2.6.0
-  espressif/icm42670:
-    component_hash: 28b56e174f75c70037f5208aaed6c3789f0d243500d975519584bf9dc8c0836c
-    dependencies:
-    - name: idf
-      require: private
-      version: '>=5.2'
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 2.0.2
   idf:
     source:
       type: idf
     version: 5.4.2
-  lvgl/lvgl:
-    component_hash: b702d642e03e95928046d5c6726558e6444e112420c77efa5fdb6650b0a13c5d
-    dependencies: []
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 9.3.0
 direct_dependencies:
-- espressif/esp-box-3
 - idf
 manifest_hash: 55f6a702498de8e87201da504a38c5c958e2094ec832c4f8d6d368369d5d0625
-target: esp32s3
+target: linux
 version: 2.0.0
diff --git a/esp32-s3-box-3/deps/libpeer b/esp32-s3-box-3/deps/libpeer
@@ -1 +1 @@
-Subproject commit 4b3f567059a3a8a91ea95ed0fcdf672149fdc8fb
+Subproject commit 9319aa434cb9e893faed0293ba9d2a21eca59c8b
diff --git a/esp32-s3-box-3/src/http.cpp b/esp32-s3-box-3/src/http.cpp
@@ -70,11 +70,82 @@ static esp_err_t http_event_handler(esp_http_client_event_t *evt) {
   return ESP_OK;
 }
 
-void pipecat_http_request(char *offer, char *answer) {
+cJSON* pipecat_cloud_start_agent() {
   esp_http_client_config_t config;
   memset(&config, 0, sizeof(esp_http_client_config_t));
 
-  config.url = PIPECAT_SMALLWEBRTC_URL;
+  char *response = (char *)malloc(MAX_HTTP_OUTPUT_BUFFER + 1);
+  memset(response, 0, MAX_HTTP_OUTPUT_BUFFER + 1);
+
+  char start_url[1024];
+  sprintf(start_url, "%s/start", PCC_AGENT_URL);
+
+  config.url = start_url;
+  config.event_handler = http_event_handler;
+  config.timeout_ms = HTTP_TIMEOUT_MS;
+  config.user_data = response;
+
+  ESP_LOGI(LOG_TAG, "Connecting to %s", config.url);
+
+  cJSON *j_body = cJSON_CreateObject();
+  if (j_body == NULL) {
+    ESP_LOGE(LOG_TAG, "Unable to create start agent body");
+    return NULL;
+  }
+  if (cJSON_AddBoolToObject(j_body, "createDailyRoom", false) == NULL) {
+    cJSON_Delete(j_body);
+    ESP_LOGE(LOG_TAG, "Unable to create start agent body");
+    return NULL;
+  }
+
+  char *j_body_str = cJSON_Print(j_body);
+
+  cJSON_Delete(j_body);
+
+  char bearer_token[1024];
+  sprintf(bearer_token, "Bearer %s", PCC_PUBLIC_API_KEY);
+
+  esp_http_client_handle_t client = esp_http_client_init(&config);
+  esp_http_client_set_method(client, HTTP_METHOD_POST);
+  esp_http_client_set_header(client, "Authorization", bearer_token);
+  esp_http_client_set_header(client, "Content-Type", "application/json");
+  esp_http_client_set_post_field(client, j_body_str, strlen(j_body_str));
+
+  esp_err_t err = esp_http_client_perform(client);
+  int status_code = esp_http_client_get_status_code(client);
+  if (err != ESP_OK || status_code != 200) {
+    ESP_LOGE(LOG_TAG, "Error perform http request %s (status %d)",
+             esp_err_to_name(err), status_code);
+#ifndef LINUX_BUILD
+    esp_restart();
+#endif
+  }
+
+  cJSON *j_response = cJSON_Parse((const char *)response);
+  if (j_response == NULL) {
+    ESP_LOGE(LOG_TAG, "Error parsing HTTP response");
+#ifndef LINUX_BUILD
+    esp_restart();
+#endif
+  }
+
+  ESP_LOGI(LOG_TAG, "RESPONSE\n%s", response);
+
+  free(response);
+
+  esp_http_client_cleanup(client);
+
+  return j_response;
+}
+
+void pipecat_http_request(char *session_id, char *offer, char *answer) {
+  esp_http_client_config_t config;
+  memset(&config, 0, sizeof(esp_http_client_config_t));
+
+  char offer_url[1024];
+  sprintf(offer_url, "%s/sessions/%s/api/offer", PCC_AGENT_URL, session_id);
+
+  config.url = offer_url;
   config.event_handler = http_event_handler;
   config.timeout_ms = HTTP_TIMEOUT_MS;
   config.user_data = answer;
@@ -97,14 +168,18 @@ void pipecat_http_request(char *offer, char *answer) {
     return;
   }
 
-  ESP_LOGD(LOG_TAG, "OFFER\n%s", offer);
+  ESP_LOGI(LOG_TAG, "OFFER\n%s", offer);
 
   char *j_offer_str = cJSON_Print(j_offer);
 
   cJSON_Delete(j_offer);
 
+  char bearer_token[1024];
+  sprintf(bearer_token, "Bearer %s", PCC_PUBLIC_API_KEY);
+
   esp_http_client_handle_t client = esp_http_client_init(&config);
   esp_http_client_set_method(client, HTTP_METHOD_POST);
+  esp_http_client_set_header(client, "Authorization", bearer_token);
   esp_http_client_set_header(client, "Content-Type", "application/json");
   esp_http_client_set_post_field(client, j_offer_str, strlen(j_offer_str));
 
@@ -137,7 +212,7 @@ void pipecat_http_request(char *offer, char *answer) {
   memset(answer, 0, MAX_HTTP_OUTPUT_BUFFER + 1);
   memcpy(answer, j_answer->valuestring, strlen(j_answer->valuestring));
 
-  ESP_LOGD(LOG_TAG, "ANSWER\n%s", answer);
+  ESP_LOGI(LOG_TAG, "ANSWER\n%s", answer);
 
   cJSON_Delete(j_response);
 
diff --git a/esp32-s3-box-3/src/main.cpp b/esp32-s3-box-3/src/main.cpp
@@ -35,7 +35,7 @@ extern "C" void app_main(void) {
 int main(void) {
   ESP_ERROR_CHECK(esp_event_loop_create_default());
   peer_init();
-  pipecat_webrtc();
+  pipecat_init_webrtc();
 
   while (1) {
     pipecat_webrtc_loop();
diff --git a/esp32-s3-box-3/src/main.h b/esp32-s3-box-3/src/main.h
@@ -1,4 +1,5 @@
 #include <peer.h>
+#include <cJSON.h>
 
 #define LOG_TAG "pipecat"
 #define MAX_HTTP_OUTPUT_BUFFER 4096
@@ -18,7 +19,8 @@ extern void pipecat_audio_decode(uint8_t *data, size_t size);
 // WebRTC / Signalling
 extern void pipecat_init_webrtc();
 extern void pipecat_webrtc_loop();
-extern void pipecat_http_request(char *offer, char *answer);
+extern cJSON* pipecat_cloud_start_agent();
+extern void pipecat_http_request(char *session_id, char *offer, char *answer);
 
 // RTVI
 typedef struct {
diff --git a/esp32-s3-box-3/src/webrtc.cpp b/esp32-s3-box-3/src/webrtc.cpp
@@ -3,6 +3,7 @@
 #include <opus.h>
 #endif
 
+#include <cJSON.h>
 #include <esp_event.h>
 #include <esp_log.h>
 #include <string.h>
@@ -28,7 +29,7 @@ static void pipecat_ondatachannel_onmessage_task(char *msg, size_t len,
 #ifdef LOG_DATACHANNEL_MESSAGES
   ESP_LOGI(LOG_TAG, "DataChannel Message: %s", msg);
 #endif
-  pipecat_rtvi_handle_message(msg);
+  //  pipecat_rtvi_handle_message(msg);
 }
 
 static void pipecat_ondatachannel_onopen_task(void *userdata) {
@@ -64,17 +65,38 @@ static void pipecat_onconnectionstatechange_task(PeerConnectionState state,
 }
 
 static void pipecat_on_icecandidate_task(char *description, void *user_data) {
+  cJSON *j_response = pipecat_cloud_start_agent();
+  if (j_response == NULL) {
+    ESP_LOGE(LOG_TAG, "Unable to get response from start agent");
+#ifndef LINUX_BUILD
+    esp_restart();
+#endif
+  }
+
+  cJSON *j_session_id = cJSON_GetObjectItem(j_response, "sessionId");
+  if (j_session_id == NULL) {
+    ESP_LOGE(LOG_TAG, "Unable to find `sessionId` field in response");
+#ifndef LINUX_BUILD
+    esp_restart();
+#endif
+  }
+
   char *local_buffer = (char *)malloc(MAX_HTTP_OUTPUT_BUFFER + 1);
   memset(local_buffer, 0, MAX_HTTP_OUTPUT_BUFFER + 1);
-  pipecat_http_request(description, local_buffer);
-  peer_connection_set_remote_description(peer_connection, local_buffer,
-                                         SDP_TYPE_ANSWER);
+  pipecat_http_request(j_session_id->valuestring, description, local_buffer);
+
+  peer_connection_set_remote_description(peer_connection, local_buffer, SDP_TYPE_ANSWER);
   free(local_buffer);
 }
 
 void pipecat_init_webrtc() {
   PeerConfiguration peer_connection_config = {
-      .ice_servers = {},
+      .ice_servers = {
+        // { .urls = "turn:turn.cloudflare.com:3478",
+        //   .username = "g0d469a73c42effe51f1a50e85148730acdc2a38244472ab2b99bb556b139cc2",
+        //   .credential = "ff4634c0424b1698e0313da9a3cc04ba4ed3c7847a6863692f10b760f8aad9ca"},
+        { .urls = "stun:stun.l.google.com:19302", .username = NULL, .credential = NULL },
+      },
       .audio_codec = CODEC_OPUS,
       .video_codec = CODEC_NONE,
       .datachannel = DATA_CHANNEL_STRING,
