Merge branch 'feature/add_pedestrian_detect_demo' into 'main'

Add pedestrian_detect into doorbell local demo

See merge request adf/esp-webrtc-solution!43

Author: TempoTian

components/esp_webrtc/include/esp_webrtc.h

@@ -186,6 +186,22 @@ int esp_webrtc_open(esp_webrtc_cfg_t *cfg, esp_webrtc_handle_t *rtc_handle);
  */
 int esp_webrtc_set_media_provider(esp_webrtc_handle_t rtc_handle, esp_webrtc_media_provider_t *provider);
 
+/**
+ * @brief  Set to disable auto capture for WebRTC
+ *
+ * @note  In default capture will start and stop internally by WebRTC
+ *        In some scenario capture is controlled by other module, capture is only one sink path of it
+ *        Users can call this API to disable auto control of capture
+ *
+ * @param[in]  rtc_handle       WebRTC handle
+ * @param[in]  no_auto_capture  Disable auto capture or not
+ *
+ * @return
+ *      - ESP_PEER_ERR_NONE         On success
+ *      - ESP_PEER_ERR_INVALID_ARG  Invalid argument
+ */
+int esp_webrtc_set_no_auto_capture(esp_webrtc_handle_t rtc_handle, bool no_auto_capture);
+
 /**
  * @brief  WebRTC set event handler
  *

components/esp_webrtc/src/esp_webrtc.c

@@ -81,6 +81,7 @@ typedef struct {
     esp_peer_signaling_ice_info_t ice_info;
     bool                          ice_info_loaded;
     bool                          signaling_connected;
+    bool                          no_auto_capture;
 
     uint8_t *aud_fifo;
     uint32_t aud_fifo_size;
@@ -201,7 +202,11 @@ static int stop_stream(webrtc_t *rtc)
         rtc->send_going = false;
         WAIT_FOR_BITS(PC_SEND_QUIT_BIT);
     }
-    esp_capture_stop(rtc->media_provider.capture);
+    if (rtc->no_auto_capture == false) {
+        esp_capture_stop(rtc->media_provider.capture);
+    } else {
+        esp_capture_sink_enable(rtc->capture_path, ESP_CAPTURE_RUN_MODE_DISABLE);
+    }
     av_render_reset(rtc->play_handle);
     return 0;
 }
@@ -790,6 +795,16 @@ int esp_webrtc_set_media_provider(esp_webrtc_handle_t handle, esp_webrtc_media_p
     return ESP_PEER_ERR_NONE;
 }
 
+int esp_webrtc_set_no_auto_capture(esp_webrtc_handle_t handle, bool no_auto_capture)
+{
+    if (handle == NULL) {
+        return ESP_PEER_ERR_INVALID_ARG;
+    }
+    webrtc_t *rtc = (webrtc_t *)handle;
+    rtc->no_auto_capture = no_auto_capture;
+    return ESP_PEER_ERR_NONE;
+}
+
 int esp_webrtc_start(esp_webrtc_handle_t handle)
 {
     if (handle == NULL) {

solutions/doorbell_local/README.md

@@ -3,7 +3,7 @@
 
 ## Overview
 
-This demo shows how to use `esp_webrtc` to build a local doorbell application. An ESP32 series board acts as an HTTPS signaling server.
+This demo shows how to use `esp_webrtc` to build a local doorbell application with optional AI-powered pedestrian detection. An ESP32 series board acts as an HTTPS signaling server and provides real-time video streaming, two-way audio communication, and intelligent motion detection capabilities.
 
 ## Hardware Requirements
 
@@ -21,13 +21,20 @@ This demo depends only on **ESP-IDF**. All other required components will be aut
 
 ### Configuration
 
-1. **Wi-Fi Settings**  
+1. **Wi-Fi Settings**
    Set your Wi-Fi SSID and password in [`settings.h`](main/settings.h).
 
-2. **Camera Configuration**  
+2. **Camera Configuration**
    If using a different camera model or resolution, update the corresponding settings in [`settings.h`](main/settings.h).
 
-3. **USB-JTAG Download (Optional)**  
+3. **Pedestrian Detection (Optional)**
+   To enable pedestrian detection functionality:
+   - Set `CONFIG_DOORBELL_SUPPORT_PEDESTRIAN_DETECT=y` in your configuration
+   - The detection resolution is defined by `DETECT_WIDTH` and `DETECT_HEIGHT` in [`settings.h`](main/settings.h)
+   - Detection frame rate is controlled by `DETECT_FPS` setting
+   - Requires additional memory and processing power
+
+4. **USB-JTAG Download (Optional)**
    If using USB-JTAG, uncomment the following line in [`sdkconfig.defaults.esp32p4`](sdkconfig.defaults.esp32p4):
    ```c
    CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y
@@ -66,35 +73,42 @@ Open **Chrome** or **Edge** and visit the printed URL. Since the site uses a sel
 
 Also, disable mDNS ICE candidates to ensure proper WebRTC connectivity:
 
-- **Chrome**: `chrome://flags/#enable-webrtc-hide-local-ips-with-mdns`  
+- **Chrome**: `chrome://flags/#enable-webrtc-hide-local-ips-with-mdns`
 - **Edge**: `edge://flags/#enable-webrtc-hide-local-ips-with-mdns`
 
-Disable the option: **WebRTC mDNS ICE candidates**  
+Disable the option: **WebRTC mDNS ICE candidates**
 Restart the browser for the change to take effect.
 
 ### Interactions
 
 After opening the page in the browser:
 
-1. **Open Door**  
-   - Click the **Door** icon.  
-   - The board will play a "Door is opened" tone.  
+1. **Open Door**
+   - Click the **Door** icon.
+   - The board will play a "Door is opened" tone.
    - The browser will display: `Receiving Door opened event`.
 
-2. **Calling (Ring Button)**  
-   - Press the **boot key** on the board.  
-   - The board plays ring music, and the browser shows **Accept Call** and **Deny Call** icons.  
-   - If accepted: starts **two-way audio** and **one-way video** (board → browser).  
-   - If denied: the call ends.  
+2. **Calling (Ring Button)**
+   - Press the **boot key** on the board.
+   - The board plays ring music, and the browser shows **Accept Call** and **Deny Call** icons.
+   - If accepted: starts **two-way audio** and **one-way video** (board → browser).
+   - If denied: the call ends.
    - To change the ring button, modify `DOOR_BELL_RING_BUTTON` in [`settings.h`](main/settings.h).
 
-3. **End Call / Cleanup**  
-   - In the browser: click the **Hangup** icon.  
+3. **Pedestrian Detection** *(if enabled)*
+   - The system automatically detects pedestrians using the camera feed.
+   - When a pedestrian is detected, a `PEDESTRIAN_DETECTED` event is sent to the browser.
+   - The browser displays detection events in a scrollable list with timestamps.
+   - Click the **Detection Events** button to view/clear the event history.
+   - Events are displayed in chronological order (newest first) with animation effects.
+
+4. **End Call / Cleanup**
+   - In the browser: click the **Hangup** icon.
    - On the board: run the `stop` command.
 
 ## Technical Details
 
-This demo use SSE to send instant message to browser and use POST to receive instant message from browser.
+This demo uses SSE (Server-Sent Events) to send instant messages to the browser and HTTP POST to receive instant messages from the browser.
 
 ### Enhancements in `esp_webrtc`
 
@@ -103,8 +117,21 @@ This demo use SSE to send instant message to browser and use POST to receive ins
 
 For a detailed call flow, refer to the [esp_webrtc connection flow](../../components/esp_webrtc/README.md#typical-call-sequence-of-esp_webrtc).
 
+### Pedestrian Detection Implementation
+
+When enabled, the pedestrian detection system:
+
+- **Camera Processing**: Captures frames from the camera at the configured resolution and frame rate
+- **AI Detection**: Uses ESP-DL (Deep Learning) library for real-time pedestrian detection
+- **Event Notification**: Sends `PEDESTRIAN_DETECTED` custom commands via WebRTC signaling
+- **Browser UI**: Displays detection events with timestamps in a scrollable, animated list
+
+The detection system operates independently of video calls and continues running in the background when enabled.
+
 ### Limitations
 
 - Only one peer can connect at a time.
 - A heartbeat timeout of 5 seconds is enforced.
 - If the peer leaves unexpectedly, wait a few seconds before reconnecting.
+- Pedestrian detection requires additional CPU and memory resources, which may affect overall system performance.
+- Detection accuracy depends on lighting conditions, camera quality, and pedestrian positioning.

solutions/doorbell_local/main/CMakeLists.txt

@@ -1,4 +1,11 @@
-idf_component_register(SRCS "webrtc.c"  "main.c" "board.c" "media_sys.c" "webrtc_http_server.c"
+
+set(srcs "webrtc.c"  "main.c" "board.c" "media_sys.c" "webrtc_http_server.c")
+
+if(CONFIG_DOORBELL_SUPPORT_PEDESTRIAN_DETECT)
+    list(APPEND srcs "pedestrain_detect.cpp")
+endif()
+
+idf_component_register(SRCS ${srcs}
                        EMBED_TXTFILES "ring.aac" "open.aac" "join.aac" "webrtc_test.html"
                        "certs/servercert.pem" "certs/prvtkey.pem"
                        INCLUDE_DIRS ".")

solutions/doorbell_local/main/Kconfig

@@ -0,0 +1,9 @@
+menu "DoorBell Local Demo"
+
+config DOORBELL_SUPPORT_PEDESTRIAN_DETECT
+   bool "Support pedestrian detection"
+   default n
+   depends on IDF_TARGET_ESP32P4 || IDF_TARGET_ESP32S3
+   help
+      Enable this option to enable pedestrian detection
+endmenu

solutions/doorbell_local/main/common.h

@@ -13,6 +13,7 @@
 #include "media_sys.h"
 #include "network.h"
 #include "sys_state.h"
+#include "esp_capture_sink.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -58,6 +59,30 @@ void send_cmd(char *cmd);
  */
 int close_data_channel(int idx);
 
+/**
+ * @brief  Get capture sink handle by index
+ */
+esp_capture_sink_handle_t get_detect_sink(void);
+
+/**
+ * @brief  pedestrian detection configuration
+ */
+typedef struct {
+   int                       (*detected)(esp_capture_rgn_t *rgn, void *ctx);
+   void                      *ctx;
+} pedestrian_detect_cfg_t;
+
+/**
+ * @brief  Start pedestrian detection
+ */
+int start_pedestrian_detection(pedestrian_detect_cfg_t *cfg);
+
+/**
+ * @brief  Stop pedestrian detection
+ */
+void stop_pedestrian_detection();
+
+
 #ifdef __cplusplus
 }
 #endif

solutions/doorbell_local/main/idf_component.yml

@@ -10,3 +10,7 @@ dependencies:
     version: "1.0.4"
     rules:
     - if: target in [esp32p4, esp32s3]
+  espressif/pedestrian_detect:
+    version: "^0.2.0"
+    rules:
+      - if: target in [esp32p4, esp32s3]

solutions/doorbell_local/main/main.c

@@ -193,7 +193,10 @@ static void thread_scheduler(const char *thread_name, media_lib_thread_cfg_t *sc
 #endif
     else if (strcmp(thread_name, "AUD_SRC") == 0) {
         schedule_cfg->priority = 15;
-    } else if (strcmp(thread_name, "pc_task") == 0) {
+    } else if (strcmp(thread_name, "detect") == 0) {
+        schedule_cfg->priority = 8;
+    }
+    else if (strcmp(thread_name, "pc_task") == 0) {
         schedule_cfg->stack_size = 25 * 1024;
         schedule_cfg->priority = 18;
         schedule_cfg->core_id = 1;
@@ -224,6 +227,8 @@ static char *get_network_ip(void)
     return media_lib_ipv4_ntoa(&ip_info.ip);
 }
 
+void sctp_show_details(bool enable);
+
 static int network_event_handler(bool connected)
 {
     if (connected) {
@@ -232,6 +237,7 @@ static int network_event_handler(bool connected)
             start_webrtc(NULL);
             ESP_LOGI(TAG, "Use browser to enter https://%s/webrtc/test for test", get_network_ip());
         });
+        // sctp_show_details(true);
     } else {
         stop_webrtc();
     }