Audio works on esp32-m5stack-atoms3r

Sean-Der · Sean-Der · commit 973b72b7c2ec · 2025-07-14T14:25:20.000-04:00
Need to implement buffering
diff --git a/esp32-m5stack-atoms3r/dependencies.lock b/esp32-m5stack-atoms3r/dependencies.lock
@@ -1,29 +1,21 @@
 dependencies:
-  idf:
-    source:
-      type: idf
-    version: 5.4.1
-  m5stack/m5gfx:
-    component_hash: a44875eb6d6577bddcf1669ad5dec35e1afa425b7ca8e99af6b38505f37840e6
-    dependencies: []
-    source:
-      registry_url: https://components.espressif.com
-      type: service
-    version: 0.2.9
-  m5stack/m5unified:
-    component_hash: 9851fefc25bc05f8cdeadd4c6118c3300cfefcebf7e8c748dc9ccc2811100de6
+  espressif/esp_codec_dev:
+    component_hash: c71e2d13dad6fc41561590dd88dbc45c79e3f4ef48d5ee3575c60e8b6c8e79d5
     dependencies:
-    - name: m5stack/m5gfx
-      registry_url: https://components.espressif.com
+    - name: idf
       require: private
-      version: ^0.2.6
+      version: '>=4.0'
     source:
       registry_url: https://components.espressif.com/
       type: service
-    version: 0.2.4
+    version: 1.3.5
+  idf:
+    source:
+      type: idf
+    version: 5.4.1
 direct_dependencies:
+- espressif/esp_codec_dev
 - idf
-- m5stack/m5unified
-manifest_hash: 6c856b0c15014cd213a8417ee6353debbf5a3f71b049e54dff1b99b6268cff03
+manifest_hash: c74d03622862bf2a85301c12f34d445e0d5bfc643f05f1b14820503605dd0b55
 target: esp32s3
 version: 2.0.0
diff --git a/esp32-m5stack-atoms3r/src/idf_component.yml b/esp32-m5stack-atoms3r/src/idf_component.yml
@@ -1,4 +1,4 @@
 dependencies:
   idf:
-    version: ">=4.1.0"
-  m5stack/m5unified: "^0.2.4"
+    version: '>=4.1.0'
+  espressif/esp_codec_dev: ^1.3.5
diff --git a/esp32-m5stack-atoms3r/src/main.cpp b/esp32-m5stack-atoms3r/src/main.cpp
@@ -16,15 +16,6 @@ extern "C" void app_main(void) {
   }
   ESP_ERROR_CHECK(ret);
 
-  auto cfg = M5.config();
-  cfg.external_speaker.atomic_echo = true;
-  M5.begin(cfg);
-
-  M5.Display.setBrightness(70);
-  M5.Display.setTextSize(1.5);
-  M5.Display.fillScreen(BLACK);
-  M5.Display.println("Pipecat ESP32 client initialized\n");
-
   ESP_ERROR_CHECK(esp_event_loop_create_default());
   peer_init();
   pipecat_init_audio_capture();
diff --git a/esp32-m5stack-atoms3r/src/main.h b/esp32-m5stack-atoms3r/src/main.h
@@ -1,9 +1,5 @@
 #include <peer.h>
 
-#ifndef LINUX_BUILD
-#include <M5Unified.h>
-#endif
-
 #define LOG_TAG "pipecat"
 #define MAX_HTTP_OUTPUT_BUFFER 4096
 #define HTTP_TIMEOUT_MS 10000
diff --git a/esp32-m5stack-atoms3r/src/media.cpp b/esp32-m5stack-atoms3r/src/media.cpp
@@ -11,6 +11,14 @@
 #include "esp_check.h"
 #include "esp_log.h"
 #include "esp_heap_caps.h"
+
+#include <esp_codec_dev.h>
+#include <esp_codec_dev_defaults.h>
+#include <esp_log.h>
+
+#include "driver/i2c_master.h"
+#include <driver/i2s_std.h>
+
 #include "main.h"
 
 #define SAMPLE_RATE (16000)
@@ -21,6 +29,168 @@
 #define OPUS_ENCODER_BITRATE 30000
 #define OPUS_ENCODER_COMPLEXITY 0
 
+i2c_master_bus_handle_t i2c_bus;
+esp_codec_dev_handle_t audio_dev;
+
+void configure_pi4ioe(void) {
+    i2c_master_dev_handle_t i2c_device;
+    i2c_device_config_t i2c_device_cfg = {
+        .dev_addr_length = I2C_ADDR_BIT_LEN_7,
+        .device_address = 0x43, // PI4IOE Address
+        .scl_speed_hz = 400 * 1000,
+        .scl_wait_us = 0,
+        .flags = {
+            .disable_ack_check = 0,
+        },
+    };
+    ESP_ERROR_CHECK(i2c_master_bus_add_device(i2c_bus, &i2c_device_cfg, &i2c_device));
+
+    auto writeRegister = [=](uint8_t reg, uint8_t value) {
+        uint8_t buffer[2] = {reg, value};
+        ESP_ERROR_CHECK(i2c_master_transmit(i2c_device, buffer, 2, 100));
+    };
+
+    writeRegister(0x07, 0x00); // Set to high-impedance
+    writeRegister(0x0D, 0xFF); // Enable pull-up
+    writeRegister(0x03, 0x6E); // Set input=0, output=1
+    writeRegister(0x05, 0xFF); // Unmute speaker
+    i2c_master_bus_rm_device(i2c_device);
+}
+
+void configure_es8311(void) {
+    i2s_chan_config_t chan_cfg = {
+        .id = I2S_NUM_0,
+        .role = I2S_ROLE_MASTER,
+        .dma_desc_num = 6,
+        .dma_frame_num = 240,
+        .auto_clear_after_cb = true,
+        .auto_clear_before_cb = false,
+        .intr_priority = 0,
+    };
+
+    i2s_chan_handle_t tx_handle = nullptr, rx_handle = nullptr;
+    ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &tx_handle, &rx_handle));
+
+    i2s_std_config_t std_cfg = {
+        .clk_cfg = {
+            .sample_rate_hz = SAMPLE_RATE,
+            .clk_src = I2S_CLK_SRC_DEFAULT,
+            .ext_clk_freq_hz = 0,
+            .mclk_multiple = I2S_MCLK_MULTIPLE_256,
+        },
+        .slot_cfg = {
+            .data_bit_width = I2S_DATA_BIT_WIDTH_16BIT,
+            .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO,
+            .slot_mode = I2S_SLOT_MODE_STEREO,
+            .slot_mask = I2S_STD_SLOT_BOTH,
+            .ws_width = I2S_DATA_BIT_WIDTH_16BIT,
+            .ws_pol = false,
+            .bit_shift = true,
+            .left_align = true,
+            .big_endian = false,
+            .bit_order_lsb = false
+        },
+        .gpio_cfg = {
+            .mclk = GPIO_NUM_NC,
+            .bclk = GPIO_NUM_8,
+            .ws = GPIO_NUM_6,
+            .dout = GPIO_NUM_5,
+            .din = GPIO_NUM_7,
+            .invert_flags = {
+                .mclk_inv = false,
+                .bclk_inv = false,
+                .ws_inv = false
+            }
+        }
+    };
+
+    ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle, &std_cfg));
+    ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle, &std_cfg));
+
+    audio_codec_i2s_cfg_t i2s_cfg = {
+        .port = I2S_NUM_0,
+        .rx_handle = rx_handle,
+        .tx_handle = tx_handle,
+    };
+    audio_codec_i2c_cfg_t i2c_cfg = {
+        .port = I2C_NUM_1,
+        .addr = ES8311_CODEC_DEFAULT_ADDR,
+        .bus_handle = i2c_bus,
+    };
+    es8311_codec_cfg_t es8311_cfg = {
+        .ctrl_if = audio_codec_new_i2c_ctrl(&i2c_cfg),
+        .gpio_if = audio_codec_new_gpio(),
+        .codec_mode = ESP_CODEC_DEV_WORK_MODE_BOTH,
+        .pa_pin = GPIO_NUM_NC,
+        .use_mclk = false,
+        .hw_gain = {
+            .pa_voltage = 5.0,
+            .codec_dac_voltage = 3.3
+        }
+    };
+
+    esp_codec_dev_cfg_t dev_cfg = {
+        .dev_type = ESP_CODEC_DEV_TYPE_IN_OUT,
+        .codec_if = es8311_codec_new(&es8311_cfg),
+        .data_if = audio_codec_new_i2s_data(&i2s_cfg),
+    };
+    audio_dev = esp_codec_dev_new(&dev_cfg);
+
+    esp_codec_dev_sample_info_t fs = {
+        .bits_per_sample = 16,
+        .channel = 1,
+        .channel_mask = 0,
+        .sample_rate = SAMPLE_RATE,
+        .mclk_multiple = 0,
+    };
+    ESP_ERROR_CHECK(esp_codec_dev_open(audio_dev, &fs));
+    ESP_ERROR_CHECK(esp_codec_dev_set_in_gain(audio_dev, 30.0));
+    ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(audio_dev, 100));
+}
+
+int Read(void* dest, int size) {
+    ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_read(audio_dev, (void*)dest, size));
+    return size;
+}
+
+int Write(const void* data, int size) {
+    ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_write(audio_dev, (void*)data, size));
+    return size;
+}
+
+void pipecat_init_audio_capture() {
+    i2c_master_bus_config_t i2c_bus_cfg = {
+        .i2c_port = I2C_NUM_1,
+        .sda_io_num = GPIO_NUM_38,
+        .scl_io_num = GPIO_NUM_39,
+        .clk_source = I2C_CLK_SRC_DEFAULT,
+        .glitch_ignore_cnt = 7,
+        .intr_priority = 0,
+        .trans_queue_depth = 0,
+        .flags = {
+            .enable_internal_pullup = 1,
+        },
+    };
+    ESP_ERROR_CHECK(i2c_new_master_bus(&i2c_bus_cfg, &i2c_bus));
+    configure_pi4ioe();
+    configure_es8311();
+}
+
+opus_int16 *decoder_buffer = NULL;
+OpusDecoder *opus_decoder = NULL;
+
+void pipecat_init_audio_decoder() {
+  int decoder_error = 0;
+  opus_decoder = opus_decoder_create(SAMPLE_RATE, 1, &decoder_error);
+  if (decoder_error != OPUS_OK) {
+    printf("Failed to create OPUS decoder");
+    return;
+  }
+
+  decoder_buffer = (opus_int16 *)malloc(PCM_BUFFER_SIZE);
+}
+
+
 std::atomic<bool> is_playing = false;
 unsigned int silence_count = 0;
 
@@ -39,47 +209,19 @@ void set_is_playing(int16_t *in_buf, size_t in_samples) {
   }
 
   if (silence_count >= 20 && is_playing) {
-    M5.Speaker.end();
-    M5.Mic.begin();
     is_playing = false;
   } else if (any_set && !is_playing) {
-    M5.Mic.end();
-    M5.Speaker.begin();
     is_playing = true;
   }
 }
 
-void pipecat_init_audio_capture() {
-  M5.Speaker.setVolume(255);
-}
-
-opus_int16 *decoder_buffer = NULL;
-OpusDecoder *opus_decoder = NULL;
-
-void pipecat_init_audio_decoder() {
-  int decoder_error = 0;
-  opus_decoder = opus_decoder_create(SAMPLE_RATE, 1, &decoder_error);
-  if (decoder_error != OPUS_OK) {
-    printf("Failed to create OPUS decoder");
-    return;
-  }
-
-  decoder_buffer = (opus_int16 *)malloc(PCM_BUFFER_SIZE);
-}
 
 void pipecat_audio_decode(uint8_t *data, size_t size) {
   int decoded_size =
       opus_decode(opus_decoder, data, size, decoder_buffer, PCM_BUFFER_SIZE, 0);
 
   if (decoded_size > 0) {
-    set_is_playing(decoder_buffer, decoded_size);
-    if (is_playing) {
-      M5.Speaker.playRaw(decoder_buffer, decoded_size, SAMPLE_RATE);
-      do {
-        M5.delay(1);
-        M5.update();
-      } while (M5.Speaker.isPlaying());
-    }
+    Write(decoder_buffer, PCM_BUFFER_SIZE);
   }
 }
 
@@ -115,7 +257,7 @@ void pipecat_send_audio(PeerConnection *peer_connection) {
     memset(read_buffer, 0, PCM_BUFFER_SIZE);
     vTaskDelay(pdMS_TO_TICKS(20));
   } else {
-    // M5.Mic.record(read_buffer, PCM_BUFFER_SIZE / sizeof(uint16_t), SAMPLE_RATE);
+    // Do record
   }
 
   auto encoded_size = opus_encode(opus_encoder, (const opus_int16 *)read_buffer,
