diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java index bd7e7ca9bb..3c0f51a596 100755 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java @@ -19,9 +19,11 @@ import android.app.NotificationManager; import android.app.PendingIntent; import android.app.Service; +import android.content.BroadcastReceiver; import android.content.ComponentName; import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; import android.content.ServiceConnection; import android.content.SharedPreferences; import android.content.pm.PackageManager; @@ -65,6 +67,7 @@ import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.HeadUpAngleEvent; import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.KeepAliveAckEvent; import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.MicModeChangedEvent; +import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.PauseAsrEvent; import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.RtmpStreamStatusEvent; import com.augmentos.augmentos_core.smarterglassesmanager.supportedglasses.SmartGlassesDevice; import com.augmentos.augmentos_core.smarterglassesmanager.utils.BitmapJavaUtils; @@ -161,6 +164,8 @@ public class AugmentosService extends LifecycleService implements AugmentOsActio public SmartGlassesManager smartGlassesManager; private boolean smartGlassesManagerBound = false; private final List smartGlassesReadyListeners = new ArrayList<>(); + + private STTControlReceiver sttControlReceiver; private byte[] hexStringToByteArray(String hex) { int len = hex.length(); @@ -544,6 +549,15 @@ public void onCreate() { EventBus.getDefault().register(this); Log.d(TAG, "🔔 EventBus registration completed for AugmentosService"); + + // Register STT control receiver for mobile audio commands + sttControlReceiver = new STTControlReceiver(); + IntentFilter sttFilter = new IntentFilter("com.augmentos.augmentos_core.STT_CONTROL"); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + registerReceiver(sttControlReceiver, sttFilter, Context.RECEIVER_EXPORTED); + } else { + registerReceiver(sttControlReceiver, sttFilter); + } ServerComms.getInstance(this); @@ -2957,6 +2971,15 @@ public void onDestroy() { } catch (Exception e) { Log.e(TAG, "Error unregistering from EventBus", e); } + + // Unregister STT control receiver + try { + if (sttControlReceiver != null) { + unregisterReceiver(sttControlReceiver); + } + } catch (Exception e) { + Log.e(TAG, "Error unregistering STT BroadcastReceiver", e); + } // Stop periodic datetime sending datetimeHandler.removeCallbacks(datetimeRunnable); @@ -3351,4 +3374,17 @@ public void stopVideoRecording(String requestId) { Log.e(TAG, "SmartGlassesManager is null, cannot stop video recording"); } } + + private class STTControlReceiver extends BroadcastReceiver { + @Override + public void onReceive(Context context, Intent intent) { + if ("com.augmentos.augmentos_core.STT_CONTROL".equals(intent.getAction())) { + boolean pauseSTT = intent.getBooleanExtra("pause_stt", false); + String source = intent.getStringExtra("source"); + + // Post the same event that native TTS uses + EventBus.getDefault().post(new PauseAsrEvent(pauseSTT)); + } + } + } } \ No newline at end of file diff --git a/docs/echo-cancellation-attempts-summary.md b/docs/echo-cancellation-attempts-summary.md new file mode 100644 index 0000000000..1cec730847 --- /dev/null +++ b/docs/echo-cancellation-attempts-summary.md @@ -0,0 +1,296 @@ +# Echo Cancellation Implementation Attempts - Technical Summary + +**Date**: September 5, 2025 +**Issue**: [GitHub #1036] SDK .speak() infinite echo loop +**Team Members**: Alfonso +**Status**: ❌ Unresolved - Requires architectural evaluation + +--- + +## Problem Statement + +SDK `.speak()` calls create infinite echo loops on Android when using phone speaker + phone microphone: + +1. User says "Hello" +2. App calls `session.audio.speak("Hello")` +3. TTS plays through phone speaker +4. Phone microphone captures TTS audio +5. Speech recognition processes captured audio as new speech +6. App responds with another `.speak()` call → **Infinite loop** + +**Expected Behavior**: Microphone should be muted during TTS playback to prevent feedback loops. + +--- + +## Root Cause Analysis + +### Initial Assumptions ❌ + +- SDK handled microphone muting internally +- Audio requests went through Android core services +- Built-in Android AEC would resolve the issue + +### Reality Discovered ✅ + +- **SDK `.speak()` provides NO microphone control** +- **Audio path**: Mobile app AudioManager → ExoPlayer → Phone speaker +- **Missing component**: Microphone pause during TTS playback +- **Core issue**: Physical acoustic coupling between speaker and microphone + +--- + +## Implementation Attempts (Chronological) + +### ❌ Attempt 1: Built-in Android AEC (BUILT-IN-AEC) + +**Approach**: Change AudioRecord source to VOICE_COMMUNICATION for automatic echo cancellation + +**Files Modified**: + +- `android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/hci/MicrophoneLocalAndBluetooth.java` + +**Implementation**: + +```java +// Changed from: +MediaRecorder.AudioSource.VOICE_RECOGNITION +// To: +MediaRecorder.AudioSource.VOICE_COMMUNICATION +``` + +**Logic**: VOICE_COMMUNICATION has built-in acoustic echo cancellation for phone calls + +**Result**: ❌ No improvement - infinite loop continued + +**Issue**: Built-in AEC insufficient for TTS playback scenarios + +--- + +### ❌ Attempt 2: Enhanced TTS Microphone Pause (ASR-PAUSE) + +**Approach**: Enhance existing TTS system to pause microphone during native TTS playback + +**Files Modified**: + +- `android_core/.../smarterglassesmanager/texttospeech/TextToSpeechSystem.java` +- `android_core/.../smarterglassesmanager/hci/PhoneMicrophoneManager.java` +- `android_core/.../smarterglassesmanager/eventbusmessages/PauseMicrophoneEvent.java` + +**Implementation**: + +```java +// TextToSpeechSystem.java - onStart() +@Override +public void onStart(String utteranceId) { + Log.d(TAG, "🔇 TTS started - pausing ASR and microphone to prevent echo loop"); + EventBus.getDefault().post(new PauseAsrEvent(true)); + EventBus.getDefault().post(new PauseMicrophoneEvent(true)); +} + +// TextToSpeechSystem.java - onDone() +@Override +public void onDone(String utteranceId) { + Log.d(TAG, "🔊 TTS finished - resuming ASR and microphone"); + EventBus.getDefault().post(new PauseAsrEvent(false)); + EventBus.getDefault().post(new PauseMicrophoneEvent(false)); +} + +// PhoneMicrophoneManager.java +@Subscribe +public void handlePauseMicrophoneEvent(PauseMicrophoneEvent event) { + if (event.pauseMicrophone) { + // Pause microphone + stopMicrophoneService(); + currentStatus = MicStatus.PAUSED; + } else { + // Resume microphone + startPreferredMicMode(); + } +} +``` + +**Result**: ❌ Worked for native TTS but not SDK `.speak()` + +**Issue**: SDK `.speak()` bypasses Android core TTS system entirely + +--- + +### ❌ Attempt 3: AugmentosService Audio Interception (AUDIO-INTERCEPT) + +**Approach**: Intercept audio requests at AugmentosService level + +**Files Modified**: + +- `android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java` + +**Target Method**: `onAudioPlayRequest(JSONObject audioRequest)` + +**Implementation**: + +```java +@Override +public void onAudioPlayRequest(JSONObject audioRequest) { + Log.d(TAG, "🔇 Pausing microphone for audio playback to prevent echo loop"); + EventBus.getDefault().post(new PauseMicrophoneEvent(true)); + + // Forward audio request to glasses/manager... +} + +@Override +public void onAudioPlayResponse(JSONObject audioResponse) { + Log.d(TAG, "🔊 Audio completed - resuming microphone"); + EventBus.getDefault().post(new PauseMicrophoneEvent(false)); + + // Forward response to cloud... +} +``` + +**Result**: ❌ Method never executed during SDK `.speak()` calls + +**Issue**: SDK audio requests don't route through Android core + +**Discovery**: SDK audio path is Mobile → AudioManager, not Mobile → Android Core + +--- + +### ❌ Attempt 4: Mobile AudioManager Integration (MOBILE-INTERCEPT) + +**Approach**: Add microphone control to mobile `AudioManagerModule.java` with broadcast communication + +**Files Modified**: + +- `mobile/android/app/src/main/java/com/mentra/mentra/AudioManagerModule.java` +- `android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java` +- `android_core/.../smarterglassesmanager/hci/PhoneMicrophoneManager.java` + +**Architecture**: + +``` +Mobile AudioManagerModule → Android BroadcastReceiver → EventBus → PhoneMicrophoneManager +``` + +**Implementation**: + +_Mobile AudioManagerModule.java_: + +```java +@ReactMethod +public void playAudio(String requestId, String audioUrl, float volume, boolean stopOtherAudio, Promise promise) { + Log.d(TAG, "🔇 Pausing microphone for SDK .speak() audio playback"); + sendMicrophonePauseCommand(true); // Pause microphone + + AudioManager audioManager = AudioManager.getInstance(reactContext); + audioManager.playAudio(requestId, audioUrl, volume, stopOtherAudio); +} + +public void sendAudioPlayResponse(String requestId, boolean success, String error, Long duration) { + if (success) { + Log.d(TAG, "🔊 Audio completed successfully - resuming microphone"); + sendMicrophonePauseCommand(false); // Resume microphone + } +} + +private void sendMicrophonePauseCommand(boolean pauseMicrophone) { + Intent intent = new Intent("com.augmentos.augmentos_core.MIC_CONTROL"); + intent.putExtra("pause_microphone", pauseMicrophone); + intent.putExtra("source", "mobile_audio_manager"); + reactContext.sendBroadcast(intent); +} +``` + +_Android Core AugmentosService.java_: + +```java +// BroadcastReceiver registration in onCreate() +microphoneControlReceiver = new MicrophoneControlReceiver(); +IntentFilter filter = new IntentFilter("com.augmentos.augmentos_core.MIC_CONTROL"); +registerReceiver(microphoneControlReceiver, filter); + +// BroadcastReceiver implementation +private class MicrophoneControlReceiver extends BroadcastReceiver { + @Override + public void onReceive(Context context, Intent intent) { + if ("com.augmentos.augmentos_core.MIC_CONTROL".equals(intent.getAction())) { + boolean pauseMicrophone = intent.getBooleanExtra("pause_microphone", false); + EventBus.getDefault().post(new PauseMicrophoneEvent(pauseMicrophone)); + } + } +} +``` + +_PhoneMicrophoneManager.java_ (Enhanced): + +```java +@Subscribe +public void handlePauseMicrophoneEvent(PauseMicrophoneEvent event) { + if (event.pauseMicrophone) { + if (currentStatus != MicStatus.PAUSED) { + statusBeforeTTSPause = currentStatus; + isPausedForTTS = true; + stopMicrophoneService(); + currentStatus = MicStatus.PAUSED; + } + } else { + if (isPausedForTTS) { + isPausedForTTS = false; + if (statusBeforeTTSPause != MicStatus.PAUSED) { + mainHandler.postDelayed(() -> { + startPreferredMicMode(); + }, 100); // Small delay for audio completion + } + } + } +} +``` + +**Result**: ✅ Communication chain working perfectly, ❌ but still infinite loop + +**Verified Working**: + +- ✅ Mobile sends broadcast commands +- ✅ Android core receives broadcast commands +- ✅ EventBus delivers PauseMicrophoneEvent +- ✅ Microphone actually pauses (SCO_MODE → PAUSED) +- ✅ Microphone actually resumes (PAUSED → SCO_MODE) + +**Debug Logs Confirmed**: + +``` +AudioManagerModule: 🔇 Pausing microphone for SDK .speak() audio playback +AugmentOSService: 🎙️ Received microphone control command from mobile_audio_manager: pause=true +PhoneMicrophoneManager: 🔇 Pausing microphone for TTS playback (current status: SCO_MODE) +PhoneMicrophoneManager: 🔍 PAUSE DEBUG: Microphone service stopped, currentStatus now=PAUSED + +[TTS Audio Plays] + +AudioManagerModule: 🔊 Audio completed successfully - resuming microphone +AugmentOSService: 🎙️ Received microphone control command from mobile_audio_manager: pause=false +PhoneMicrophoneManager: 🔊 Resuming microphone after TTS playback (restore to: SCO_MODE) +PhoneMicrophoneManager: 🔊 Actually resuming microphone service +``` + +**Issue**: Despite perfect software execution, acoustic coupling between phone speaker and microphone creates unavoidable feedback + +--- + +### ❌ Attempt 5: Extended Timing Delays + +**Approach**: Increase microphone resume delay to allow acoustic settling + +**Implementation**: + +```java +// Extended resume delay from 100ms to 2000ms (2 seconds) +mainHandler.postDelayed(() -> { + Log.d(TAG, "🔊 Actually resuming microphone service"); + startPreferredMicMode(); +}, 2000); // 2000ms delay to ensure all audio is finished and echoes have died down +``` + +**Logic**: Allow TTS audio and room acoustics to fully dissipate before resuming microphone + +**Result**: ❌ Still creates infinite loops + +**Issue**: Timing approach insufficient for physical acoustic coupling between speaker and microphone + +--- diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index a605bb840a..8e397f8155 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -113,7 +113,7 @@ android { minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion versionCode 44 - versionName "2.2.8" + versionName "2.2.7" } signingConfigs { debug { diff --git a/mobile/android/app/src/main/java/com/mentra/mentra/AudioManagerModule.java b/mobile/android/app/src/main/java/com/mentra/mentra/AudioManagerModule.java index 0775b1092a..5866296d75 100644 --- a/mobile/android/app/src/main/java/com/mentra/mentra/AudioManagerModule.java +++ b/mobile/android/app/src/main/java/com/mentra/mentra/AudioManagerModule.java @@ -8,6 +8,9 @@ import com.facebook.react.bridge.WritableMap; import com.facebook.react.bridge.Arguments; +import android.content.Intent; +import android.os.Handler; +import android.os.Looper; import android.util.Log; import androidx.annotation.NonNull; @@ -18,6 +21,9 @@ public class AudioManagerModule extends ReactContextBaseJavaModule { private static final String TAG = "AudioManagerModule"; private ReactApplicationContext reactContext; + private static boolean sttPaused = false; + private Handler timeoutHandler = new Handler(Looper.getMainLooper()); + private Runnable currentTimeoutRunnable; public AudioManagerModule(ReactApplicationContext reactContext) { super(reactContext); @@ -50,6 +56,10 @@ public void sendAudioPlayResponse(String requestId, boolean success, String erro .emit("AudioPlayResponse", params); } + if (success) { + sendSTTPauseCommand(false); // Resume STT processing + } + Log.d(TAG, "Sent audio play response - requestId: " + requestId + ", success: " + success + ", error: " + error); } @@ -63,6 +73,7 @@ public void playAudio( ) { try { Log.d(TAG, "playAudio called with requestId: " + requestId); + sendSTTPauseCommand(true); // Pause STT processing AudioManager audioManager = AudioManager.getInstance(reactContext); @@ -78,7 +89,9 @@ public void playAudio( promise.resolve("Audio play started"); } catch (Exception e) { - Log.e(TAG, "Failed to play audio", e); + // CRITICAL: Resume STT if audio failed to start + sendSTTPauseCommand(false); + Log.e(TAG, "Failed to play audio, resuming STT", e); promise.reject("AUDIO_PLAY_ERROR", e.getMessage(), e); } } @@ -112,4 +125,40 @@ public void stopAllAudio(Promise promise) { promise.reject("AUDIO_STOP_ALL_ERROR", e.getMessage(), e); } } + + private void sendSTTPauseCommand(boolean pauseSTT) { + // Prevent redundant commands + if (sttPaused == pauseSTT) { + Log.d(TAG, "STT already in requested state: " + pauseSTT); + return; + } + + sttPaused = pauseSTT; + Intent intent = new Intent("com.augmentos.augmentos_core.STT_CONTROL"); + intent.putExtra("pause_stt", pauseSTT); + intent.putExtra("source", "mobile_audio_manager"); + reactContext.sendBroadcast(intent); + + if (pauseSTT) { + // Cancel any existing timeout first + if (currentTimeoutRunnable != null) { + timeoutHandler.removeCallbacks(currentTimeoutRunnable); + } + + // Create new timeout + currentTimeoutRunnable = () -> { + if (sttPaused) { + Log.w(TAG, "STT timeout safety - force resuming STT after 30s"); + sendSTTPauseCommand(false); + } + }; + timeoutHandler.postDelayed(currentTimeoutRunnable, 30000); + } else { + // Cancel timeout when manually resuming + if (currentTimeoutRunnable != null) { + timeoutHandler.removeCallbacks(currentTimeoutRunnable); + currentTimeoutRunnable = null; + } + } + } } \ No newline at end of file diff --git a/mobile/ios/AOS/Info.plist b/mobile/ios/AOS/Info.plist index dd7efedd5d..d454b7f394 100644 --- a/mobile/ios/AOS/Info.plist +++ b/mobile/ios/AOS/Info.plist @@ -19,7 +19,7 @@ CFBundlePackageType $(PRODUCT_BUNDLE_PACKAGE_TYPE) CFBundleShortVersionString - 2.2.8 + 2.2.7 CFBundleSignature ???? CFBundleURLTypes