diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index c9104620c5..cf9103af3f 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -108,7 +108,7 @@ android { minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion versionCode 49 - versionName "2.2.13" + versionName "2.2.7" } signingConfigs { debug { diff --git a/mobile/app.json b/mobile/app.json index 67fcb1a19c..c6985dbd1c 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -43,7 +43,7 @@ "icon": "./assets/app-icons/ic_launcher.png", "supportsTablet": false, "requireFullScreen": true, - "bundleIdentifier": "com.mentra.mentra", + "bundleIdentifier": "com.vanwifferen.mentra1111112", "associatedDomains": ["applinks:apps.mentra.glass"], "infoPlist": { "NSCameraUsageDescription": "This app needs access to your camera to capture images.", diff --git a/mobile/ios/AOS.xcodeproj/project.pbxproj b/mobile/ios/AOS.xcodeproj/project.pbxproj index b11dc225b9..747675bf96 100644 --- a/mobile/ios/AOS.xcodeproj/project.pbxproj +++ b/mobile/ios/AOS.xcodeproj/project.pbxproj @@ -52,6 +52,11 @@ C5DE98972DE267CD0032FC99 /* Bridge.m in Sources */ = {isa = PBXBuildFile; fileRef = C5DE98952DE267CD0032FC99 /* Bridge.m */; }; C5DE98982DE267CD0032FC99 /* Bridge.swift in Sources */ = {isa = PBXBuildFile; fileRef = C5DE98962DE267CD0032FC99 /* Bridge.swift */; }; C5F7A1012E6C111500123456 /* libbz2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = C5F7A1002E6C111500123456 /* libbz2.tbd */; }; + E1A4F0A13E6B4B8800F1AA01 /* MentraManager+Head.swift in Sources */ = {isa = PBXBuildFile; fileRef = E1A4F0A03E6B4B8800F1AA01 /* MentraManager+Head.swift */; }; + E1A4F0A33E6B4B8800F1AA01 /* MentraManager+Mic.swift in Sources */ = {isa = PBXBuildFile; fileRef = E1A4F0A23E6B4B8800F1AA01 /* MentraManager+Mic.swift */; }; + E1A4F0A53E6B4B8800F1AA01 /* MentraManager+Display.swift in Sources */ = {isa = PBXBuildFile; fileRef = E1A4F0A43E6B4B8800F1AA01 /* MentraManager+Display.swift */; }; + E1A4F0A73E6B4B8800F1AA01 /* MentraManager+Device.swift in Sources */ = {isa = PBXBuildFile; fileRef = E1A4F0A63E6B4B8800F1AA01 /* MentraManager+Device.swift */; }; + E1A4F0A93E6B4B8800F1AA01 /* MentraManager+Status.swift in Sources */ = {isa = PBXBuildFile; fileRef = E1A4F0A83E6B4B8800F1AA01 /* MentraManager+Status.swift */; }; F314884EB5724196A394670E /* noop-file.swift in Sources */ = {isa = PBXBuildFile; fileRef = 14C4353E097941A5AC9B6A09 /* noop-file.swift */; }; /* End PBXBuildFile section */ @@ -128,6 +133,11 @@ C5DE98952DE267CD0032FC99 /* Bridge.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = Bridge.m; sourceTree = ""; }; C5DE98962DE267CD0032FC99 /* Bridge.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Bridge.swift; sourceTree = ""; }; C5F7A1002E6C111500123456 /* libbz2.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libbz2.tbd; path = usr/lib/libbz2.tbd; sourceTree = SDKROOT; }; + E1A4F0A03E6B4B8800F1AA01 /* MentraManager+Head.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MentraManager+Head.swift"; sourceTree = ""; }; + E1A4F0A23E6B4B8800F1AA01 /* MentraManager+Mic.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MentraManager+Mic.swift"; sourceTree = ""; }; + E1A4F0A43E6B4B8800F1AA01 /* MentraManager+Display.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MentraManager+Display.swift"; sourceTree = ""; }; + E1A4F0A63E6B4B8800F1AA01 /* MentraManager+Device.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MentraManager+Device.swift"; sourceTree = ""; }; + E1A4F0A83E6B4B8800F1AA01 /* MentraManager+Status.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MentraManager+Status.swift"; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; FAC715A2D49A985799AEE119 /* ExpoModulesProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; name = ExpoModulesProvider.swift; path = "Pods/Target Support Files/Pods-AOS/ExpoModulesProvider.swift"; sourceTree = ""; }; /* End PBXFileReference section */ @@ -276,6 +286,11 @@ C5DE98952DE267CD0032FC99 /* Bridge.m */, C5DE98962DE267CD0032FC99 /* Bridge.swift */, C5DE97882DDD0EA80032FC99 /* MentraManager.swift */, + E1A4F0A03E6B4B8800F1AA01 /* MentraManager+Head.swift */, + E1A4F0A23E6B4B8800F1AA01 /* MentraManager+Mic.swift */, + E1A4F0A43E6B4B8800F1AA01 /* MentraManager+Display.swift */, + E1A4F0A63E6B4B8800F1AA01 /* MentraManager+Device.swift */, + E1A4F0A83E6B4B8800F1AA01 /* MentraManager+Status.swift */, C5DE97892DDD0EA80032FC99 /* BridgeModule.h */, C5DE978A2DDD0EA80032FC99 /* BridgeModule.m */, C5DE98672DDD1FB00032FC99 /* Bridging-Header.h */, @@ -619,6 +634,11 @@ 13B07FC11A68108700A75B9A /* main.m in Sources */, C5DE979E2DDD0EA80032FC99 /* BridgeModule.m in Sources */, C5DE979F2DDD0EA80032FC99 /* MentraManager.swift in Sources */, + E1A4F0A13E6B4B8800F1AA01 /* MentraManager+Head.swift in Sources */, + E1A4F0A33E6B4B8800F1AA01 /* MentraManager+Mic.swift in Sources */, + E1A4F0A53E6B4B8800F1AA01 /* MentraManager+Display.swift in Sources */, + E1A4F0A73E6B4B8800F1AA01 /* MentraManager+Device.swift in Sources */, + E1A4F0A93E6B4B8800F1AA01 /* MentraManager+Status.swift in Sources */, C5DE98472DDD0F420032FC99 /* SileroVAD.swift in Sources */, C5DE98482DDD0F420032FC99 /* lc3.c in Sources */, C5DE98492DDD0F420032FC99 /* ContentData.swift in Sources */, @@ -663,7 +683,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = AOS/AOS.entitlements; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = T5XXXL6N36; + DEVELOPMENT_TEAM = 58Q847YBLM; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -804,7 +824,7 @@ "-lc++", ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG"; - PRODUCT_BUNDLE_IDENTIFIER = com.mentra.mentra; + PRODUCT_BUNDLE_IDENTIFIER = com.vanwifferen.mentra1111112; PRODUCT_NAME = "MentraOS"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; SUPPORTS_MACCATALYST = NO; @@ -826,7 +846,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = AOS/AOS.entitlements; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = T5XXXL6N36; + DEVELOPMENT_TEAM = 58Q847YBLM; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -963,7 +983,7 @@ "-lc++", ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE"; - PRODUCT_BUNDLE_IDENTIFIER = com.mentra.mentra; + PRODUCT_BUNDLE_IDENTIFIER = com.vanwifferen.mentra1111112; PRODUCT_NAME = "MentraOS"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; SUPPORTS_MACCATALYST = NO; diff --git a/mobile/ios/AOS.xcodeproj/xcshareddata/xcschemes/AOS.xcscheme b/mobile/ios/AOS.xcodeproj/xcshareddata/xcschemes/AOS.xcscheme index 0d9bc082a6..38174e3d67 100644 --- a/mobile/ios/AOS.xcodeproj/xcshareddata/xcschemes/AOS.xcscheme +++ b/mobile/ios/AOS.xcodeproj/xcshareddata/xcschemes/AOS.xcscheme @@ -41,7 +41,7 @@ CFBundlePackageType $(PRODUCT_BUNDLE_PACKAGE_TYPE) CFBundleShortVersionString - 2.2.13 + 2.2.7 CFBundleSignature ???? CFBundleURLTypes @@ -28,7 +28,7 @@ CFBundleURLSchemes com.mentra - com.mentra.mentra + com.vanwifferen.mentra1111112 diff --git a/mobile/ios/Source/Bridge.swift b/mobile/ios/Source/Bridge.swift index 9f16181cb6..a68caaa1ef 100644 --- a/mobile/ios/Source/Bridge.swift +++ b/mobile/ios/Source/Bridge.swift @@ -85,7 +85,9 @@ class Bridge: RCTEventEmitter { } } - static func sendLocationUpdate(lat: Double, lng: Double, accuracy: Double?, correlationId: String?) { + static func sendLocationUpdate( + lat: Double, lng: Double, accuracy: Double?, correlationId: String? + ) { do { var event: [String: Any] = [ "type": "location_update", @@ -161,8 +163,12 @@ class Bridge: RCTEventEmitter { } } - func sendAudioPlayResponse(requestId: String, success: Bool, error: String? = nil, duration: Double? = nil) { - Bridge.log("ServerComms: Sending audio play response - requestId: \(requestId), success: \(success), error: \(error ?? "none")") + func sendAudioPlayResponse( + requestId: String, success: Bool, error: String? = nil, duration: Double? = nil + ) { + Bridge.log( + "ServerComms: Sending audio play response - requestId: \(requestId), success: \(success), error: \(error ?? "none")" + ) let message: [String: Any] = [ "type": "audio_play_response", "requestId": requestId, @@ -426,6 +432,7 @@ class Bridge: RCTEventEmitter { case rgb_led_control case microphone_state_change case restart_transcriber + case set_foreground_app_open case unknown } @@ -436,7 +443,9 @@ class Bridge: RCTEventEmitter { } do { - if let jsonDict = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] { + if let jsonDict = try JSONSerialization.jsonObject(with: data, options: []) + as? [String: Any] + { // Extract command type guard let commandString = jsonDict["command"] as? String else { Bridge.log("CommandBridge: Invalid command format: missing 'command' field") @@ -485,8 +494,10 @@ class Bridge: RCTEventEmitter { case .forget_smart_glasses: m.forgetSmartGlasses() case .search_for_compatible_device_names: - guard let params = params, let modelName = params["model_name"] as? String else { - Bridge.log("CommandBridge: search_for_compatible_device_names invalid params") + guard let params = params, let modelName = params["model_name"] as? String + else { + Bridge.log( + "CommandBridge: search_for_compatible_device_names invalid params") break } m.handleSearchForCompatibleDeviceNames(modelName) @@ -548,7 +559,9 @@ class Bridge: RCTEventEmitter { Bridge.log("CommandBridge: save_buffer_video invalid params") break } - Bridge.log("CommandBridge: Saving buffer video: requestId=\(requestId), duration=\(durationSeconds)s") + Bridge.log( + "CommandBridge: Saving buffer video: requestId=\(requestId), duration=\(durationSeconds)s" + ) m.handle_save_buffer_video(requestId, durationSeconds) case .start_video_recording: guard let params = params, @@ -558,7 +571,9 @@ class Bridge: RCTEventEmitter { Bridge.log("CommandBridge: start_video_recording invalid params") break } - Bridge.log("CommandBridge: Starting video recording: requestId=\(requestId), save=\(save)") + Bridge.log( + "CommandBridge: Starting video recording: requestId=\(requestId), save=\(save)" + ) m.handle_start_video_recording(requestId, save) case .stop_video_recording: guard let params = params, @@ -607,8 +622,16 @@ class Bridge: RCTEventEmitter { } // Convert string array to enum array var requiredData = SpeechRequiredDataType.fromStringArray(requiredDataStrings) - Bridge.log("ServerComms: requiredData = \(requiredDataStrings), bypassVad = \(bypassVad)") + Bridge.log( + "ServerComms: requiredData = \(requiredDataStrings), bypassVad = \(bypassVad)" + ) m.handle_microphone_state_change(requiredData, bypassVad) + case .set_foreground_app_open: + guard let params = params, let active = params["active"] as? Bool else { + Bridge.log("CommandBridge: set_foreground_app_open invalid params") + break + } + m.setForegroundAppOpen(active) case .update_settings: guard let params else { Bridge.log("CommandBridge: update_settings invalid params") @@ -657,7 +680,9 @@ class Bridge: RCTEventEmitter { else { Bridge.log("CommandBridge: rgb_led_control invalid params") if let maybeRequestId = params?["requestId"] as? String { - Bridge.sendRgbLedControlResponse(requestId: maybeRequestId, success: false, error: "invalid_params") + Bridge.sendRgbLedControlResponse( + requestId: maybeRequestId, success: false, error: "invalid_params" + ) } break } @@ -678,13 +703,15 @@ class Bridge: RCTEventEmitter { let count = parseInt(params["count"]) ?? 1 let packageName = params["packageName"] as? String - m.handleRgbLedControl(requestId: requestId, - packageName: packageName, - action: action, - color: color, - ontime: ontime, - offtime: offtime, - count: count) + m.handleRgbLedControl( + requestId: requestId, + packageName: packageName, + action: action, + color: color, + ontime: ontime, + offtime: offtime, + count: count + ) // STT: case .set_stt_model_details: guard let params = params, @@ -715,7 +742,9 @@ class Bridge: RCTEventEmitter { Bridge.log("CommandBridge: extract_tar_bz2 invalid params") break } - return STTTools.extractTarBz2(sourcePath: sourcePath, destinationPath: destinationPath) + return STTTools.extractTarBz2( + sourcePath: sourcePath, destinationPath: destinationPath + ) case .restart_transcriber: m.restartTranscriber() } diff --git a/mobile/ios/Source/MentraManager+Device.swift b/mobile/ios/Source/MentraManager+Device.swift new file mode 100644 index 0000000000..37491504c0 --- /dev/null +++ b/mobile/ios/Source/MentraManager+Device.swift @@ -0,0 +1,211 @@ +// +// MentraManager+Device.swift +// MentraOS_Manager +// +// Created by Codex on 3/17/24. +// + +import Foundation + +extension MentraManager { + func initSGC(_ wearable: String) { + Bridge.log("Initializing manager for wearable: \(wearable)") + guard sgc == nil else { return } + + if wearable.contains("G1") { + sgc = G1() + } else if wearable.contains("Live") { + sgc = MentraLive() + } else if wearable.contains("Mach1") { + sgc = Mach1() + } else if wearable.contains("Frame") || wearable.contains("Brilliant Labs") { + sgc = FrameManager() + } + } + + func initSGCCallbacks() { + // TODO: make sure this functionality is baked into the SGCs! + } + + func handleConnectionStateChange() { + Bridge.log("Mentra: Glasses: connection state changed!") + guard let sgc else { return } + + if sgc.ready { + handleDeviceReady() + } else { + handleDeviceDisconnected() + handle_request_status() + } + } + + func disconnectWearable() { + sendText(" ") + Task { + connectTask?.cancel() + sgc?.disconnect() + isSearching = false + handle_request_status() + } + } + + func forgetSmartGlasses() { + disconnectWearable() + defaultWearable = "" + deviceName = "" + sgc?.forget() + sgc = nil + Bridge.saveSetting("default_wearable", "") + Bridge.saveSetting("device_name", "") + handle_request_status() + } + + func handleSearchForCompatibleDeviceNames(_ modelName: String) { + Bridge.log("Mentra: Searching for compatible device names for: \(modelName)") + if modelName.contains("Simulated") { + defaultWearable = "Simulated Glasses" + handle_request_status() + return + } + if modelName.contains("G1") { + pendingWearable = "Even Realities G1" + } else if modelName.contains("Live") { + pendingWearable = "Mentra Live" + } else if modelName.contains("Mach1") || modelName.contains("Z100") { + pendingWearable = "Mach1" + } + initSGC(pendingWearable) + sgc?.findCompatibleDevices() + } + + func handle_connect_wearable(_ deviceName: String, modelName: String? = nil) { + Bridge.log( + "Mentra: Connecting to modelName: \(modelName ?? "nil") deviceName: \(deviceName) defaultWearable: \(defaultWearable) pendingWearable: \(pendingWearable) selfDeviceName: \(self.deviceName)" + ) + + if let modelName { + pendingWearable = modelName + } + + if pendingWearable.contains("Simulated") { + Bridge.log( + "Mentra: Pending wearable is simulated, setting default wearable to Simulated Glasses" + ) + defaultWearable = "Simulated Glasses" + handle_request_status() + return + } + + if pendingWearable.isEmpty, defaultWearable.isEmpty { + Bridge.log("Mentra: No pending or default wearable, returning") + return + } + + if pendingWearable.isEmpty, !defaultWearable.isEmpty { + Bridge.log("Mentra: No pending wearable, using default wearable: \(defaultWearable)") + pendingWearable = defaultWearable + } + + Task { + disconnectWearable() + + try? await Task.sleep(nanoseconds: 100 * 1_000_000) + self.isSearching = true + handle_request_status() + + if !deviceName.isEmpty { + self.deviceName = deviceName + } + + initSGC(self.pendingWearable) + sgc?.connectById(self.deviceName) + } + } + + func handleDeviceReady() { + guard let sgc else { + Bridge.log("Mentra: SGC is nil, returning") + return + } + + Bridge.log("Mentra: handleDeviceReady(): \(sgc.type)") + Bridge.sendBatteryStatus(level: sgc.batteryLevel ?? -1, charging: false) + Bridge.sendGlassesConnectionState(modelName: defaultWearable, status: "CONNECTED") + + pendingWearable = "" + defaultWearable = sgc.type + isSearching = false + handle_request_status() + + if defaultWearable.contains("G1") { + handleG1Ready() + } else if defaultWearable.contains("Mach1") { + handleMach1Ready() + } + + Bridge.saveSetting("default_wearable", defaultWearable) + Bridge.saveSetting("device_name", deviceName) + } + + func handleDeviceDisconnected() { + Bridge.log("Mentra: Device disconnected") + handle_microphone_state_change([], false) + Bridge.sendGlassesConnectionState(modelName: defaultWearable, status: "DISCONNECTED") + handle_request_status() + } + + func startBufferRecording() { + sgc?.startBufferRecording() + } + + func stopBufferRecording() { + sgc?.stopBufferRecording() + } + + func isSomethingConnected() -> Bool { + if sgc?.ready == true { + return true + } + if defaultWearable.contains("Simulated") { + return true + } + return false + } +} + +private extension MentraManager { + func handleG1Ready() { + Task { + try? await Task.sleep(nanoseconds: 1_000_000_000) + await sgc?.setSilentMode(false) + await sgc?.getBatteryStatus() + + if shouldSendBootingMessage { + sendText("// BOOTING MENTRAOS") + } + + try? await Task.sleep(nanoseconds: 400_000_000) + sgc?.setHeadUpAngle(headUpAngle) + try? await Task.sleep(nanoseconds: 400_000_000) + sgc?.setBrightness(brightness, autoMode: autoBrightness) + + if shouldSendBootingMessage { + sendText("// MENTRAOS CONNECTED") + try? await Task.sleep(nanoseconds: 1_000_000_000) + sendText(" ") + } + + shouldSendBootingMessage = false + handle_request_status() + } + } + + func handleMach1Ready() { + Task { + sendText("MENTRAOS CONNECTED") + try? await Task.sleep(nanoseconds: 1_000_000_000) + clearDisplay() + handle_request_status() + } + } +} diff --git a/mobile/ios/Source/MentraManager+Display.swift b/mobile/ios/Source/MentraManager+Display.swift new file mode 100644 index 0000000000..6b11b1e4e6 --- /dev/null +++ b/mobile/ios/Source/MentraManager+Display.swift @@ -0,0 +1,252 @@ +// +// MentraManager+Display.swift +// MentraOS_Manager +// +// Created by Codex on 3/17/24. +// + +import Foundation + +extension MentraManager { + func clearState() { + sendCurrentState(sgc?.isHeadUp ?? false) + } + + func sendCurrentState(_ requestDashboard: Bool) { + guard !isUpdatingScreen else { return } + + let effectiveHeadUpForUI = isHeadUpEffectiveForUI + let shouldDisplayDashboard = requestDashboard && effectiveHeadUpForUI + + Bridge.log( + "Mentra: DISPLAY: sendCurrentState(request=\(requestDashboard), effectiveHeadUpForUI=\(effectiveHeadUpForUI), rawHeadUp=\(isHeadUp), contextualDashboard=\(contextualDashboard), timeoutElapsed=\(headUpMicTimeoutElapsed), hasFG=\(hasForegroundAppOpen))" + ) + + if requestDashboard && !shouldDisplayDashboard { + Bridge.log( + "Mentra: DISPLAY: WARNING: dashboard requested but effective head-up is false; forcing main view" + ) + } + + if requestDashboard && !contextualDashboard { + Bridge.log("Mentra: DISPLAY: contextualDashboard disabled, skip dashboard") + return + } + + guard isRealWearableConnected else { + Bridge.log("Mentra: DISPLAY: no real glasses connected, skipping") + return + } + + guard isSomethingConnected() else { + Bridge.log("Mentra: DISPLAY: no device connection, skipping") + return + } + + sendStateWorkItem?.cancel() + + let viewState = shouldDisplayDashboard ? viewStates[1] : viewStates[0] + let layout = MentraDisplayLayout(from: viewState.layoutType) + + Task { + await render(viewState, layout: layout) + } + } + + func parsePlaceholders(_ text: String) -> String { + let dateFormatter = DateFormatter() + dateFormatter.dateFormat = "M/dd, h:mm" + let formattedDate = dateFormatter.string(from: Date()) + + let time12Format = DateFormatter() + time12Format.dateFormat = "hh:mm" + let time12 = time12Format.string(from: Date()) + + let time24Format = DateFormatter() + time24Format.dateFormat = "HH:mm" + let time24 = time24Format.string(from: Date()) + + let dateFormat = DateFormatter() + dateFormat.dateFormat = "MM/dd" + let currentDate = dateFormat.string(from: Date()) + + var placeholders: [String: String] = [ + "$no_datetime$": formattedDate, + "$DATE$": currentDate, + "$TIME12$": time12, + "$TIME24$": time24, + ] + + if let battery = sgc?.batteryLevel, battery >= 0 { + placeholders["$GBATT$"] = "\(battery)%" + } else { + placeholders["$GBATT$"] = "" + } + + if !glassesWifiSsid.isEmpty { + placeholders["$GWIFI_SSID$"] = glassesWifiSsid + } + + placeholders["$CONNECTION_STATUS$"] = "Connected" + placeholders["$GCLOUD$"] = "$CORE_CONNECTION$" + + var parsedText = text + for (placeholder, value) in placeholders { + parsedText = parsedText.replacingOccurrences(of: placeholder, with: value) + } + return parsedText + } + + func handle_display_text(_ params: [String: Any]) { + guard let text = params["text"] as? String else { + Bridge.log("Mentra: display_text missing text parameter") + return + } + + Bridge.log("Mentra: Displaying text: \(text)") + sendText(text) + } + + func handle_display_event(_ event: [String: Any]) { + guard let view = event["view"] as? String else { + Bridge.log("Mentra: invalid view") + return + } + + let isDashboard = view == "dashboard" + let stateIndex = isDashboard ? 1 : 0 + + guard let layout = event["layout"] as? [String: Any] else { + Bridge.log("Mentra: layout payload missing") + return + } + + let layoutType = layout["layoutType"] as? String ?? MentraDisplayLayout.textWall.rawValue + var text = layout["text"] as? String ?? " " + var topText = layout["topText"] as? String ?? " " + var bottomText = layout["bottomText"] as? String ?? " " + var title = layout["title"] as? String ?? " " + let data = layout["data"] as? String ?? "" + + text = parsePlaceholders(text) + topText = parsePlaceholders(topText) + bottomText = parsePlaceholders(bottomText) + title = parsePlaceholders(title) + + var newViewState = ViewState( + topText: topText, + bottomText: bottomText, + title: title, + layoutType: layoutType, + text: text, + data: data, + animationData: nil + ) + + if layoutType == "bitmap_animation" { + if let frames = layout["frames"] as? [String], + let interval = layout["interval"] as? Double + { + newViewState.animationData = [ + "frames": frames, + "interval": interval, + "repeat": layout["repeat"] as? Bool ?? true, + ] + Bridge.log( + "Mentra: Parsed bitmap_animation with \(frames.count) frames, interval: \(interval)ms" + ) + } else { + Bridge.log("Mentra: ERROR: bitmap_animation missing frames or interval") + } + } + + let currentState = viewStates[stateIndex] + let newStateKey = + newViewState.layoutType + newViewState.text + newViewState.topText + + newViewState.bottomText + newViewState.title + (newViewState.data ?? "") + let currentStateKey = + currentState.layoutType + currentState.text + currentState.topText + + currentState.bottomText + currentState.title + (currentState.data ?? "") + + guard newStateKey != currentStateKey else { + return + } + + Bridge.log( + "Updating view state \(stateIndex) with \(layoutType) \(text) \(topText) \(bottomText)" + ) + + viewStates[stateIndex] = newViewState + sendCurrentState(isDashboard) + } + + func clearDisplay() { + guard let sgc else { return } + + if sgc is G1 { + let g1 = sgc as? G1 + g1?.clearDisplay() + g1?.sendTextWall(" ") + + if powerSavingMode { + sendStateWorkItem?.cancel() + Bridge.log("Mentra: Clearing display after 3 seconds") + + let workItem = DispatchWorkItem { [weak self] in + guard let self else { return } + if self.isHeadUp { + return + } + g1?.clearDisplay() + } + sendStateWorkItem = workItem + sendStateQueue.asyncAfter(deadline: .now() + 3, execute: workItem) + } + } else { + sgc.clearDisplay() + } + } + + func sendText(_ text: String) { + guard let sgc else { return } + + guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + clearDisplay() + return + } + + sgc.sendTextWall(text) + } +} + +private extension MentraManager { + var isRealWearableConnected: Bool { + !defaultWearable.contains("Simulated") && !defaultWearable.isEmpty + } + + @MainActor + func render(_ viewState: ViewState, layout: MentraDisplayLayout) async { + Bridge.log("Mentra: DISPLAY: rendering layoutType=\(layout.rawValue)") + switch layout { + case .textWall: + sendText(viewState.text) + case .doubleTextWall: + sgc?.sendDoubleTextWall(viewState.topText, viewState.bottomText) + sgc?.sendDoubleTextWall(viewState.topText, viewState.bottomText) + case .referenceCard: + sendText("\(viewState.title)\n\n\(viewState.text)") + case .bitmap: + guard let data = viewState.data else { + Bridge.log("Mentra: ERROR: bitmap_view missing data field") + return + } + Bridge.log("Mentra: Processing bitmap_view with base64 data, length: \(data.count)") + await sgc?.displayBitmap(base64ImageData: data) + case .clear: + Bridge.log("Mentra: Processing clear_view layout - clearing display") + clearDisplay() + case .unknown: + Bridge.log("Mentra: DISPLAY: unknown layout \(viewState.layoutType)") + } + } +} diff --git a/mobile/ios/Source/MentraManager+Head.swift b/mobile/ios/Source/MentraManager+Head.swift new file mode 100644 index 0000000000..d60f976a47 --- /dev/null +++ b/mobile/ios/Source/MentraManager+Head.swift @@ -0,0 +1,146 @@ +// +// MentraManager+Head.swift +// MentraOS_Manager +// +// Created by Codex on 3/17/24. +// + +import Foundation + +extension MentraManager { + enum HeadRecomputeReason: String { + case headPositionChanged = "head_position_changed" + case headUpTimeoutElapsed = "head_up_timeout_elapsed" + case foregroundAppChanged = "foreground_app_changed" + case timeoutCancelled = "head_up_timeout_cancelled" + case manual + } + + /// True when the UI/mic should treat the head as "up". + var isHeadUpEffective: Bool { + isHeadUp && (!micBlockedByTimeout || hasForegroundAppOpen) + } + + /// UI-specific gating that mirrors the legacy dashboard heuristics. + var isHeadUpEffectiveForUI: Bool { + isHeadUp + && !(headUpMicTimeoutEnabled && headUpMicTimeoutElapsed && !hasForegroundAppOpen) + } + + func updateHeadUp(_ isHeadUp: Bool) { + let previous = self.isHeadUp + self.isHeadUp = isHeadUp + + Bridge.log("Mentra: HEAD: updateHeadUp(prev=\(previous) -> now=\(isHeadUp))") + + if isHeadUp { + headUpMicTimeoutElapsed = false + micBlockedByTimeout = false + scheduleHeadUpTimeoutIfNeeded() + } else { + cancelHeadUpTimeout() + } + + if previous != isHeadUp { + recomputeMicAndUI(.headPositionChanged) + } else { + sendCurrentState(isHeadUpEffective) + } + + Bridge.sendHeadUp(isHeadUp) + } + + func setForegroundAppOpen(_ active: Bool) { + let previous = hasForegroundAppOpen + hasForegroundAppOpen = active + + Bridge.log( + "Mentra: FG_APP: setForegroundAppOpen(prev=\(previous) -> now=\(active)), rawHeadUp=\(isHeadUp)" + ) + + Bridge.showBanner( + type: "info", message: active ? "Foreground app: ON" : "Foreground app: OFF" + ) + + if active { + // Foreground app opened: cancel any pending timeout without resetting flags. + // This preserves "one-shot per head-up" semantics until head goes down. + headUpMicTimeoutWorkItem?.cancel() + headUpMicTimeoutWorkItem = nil + Bridge.log("Mentra: HEAD: timeout canceled due to foreground app open (no reset)") + } else { + // Foreground app closed: do not reset elapsed/blocked. Only schedule if still eligible. + scheduleHeadUpTimeoutIfNeeded() + } + + recomputeMicAndUI(.foregroundAppChanged) + } + + func scheduleHeadUpTimeoutIfNeeded() { + headUpMicTimeoutWorkItem?.cancel() + + // Only applicable when "head_up" activation is in use and no foreground app is open + guard isHeadUp, headUpMicTimeoutEnabled, micActivationMode.requiresHeadUp, + !hasForegroundAppOpen + else { + return + } + + // Run only once per head-up cycle; do not reschedule after it has elapsed/blocked + guard !headUpMicTimeoutElapsed && !micBlockedByTimeout else { + Bridge.log( + "Mentra: HEAD: timeout already elapsed/blocked for this head-up cycle; not rescheduling" + ) + return + } + + let seconds = headUpMicTimeoutSeconds + + Bridge.log( + "Mentra: HEAD: scheduling timeout in \(seconds)s (isHeadUp=\(isHeadUp), hasFG=\(hasForegroundAppOpen))" + ) + + let workItem = DispatchWorkItem { [weak self] in + guard let self else { return } + + guard self.isHeadUp, !self.hasForegroundAppOpen else { + Bridge.log( + "Mentra: HEAD: timeout fired but conditions changed; ignoring (isHeadUp=\(self.isHeadUp), hasFG=\(self.hasForegroundAppOpen))" + ) + return + } + + self.headUpMicTimeoutElapsed = true + self.micBlockedByTimeout = true + self.micSessionActive = false + self.headUpMicTimeoutWorkItem = nil + + Bridge.log( + "Mentra: HEAD: timeout elapsed (\(seconds)s). micBlockedByTimeout=true; recomputing" + ) + + self.recomputeMicAndUI(.headUpTimeoutElapsed) + } + + headUpMicTimeoutWorkItem = workItem + sendStateQueue.asyncAfter(deadline: .now() + .seconds(seconds), execute: workItem) + } + + func cancelHeadUpTimeout() { + headUpMicTimeoutWorkItem?.cancel() + headUpMicTimeoutWorkItem = nil + headUpMicTimeoutElapsed = false + micBlockedByTimeout = false + + Bridge.log("Mentra: HEAD: timeout canceled and elapsed cleared") + } + + func recomputeMicAndUI(_ reason: HeadRecomputeReason) { + Bridge.log( + "Mentra: RECOMPUTE: reason=\(reason.rawValue), rawHeadUp=\(isHeadUp), effectiveHeadUp=\(isHeadUpEffective), hasFG=\(hasForegroundAppOpen), timeoutEnabled=\(headUpMicTimeoutEnabled), timeoutElapsed=\(headUpMicTimeoutElapsed)" + ) + + handle_microphone_state_change(currentRequiredData, bypassVadForPCM) + sendCurrentState(isHeadUpEffective) + } +} diff --git a/mobile/ios/Source/MentraManager+Mic.swift b/mobile/ios/Source/MentraManager+Mic.swift new file mode 100644 index 0000000000..67c64ee7e2 --- /dev/null +++ b/mobile/ios/Source/MentraManager+Mic.swift @@ -0,0 +1,316 @@ +// +// MentraManager+Mic.swift +// MentraOS_Manager +// +// Created by Codex on 3/17/24. +// + +import Foundation + +#if canImport(UIKit) + import UIKit +#endif + +extension MentraManager { + func handleGlassesMicData(_ rawLC3Data: Data) { + guard rawLC3Data.count > 2 else { + Bridge.log("Received invalid PCM data size: \(rawLC3Data.count)") + return + } + + let lc3Data = rawLC3Data.subdata(in: 2 ..< rawLC3Data.count) + guard !lc3Data.isEmpty else { + Bridge.log("No LC3 data after removing command bytes") + return + } + + if bypassVad || bypassVadForPCM { + Bridge.log( + "Mentra: Glasses mic VAD bypassed - bypassVad=\(bypassVad), bypassVadForPCM=\(bypassVadForPCM)" + ) + checkSetVadStatus(speaking: true) + emptyVadBuffer() + + let pcmData = PcmConverter().decode(lc3Data) as Data + Bridge.sendMicData(pcmData) + return + } + + let pcmData = PcmConverter().decode(lc3Data) as Data + guard !pcmData.isEmpty else { + Bridge.log("PCM conversion resulted in empty data") + return + } + + guard let vad else { + Bridge.log("VAD not initialized") + return + } + + let pcmDataArray = pcmData.withUnsafeBytes { pointer -> [Int16] in + Array( + UnsafeBufferPointer( + start: pointer.bindMemory(to: Int16.self).baseAddress, + count: pointer.count / MemoryLayout.stride + )) + } + + vad.checkVAD(pcm: pcmDataArray) { [weak self] state in + guard let self else { return } + Bridge.log("VAD State: \(state)") + } + + if vad.currentState() == .speeching { + checkSetVadStatus(speaking: true) + emptyVadBuffer() + Bridge.sendMicData(pcmData) + } else { + checkSetVadStatus(speaking: false) + addToVadBuffer(pcmData) + } + } + + func handlePcm(_ pcmData: Data) { + guard let vad else { + Bridge.log("VAD not initialized") + return + } + + if bypassVad || bypassVadForPCM { + if shouldSendPcmData { + Bridge.sendMicData(pcmData) + } + + if shouldSendTranscript { + transcriber?.acceptAudio(pcm16le: pcmData) + } + return + } + + let pcmDataArray = pcmData.withUnsafeBytes { pointer -> [Int16] in + Array( + UnsafeBufferPointer( + start: pointer.bindMemory(to: Int16.self).baseAddress, + count: pointer.count / MemoryLayout.stride + )) + } + + vad.checkVAD(pcm: pcmDataArray) { [weak self] state in + guard let self else { return } + Bridge.log("VAD State: \(state)") + } + + if vad.currentState() == .speeching { + checkSetVadStatus(speaking: true) + emptyVadBuffer() + + if shouldSendPcmData { + Bridge.sendMicData(pcmData) + } + + if shouldSendTranscript { + transcriber?.acceptAudio(pcm16le: pcmData) + } + } else { + checkSetVadStatus(speaking: false) + addToVadBuffer(pcmData) + } + } + + func handle_microphone_state_change(_ requiredData: [SpeechRequiredDataType], _ bypassVad: Bool) { + var normalizedRequiredData = normalizeRequiredData(requiredData) + Bridge.log( + "Mentra: MIC: @@@@@@@@ changing mic with requiredData: \(normalizedRequiredData) bypassVad=\(bypassVad) enforceLocalTranscription=\(enforceLocalTranscription) @@@@@@@@@@@@@@@@" + ) + Bridge.log( + "Mentra: MIC: state before decision -> micEnabled=\(micEnabled), micSessionActive=\(micSessionActive), sensingEnabled=\(sensingEnabled), isHeadUp=\(isHeadUp), hasForegroundAppOpen=\(hasForegroundAppOpen), micActivationMode=\(micActivationMode.rawValue), headUpTimeoutEnabled=\(headUpMicTimeoutEnabled), headUpTimeoutElapsed=\(headUpMicTimeoutElapsed)" + ) + + let outputs = resolveSpeechOutputs(for: normalizedRequiredData) + shouldSendPcmData = outputs.sendPcm + shouldSendTranscript = outputs.sendTranscript + + currentRequiredData = normalizedRequiredData + vadBuffer.removeAll() + + micEnabled = !normalizedRequiredData.isEmpty + + let requestActive = micEnabled + let sensorsAllowMic = sensingEnabled + let headUp = isHeadUp + + if !requestActive || !sensorsAllowMic { + micSessionActive = false + } + + var shouldEnableMic = requestActive && sensorsAllowMic + let micBlocked = + micBlockedByTimeout && micActivationMode.requiresHeadUp && !hasForegroundAppOpen + + if micBlocked { + shouldEnableMic = false + micSessionActive = false + } else if shouldEnableMic { + if micActivationMode == .alwaysOn || hasForegroundAppOpen || isHeadUpEffective { + micSessionActive = true + } else { + shouldEnableMic = false + micSessionActive = false + } + } + + Bridge.log( + "Mentra: MIC: mid decision -> requestActive=\(requestActive), sensorsAllowMic=\(sensorsAllowMic), rawHeadUp=\(headUp), effectiveHeadUp=\(isHeadUpEffective), hasForegroundAppOpen=\(hasForegroundAppOpen), micActivationMode=\(micActivationMode.rawValue), micBlockedByTimeout=\(micBlockedByTimeout), shouldEnableMic=\(shouldEnableMic), micSessionActive=\(micSessionActive)" + ) + + let allowMicSession = shouldEnableMic + + Task { + let isBackground: Bool + #if canImport(UIKit) + isBackground = UIApplication.shared.applicationState == .background + #else + isBackground = false + #endif + + let glassesHasMic = (sgc?.hasMic ?? getGlassesHasMic()) && (sgc?.ready == true) + + // Determine if mic should be active at all + var shouldBeActive = + allowMicSession + && !(micBlockedByTimeout && micActivationMode.requiresHeadUp + && !hasForegroundAppOpen) + + if !shouldBeActive { + // Ensure everything is off + await sgc?.setMicEnabled(false) + setOnboardMicEnabled(false) + micSessionActive = false + Bridge.log("Mentra: MIC: final decision -> disabled (blocked/toggled off)") + return + } + + // Choose audio source (favor glasses when available for stability) + var useGlassesMic = false + var useOnboardMic = false + + switch preferredMic { + case .glasses: + useGlassesMic = glassesHasMic + useOnboardMic = !glassesHasMic && !onboardMicUnavailable + case .phone: + // Auto-upgrade to glasses mic if available and keep it active persistently + if glassesHasMic { + preferredMic = .glasses + Bridge.saveSetting("preferred_mic", "glasses") + useGlassesMic = true + useOnboardMic = false + } else { + useOnboardMic = !onboardMicUnavailable + } + } + + // iOS background: if background and phone mic requested, prefer glasses when present + if isBackground && !useGlassesMic && glassesHasMic { + useGlassesMic = true + useOnboardMic = false + } + + micSessionActive = true + + Bridge.log( + "Mentra: MIC: final decision -> appState=\(isBackground ? "background" : "foreground"), preferred=\(preferredMic.rawValue), glassesHasMic=\(glassesHasMic), useGlassesMic=\(useGlassesMic), useOnboardMic=\(useOnboardMic), onboardMicUnavailable=\(onboardMicUnavailable)" + ) + + if sgc?.hasMic ?? false { + await sgc?.setMicEnabled(useGlassesMic) + } + + setOnboardMicEnabled(useOnboardMic) + } + } + + func setOnboardMicEnabled(_ isEnabled: Bool) { + Task { + if isEnabled { + guard PhoneMic.shared.checkPermissions() else { + Bridge.log("Microphone permissions not granted. Cannot enable microphone.") + return + } + + let success = PhoneMic.shared.startRecording() + if !success, getGlassesHasMic() { + await enableGlassesMic(true) + } + } else { + PhoneMic.shared.stopRecording() + } + } + } + + func enableGlassesMic(_: Bool) async { + await sgc?.setMicEnabled(true) + } +} + +// MARK: - Private helpers + +private extension MentraManager { + func normalizeRequiredData(_ requiredData: [SpeechRequiredDataType]) + -> [SpeechRequiredDataType] + { + var requiredData = requiredData + if offlineModeEnabled, !requiredData.contains(.PCM_OR_TRANSCRIPTION), + !requiredData.contains(.TRANSCRIPTION) + { + requiredData.append(.TRANSCRIPTION) + } + return requiredData + } + + func resolveSpeechOutputs(for requiredData: [SpeechRequiredDataType]) + -> (sendPcm: Bool, sendTranscript: Bool) + { + var sendPcm = false + var sendTranscript = false + + if requiredData.contains(.PCM) && requiredData.contains(.TRANSCRIPTION) { + sendPcm = true + sendTranscript = true + } else if requiredData.contains(.PCM) { + sendPcm = true + } else if requiredData.contains(.TRANSCRIPTION) { + sendTranscript = true + } else if requiredData.contains(.PCM_OR_TRANSCRIPTION) { + if enforceLocalTranscription { + sendTranscript = true + } else { + sendPcm = true + } + } + + return (sendPcm, sendTranscript) + } + + func checkSetVadStatus(speaking: Bool) { + if speaking != isSpeaking { + isSpeaking = speaking + Bridge.sendVadStatus(isSpeaking) + } + } + + func emptyVadBuffer() { + while !vadBuffer.isEmpty { + let chunk = vadBuffer.removeFirst() + Bridge.sendMicData(chunk) + } + } + + func addToVadBuffer(_ chunk: Data) { + let maxBufferSize = 20 + vadBuffer.append(chunk) + while vadBuffer.count > maxBufferSize { + vadBuffer.removeFirst() + } + } +} diff --git a/mobile/ios/Source/MentraManager+Status.swift b/mobile/ios/Source/MentraManager+Status.swift new file mode 100644 index 0000000000..f2c14b0b3f --- /dev/null +++ b/mobile/ios/Source/MentraManager+Status.swift @@ -0,0 +1,275 @@ +// +// MentraManager+Status.swift +// MentraOS_Manager +// +// Created by Codex on 3/17/24. +// + +import Foundation + +extension MentraManager { + func handle_request_status() { + let simulatedConnected = defaultWearable == "Simulated Glasses" + let isGlassesConnected = sgc?.ready ?? false + if isGlassesConnected { + isSearching = false + } + + let connectedGlasses = buildConnectedGlassesInfo( + isGlassesConnected: isGlassesConnected, simulatedConnected: simulatedConnected + ) + let glassesSettings = buildGlassesSettings() + + let coreInfo: [String: Any] = [ + "augmentos_core_version": "Unknown", + "default_wearable": defaultWearable as Any, + "preferred_mic": preferredMic.rawValue, + "is_searching": isSearching, + "is_mic_enabled_for_frontend": micEnabled && preferredMic == .glasses + && isSomethingConnected(), + "sensing_enabled": sensingEnabled, + "power_saving_mode": powerSavingMode, + "always_on_status_bar": alwaysOnStatusBar, + "bypass_vad_for_debugging": bypassVad, + "enforce_local_transcription": enforceLocalTranscription, + "bypass_audio_encoding_for_debugging": bypassAudioEncoding, + "core_token": coreToken, + "puck_connected": true, + "metric_system_enabled": metricSystemEnabled, + "contextual_dashboard_enabled": contextualDashboard, + "head_up_mic_timeout_enabled": headUpMicTimeoutEnabled, + "head_up_mic_timeout_seconds": headUpMicTimeoutSeconds, + "mic_blocked_by_timeout": micBlockedByTimeout, + ] + + let authObj: [String: Any] = ["core_token_owner": coreTokenOwner] + + let statusObj: [String: Any] = [ + "connected_glasses": connectedGlasses, + "glasses_settings": glassesSettings, + "apps": [[String: Any]](), + "core_info": coreInfo, + "auth": authObj, + ] + + lastStatusObj = statusObj + Bridge.sendStatus(statusObj) + } + + func triggerStatusUpdate() { + Bridge.log("🔄 Triggering immediate status update") + handle_request_status() + } + + func handle_update_settings(_ settings: [String: Any]) { + Bridge.log("Mentra: Received update settings: \(settings)") + + if let newPreferredMic = settings["preferred_mic"] as? String { + let preference = MicPreference.from(newPreferredMic) + if preference != preferredMic { + setPreferredMic(newPreferredMic) + } + } + + if let newMicActivationMode = settings["mic_activation_mode"] as? String { + let activationMode = MicActivationMode.from(newMicActivationMode) + if activationMode != micActivationMode { + setMicActivationMode(newMicActivationMode) + } + } + + if let newHeadUpAngle = settings["head_up_angle"] as? Int, newHeadUpAngle != headUpAngle { + updateGlassesHeadUpAngle(newHeadUpAngle) + } + + if let newBrightness = settings["brightness"] as? Int, newBrightness != brightness { + updateGlassesBrightness(newBrightness, autoBrightness: false) + } + + if let newDashboardHeight = settings["dashboard_height"] as? Int, + newDashboardHeight != dashboardHeight + { + updateGlassesHeight(newDashboardHeight) + } + + if let newDashboardDepth = settings["dashboard_depth"] as? Int, + newDashboardDepth != dashboardDepth + { + updateGlassesDepth(newDashboardDepth) + } + + if let newAutoBrightness = settings["auto_brightness"] as? Bool, + newAutoBrightness != autoBrightness + { + updateGlassesBrightness(brightness, autoBrightness: newAutoBrightness) + } + + if let sensingEnabled = settings["sensing_enabled"] as? Bool, + sensingEnabled != self.sensingEnabled + { + enableSensing(sensingEnabled) + } + + if let powerSavingMode = settings["power_saving_mode"] as? Bool, + powerSavingMode != self.powerSavingMode + { + enablePowerSavingMode(powerSavingMode) + } + + if let newAlwaysOnStatusBar = settings["always_on_status_bar_enabled"] as? Bool, + newAlwaysOnStatusBar != alwaysOnStatusBar + { + enableAlwaysOnStatusBar(newAlwaysOnStatusBar) + } + + if let timeoutEnabled = settings["head_up_mic_timeout_enabled"] as? Bool { + headUpMicTimeoutEnabled = timeoutEnabled + if !timeoutEnabled { + // Disabling the feature immediately cancels any pending timeout + cancelHeadUpTimeout() + } else { + // Enabling while head is already up: only schedule if eligible. + // Do NOT reset elapsed/blocked to preserve one-shot semantics. + scheduleHeadUpTimeoutIfNeeded() + } + } + if let timeoutSeconds = settings["head_up_mic_timeout_seconds"] as? Int { + headUpMicTimeoutSeconds = max(5, min(300, timeoutSeconds)) + // Do NOT reset elapsed/blocked when changing duration; honor one-shot semantics. + // Only schedule if currently eligible. + scheduleHeadUpTimeoutIfNeeded() + } + + if let newBypassVad = settings["bypass_vad_for_debugging"] as? Bool, + newBypassVad != bypassVad + { + bypassVad(newBypassVad) + } + + if let newEnforceLocalTranscription = settings["enforce_local_transcription"] as? Bool, + newEnforceLocalTranscription != enforceLocalTranscription + { + enforceLocalTranscription(newEnforceLocalTranscription) + } + + if let newEnableOfflineMode = settings["offline_captions_app_running"] as? Bool, + newEnableOfflineMode != offlineModeEnabled + { + enableOfflineMode(newEnableOfflineMode) + } + + if let newMetricSystemEnabled = settings["metric_system_enabled"] as? Bool, + newMetricSystemEnabled != metricSystemEnabled + { + setMetricSystemEnabled(newMetricSystemEnabled) + } + + if let newContextualDashboard = settings["contextual_dashboard_enabled"] as? Bool, + newContextualDashboard != contextualDashboard + { + enableContextualDashboard(newContextualDashboard) + } + + if let newButtonMode = settings["button_mode"] as? String, newButtonMode != buttonPressMode { + setButtonMode(newButtonMode) + } + + if let newFps = settings["button_video_fps"] as? Int, newFps != buttonVideoFps { + setButtonVideoSettings(width: buttonVideoWidth, height: buttonVideoHeight, fps: newFps) + } + + if let newWidth = settings["button_video_width"] as? Int, newWidth != buttonVideoWidth { + setButtonVideoSettings(width: newWidth, height: buttonVideoHeight, fps: buttonVideoFps) + } + + if let newHeight = settings["button_video_height"] as? Int, newHeight != buttonVideoHeight { + setButtonVideoSettings(width: buttonVideoWidth, height: newHeight, fps: buttonVideoFps) + } + + if let newPhotoSize = settings["button_photo_size"] as? String, + newPhotoSize != buttonPhotoSize + { + setButtonPhotoSize(newPhotoSize) + } + + if let newDefaultWearable = settings["default_wearable"] as? String, + newDefaultWearable != defaultWearable + { + defaultWearable = newDefaultWearable + Bridge.saveSetting("default_wearable", newDefaultWearable) + } + } +} + +private extension MentraManager { + func buildConnectedGlassesInfo(isGlassesConnected: Bool, simulatedConnected: Bool) + -> [String: Any] + { + var connectedGlasses: [String: Any] = [:] + + if isGlassesConnected { + connectedGlasses = [ + "model_name": defaultWearable, + "battery_level": sgc?.batteryLevel ?? -1, + "glasses_app_version": sgc?.glassesAppVersion ?? "", + "glasses_build_number": sgc?.glassesBuildNumber ?? "", + "glasses_device_model": sgc?.glassesDeviceModel ?? "", + "glasses_android_version": sgc?.glassesAndroidVersion ?? "", + "glasses_ota_version_url": sgc?.glassesOtaVersionUrl ?? "", + ] + } else if simulatedConnected { + connectedGlasses["model_name"] = defaultWearable + } + + if sgc is G1 { + connectedGlasses["case_removed"] = sgc?.caseRemoved ?? true + connectedGlasses["case_open"] = sgc?.caseOpen ?? true + connectedGlasses["case_charging"] = sgc?.caseCharging ?? false + connectedGlasses["case_battery_level"] = sgc?.caseBatteryLevel ?? -1 + + if let serialNumber = sgc?.glassesSerialNumber, !serialNumber.isEmpty { + connectedGlasses["glasses_serial_number"] = serialNumber + connectedGlasses["glasses_style"] = sgc?.glassesStyle ?? "" + connectedGlasses["glasses_color"] = sgc?.glassesColor ?? "" + } + } + + if let live = sgc as? MentraLive { + if let wifiSsid = live.wifiSsid, !wifiSsid.isEmpty { + connectedGlasses["glasses_wifi_ssid"] = wifiSsid + connectedGlasses["glasses_wifi_connected"] = live.wifiConnected + connectedGlasses["glasses_wifi_local_ip"] = live.wifiLocalIp ?? "" + } + + connectedGlasses["glasses_hotspot_enabled"] = live.isHotspotEnabled ?? false + connectedGlasses["glasses_hotspot_ssid"] = live.hotspotSsid ?? "" + connectedGlasses["glasses_hotspot_password"] = live.hotspotPassword ?? "" + connectedGlasses["glasses_hotspot_gateway_ip"] = live.hotspotGatewayIp ?? "" + } + + if let bluetoothName = sgc?.getConnectedBluetoothName() { + connectedGlasses["bluetooth_name"] = bluetoothName + } + + return connectedGlasses + } + + func buildGlassesSettings() -> [String: Any] { + [ + "brightness": brightness, + "auto_brightness": autoBrightness, + "dashboard_height": dashboardHeight, + "dashboard_depth": dashboardDepth, + "head_up_angle": headUpAngle, + "button_mode": buttonPressMode, + "button_photo_size": buttonPhotoSize, + "button_video_settings": [ + "width": buttonVideoWidth, + "height": buttonVideoHeight, + "fps": buttonVideoFps, + ], + "button_max_recording_time_minutes": buttonMaxRecordingTimeMinutes, + "button_camera_led": buttonCameraLed, + ] + } +} diff --git a/mobile/ios/Source/MentraManager.swift b/mobile/ios/Source/MentraManager.swift index e2b51c23c2..731857f44e 100644 --- a/mobile/ios/Source/MentraManager.swift +++ b/mobile/ios/Source/MentraManager.swift @@ -22,6 +22,39 @@ struct ViewState { var animationData: [String: Any]? } +enum MicPreference: String { + case glasses + case phone + + static func from(_ value: String) -> MicPreference { + MicPreference(rawValue: value) ?? .glasses + } +} + +enum MicActivationMode: String { + case headUp = "head_up" + case alwaysOn = "always_on" + + static func from(_ value: String) -> MicActivationMode { + MicActivationMode(rawValue: value) ?? .headUp + } + + var requiresHeadUp: Bool { self == .headUp } +} + +enum MentraDisplayLayout: String { + case textWall = "text_wall" + case doubleTextWall = "double_text_wall" + case referenceCard = "reference_card" + case bitmap = "bitmap_view" + case clear = "clear_view" + case unknown + + init(from rawValue: String) { + self = MentraDisplayLayout(rawValue: rawValue) ?? .unknown + } +} + // This class handles logic for managing devices and connections to AugmentOS servers @objc(MentraManager) class MentraManager: NSObject { static let shared = MentraManager() @@ -31,48 +64,60 @@ struct ViewState { } var coreToken: String = "" - private var coreTokenOwner: String = "" + var coreTokenOwner: String = "" var sgc: SGCManager? - private var lastStatusObj: [String: Any] = [:] - - private var cancellables = Set() - private var defaultWearable: String = "" - private var pendingWearable: String = "" - private var deviceName: String = "" - private var contextualDashboard = true - private var headUpAngle = 30 - private var brightness = 50 - private var autoBrightness: Bool = true - private var dashboardHeight: Int = 4 - private var dashboardDepth: Int = 5 - private var sensingEnabled: Bool = true - private var powerSavingMode: Bool = false - private var isSearching: Bool = false - private var isUpdatingScreen: Bool = false - private var alwaysOnStatusBar: Bool = false - private var bypassVad: Bool = true - private var bypassVadForPCM: Bool = false // NEW: PCM subscription bypass - private var enforceLocalTranscription: Bool = false - private var offlineModeEnabled: Bool = false - private var bypassAudioEncoding: Bool = false - private var onboardMicUnavailable: Bool = false - private var metricSystemEnabled: Bool = false - private var settingsLoaded = false - private let settingsLoadedSemaphore = DispatchSemaphore(value: 0) - private var connectTask: Task? - private var glassesWifiConnected: Bool = false - private var glassesWifiSsid: String = "" - private var isHeadUp: Bool = false - private var sendStateWorkItem: DispatchWorkItem? - private let sendStateQueue = DispatchQueue(label: "sendStateQueue", qos: .userInitiated) - private var shouldSendBootingMessage = true + var lastStatusObj: [String: Any] = [:] + + var cancellables = Set() + var defaultWearable: String = "" + var pendingWearable: String = "" + var deviceName: String = "" + var contextualDashboard = true + var headUpAngle = 30 + var brightness = 50 + var autoBrightness: Bool = true + var dashboardHeight: Int = 4 + var dashboardDepth: Int = 5 + var sensingEnabled: Bool = true + var powerSavingMode: Bool = false + var isSearching: Bool = false + var isUpdatingScreen: Bool = false + var alwaysOnStatusBar: Bool = false + var bypassVad: Bool = true + var bypassVadForPCM: Bool = false // NEW: PCM subscription bypass + var enforceLocalTranscription: Bool = false + var offlineModeEnabled: Bool = false + var bypassAudioEncoding: Bool = false + var onboardMicUnavailable: Bool = false + var metricSystemEnabled: Bool = false + var settingsLoaded = false + let settingsLoadedSemaphore = DispatchSemaphore(value: 0) + var connectTask: Task? + var glassesWifiConnected: Bool = false + var glassesWifiSsid: String = "" + var isHeadUp: Bool = false + var sendStateWorkItem: DispatchWorkItem? + let sendStateQueue = DispatchQueue(label: "sendStateQueue", qos: .userInitiated) + var shouldSendBootingMessage = true // mic: - private var useOnboardMic = false - private var preferredMic = "glasses" - private var micEnabled = false - private var currentRequiredData: [SpeechRequiredDataType] = [] + var useOnboardMic = false + var preferredMic: MicPreference = .glasses + var micEnabled = false + var micSessionActive = false + var micActivationMode: MicActivationMode = .headUp + var currentRequiredData: [SpeechRequiredDataType] = [] + // Track whether any foreground (standard) app is open and running + var hasForegroundAppOpen = false + + // Head-up mic timeout (auto-disable after grace period) + var headUpMicTimeoutWorkItem: DispatchWorkItem? + var headUpMicTimeoutElapsed = false + var headUpMicTimeoutSeconds: Int = 20 + var headUpMicTimeoutEnabled: Bool = true + // When true, mic/UI should be gated until head goes down or a foreground app opens + var micBlockedByTimeout: Bool = false // button settings: var buttonPressMode = "photo" @@ -84,14 +129,14 @@ struct ViewState { var buttonCameraLed = true // VAD: - private var vad: SileroVADStrategy? - private var vadBuffer = [Data]() - private var isSpeaking = false + var vad: SileroVADStrategy? + var vadBuffer = [Data]() + var isSpeaking = false // STT: - private var transcriber: SherpaOnnxTranscriber? - private var shouldSendPcmData = false - private var shouldSendTranscript = false + var transcriber: SherpaOnnxTranscriber? + var shouldSendPcmData = false + var shouldSendTranscript = false var viewStates: [ViewState] = [ ViewState( @@ -146,107 +191,6 @@ struct ViewState { // MARK: - Public Methods (for React Native) - func initSGC(_ wearable: String) { - Bridge.log("Initializing manager for wearable: \(wearable)") - if wearable.contains("G1") && sgc == nil { - sgc = G1() - } else if wearable.contains("Live") && sgc == nil { - sgc = MentraLive() - } else if wearable.contains("Mach1") && sgc == nil { - sgc = Mach1() - } else if wearable.contains("Frame") || wearable.contains("Brilliant Labs"), - sgc == nil - { - sgc = FrameManager() - } - } - - func initSGCCallbacks() { - // TODO: make sure this functionality is baked into the SGCs! - - // if sgc is MentraLive { - // let live = sgc as? MentraLive - // live!.onConnectionStateChanged = { [weak self] in - // guard let self = self else { return } - // Bridge.log( - // "Live glasses connection changed to: \(live!.ready ? "Connected" : "Disconnected")" - // ) - // if live!.ready { - // handleDeviceReady() - // } else { - // handleDeviceDisconnected() - // handle_request_status() - // } - // } - // - // live!.$batteryLevel.sink { [weak self] (level: Int) in - // guard let self = self else { return } - // guard level >= 0 else { return } - // self.batteryLevel = level - // Bridge.sendBatteryStatus(level: self.batteryLevel, charging: false) - // handle_request_status() - // }.store(in: &cancellables) - // - // live!.$wifiConnected.sink { [weak self] (isConnected: Bool) in - // guard let self = self else { return } - // self.glassesWifiConnected = isConnected - // handle_request_status() - // }.store(in: &cancellables) - // - // live!.onButtonPress = { [weak self] (buttonId: String, pressType: String) in - // guard let self = self else { return } - // Bridge.sendButtonPress(buttonId: buttonId, pressType: pressType) - // } - // live!.onPhotoRequest = { [weak self] (requestId: String, photoUrl: String) in - // guard let self = self else { return } - // Bridge.sendPhotoResponse(requestId: requestId, photoUrl: photoUrl) - // } - // live!.onVideoStreamResponse = { [weak self] (appId: String, streamUrl: String) in - // guard let self = self else { return } - // Bridge.sendVideoStreamResponse(appId: appId, streamUrl: streamUrl) - // } - // } - // - // if sgc is Mach1 { - // let mach1 = sgc as? Mach1 - // mach1!.onConnectionStateChanged = { [weak self] in - // guard let self = self else { return } - // Bridge.log( - // "Mach1 glasses connection changed to: \(mach1!.ready ? "Connected" : "Disconnected")" - // ) - // if mach1!.ready { - // handleDeviceReady() - // } else { - // handleDeviceDisconnected() - // handle_request_status() - // } - // } - // - // mach1!.$batteryLevel.sink { [weak self] (level: Int) in - // guard let self = self else { return } - // guard level >= 0 else { return } - // self.batteryLevel = level - // Bridge.sendBatteryStatus(level: self.batteryLevel, charging: false) - // handle_request_status() - // }.store(in: &cancellables) - // - // mach1!.$isHeadUp.sink { [weak self] (value: Bool) in - // guard let self = self else { return } - // updateHeadUp(value) - // }.store(in: &cancellables) - // } - } - - func updateHeadUp(_ isHeadUp: Bool) { - self.isHeadUp = isHeadUp - sendCurrentState(isHeadUp) - Bridge.sendHeadUp(isHeadUp) - } - - func onAppStateChange(_: [ThirdPartyCloudApp]) { - handle_request_status() - } - func onConnectionError(_: String) { handle_request_status() } @@ -255,305 +199,15 @@ struct ViewState { // MARK: - Voice Data Handling - private func checkSetVadStatus(speaking: Bool) { - if speaking != isSpeaking { - isSpeaking = speaking - Bridge.sendVadStatus(isSpeaking) - } - } - - private func emptyVadBuffer() { - // go through the buffer, popping from the first element in the array (FIFO): - while !vadBuffer.isEmpty { - let chunk = vadBuffer.removeFirst() - Bridge.sendMicData(chunk) - } - } - - private func addToVadBuffer(_ chunk: Data) { - let MAX_BUFFER_SIZE = 20 - vadBuffer.append(chunk) - while vadBuffer.count > MAX_BUFFER_SIZE { - // pop from the front of the array: - vadBuffer.removeFirst() - } - } - - func handleGlassesMicData(_ rawLC3Data: Data) { - // decode the g1 audio data to PCM and feed to the VAD: - - // Ensure we have enough data to process - guard rawLC3Data.count > 2 else { - Bridge.log("Received invalid PCM data size: \(rawLC3Data.count)") - return - } - - // Skip the first 2 bytes which are command bytes - let lc3Data = rawLC3Data.subdata(in: 2 ..< rawLC3Data.count) - - // Ensure we have valid PCM data - guard lc3Data.count > 0 else { - Bridge.log("No LC3 data after removing command bytes") - return - } - - if bypassVad || bypassVadForPCM { - Bridge.log( - "Mentra: Glasses mic VAD bypassed - bypassVad=\(bypassVad), bypassVadForPCM=\(bypassVadForPCM)" - ) - checkSetVadStatus(speaking: true) - // first send out whatever's in the vadBuffer (if there is anything): - emptyVadBuffer() - let pcmConverter = PcmConverter() - let pcmData = pcmConverter.decode(lc3Data) as Data - // self.serverComms.sendAudioChunk(lc3Data) - Bridge.sendMicData(pcmData) - return - } - - let pcmConverter = PcmConverter() - let pcmData = pcmConverter.decode(lc3Data) as Data - - guard pcmData.count > 0 else { - Bridge.log("PCM conversion resulted in empty data") - return - } - - // feed PCM to the VAD: - guard let vad = vad else { - Bridge.log("VAD not initialized") - return - } - - // convert audioData to Int16 array: - let pcmDataArray = pcmData.withUnsafeBytes { pointer -> [Int16] in - Array( - UnsafeBufferPointer( - start: pointer.bindMemory(to: Int16.self).baseAddress, - count: pointer.count / MemoryLayout.stride - )) - } - - vad.checkVAD(pcm: pcmDataArray) { [weak self] state in - guard let self = self else { return } - Bridge.log("VAD State: \(state)") - } - - let vadState = vad.currentState() - if vadState == .speeching { - checkSetVadStatus(speaking: true) - // first send out whatever's in the vadBuffer (if there is anything): - emptyVadBuffer() - // self.serverComms.sendAudioChunk(lc3Data) - Bridge.sendMicData(pcmData) - } else { - checkSetVadStatus(speaking: false) - // add to the vadBuffer: - // addToVadBuffer(lc3Data) - addToVadBuffer(pcmData) - } - } - - func handlePcm(_ pcmData: Data) { - // handle incoming PCM data from the microphone manager and feed to the VAD: - - // feed PCM to the VAD: - guard let vad = vad else { - Bridge.log("VAD not initialized") - return - } - - if bypassVad || bypassVadForPCM { - // let pcmConverter = PcmConverter() - // let lc3Data = pcmConverter.encode(pcmData) as Data - // checkSetVadStatus(speaking: true) - // // first send out whatever's in the vadBuffer (if there is anything): - // emptyVadBuffer() - // self.serverComms.sendAudioChunk(lc3Data) - if shouldSendPcmData { - // Bridge.log("Mentra: Sending PCM data to server") - Bridge.sendMicData(pcmData) - } - - // Also send to local transcriber when bypassing VAD - if shouldSendTranscript { - transcriber?.acceptAudio(pcm16le: pcmData) - } - return - } - - // convert audioData to Int16 array: - let pcmDataArray = pcmData.withUnsafeBytes { pointer -> [Int16] in - Array( - UnsafeBufferPointer( - start: pointer.bindMemory(to: Int16.self).baseAddress, - count: pointer.count / MemoryLayout.stride - )) - } - - vad.checkVAD(pcm: pcmDataArray) { [weak self] state in - guard let self = self else { return } - // self.handler?(state) - Bridge.log("VAD State: \(state)") - } - - // encode the pcmData as LC3: - // let pcmConverter = PcmConverter() - // let lc3Data = pcmConverter.encode(pcmData) as Data - - let vadState = vad.currentState() - if vadState == .speeching { - checkSetVadStatus(speaking: true) - // first send out whatever's in the vadBuffer (if there is anything): - emptyVadBuffer() - // self.serverComms.sendAudioChunk(lc3Data) - if shouldSendPcmData { - Bridge.sendMicData(pcmData) - } - - // Send to local transcriber when speech is detected - if shouldSendTranscript { - transcriber?.acceptAudio(pcm16le: pcmData) - } - } else { - checkSetVadStatus(speaking: false) - // add to the vadBuffer: - // addToVadBuffer(lc3Data) - addToVadBuffer(pcmData) - } - } - - func handleConnectionStateChange() { - Bridge.log("Mentra: Glasses: connection state changed!") - if sgc == nil { return } - if sgc!.ready { - handleDeviceReady() - } else { - handleDeviceDisconnected() - handle_request_status() - } - } - // MARK: - ServerCommsCallback Implementation - func handle_microphone_state_change(_ requiredData: [SpeechRequiredDataType], _ bypassVad: Bool) { - var requiredData = requiredData // make mutable - Bridge.log( - "Mentra: MIC: @@@@@@@@ changing mic with requiredData: \(requiredData) bypassVad=\(bypassVad) enforceLocalTranscription=\(enforceLocalTranscription) @@@@@@@@@@@@@@@@" - ) - - bypassVadForPCM = bypassVad - - shouldSendPcmData = false - shouldSendTranscript = false - - // this must be done before the requiredData is modified by offlineStt: - currentRequiredData = requiredData - - if offlineModeEnabled, !requiredData.contains(.PCM_OR_TRANSCRIPTION), - !requiredData.contains(.TRANSCRIPTION) - { - requiredData.append(.TRANSCRIPTION) - } - - if requiredData.contains(.PCM), requiredData.contains(.TRANSCRIPTION) { - shouldSendPcmData = true - shouldSendTranscript = true - } else if requiredData.contains(.PCM) { - shouldSendPcmData = true - shouldSendTranscript = false - } else if requiredData.contains(.TRANSCRIPTION) { - shouldSendTranscript = true - shouldSendPcmData = false - } else if requiredData.contains(.PCM_OR_TRANSCRIPTION) { - // TODO: Later add bandwidth based logic - if enforceLocalTranscription { - shouldSendTranscript = true - shouldSendPcmData = false - } else { - shouldSendPcmData = true - shouldSendTranscript = false - } - } - - // Core.log("Mentra: MIC: shouldSendPcmData=\(shouldSendPcmData), shouldSendTranscript=\(shouldSendTranscript)") - - // in any case, clear the vadBuffer: - vadBuffer.removeAll() - micEnabled = !requiredData.isEmpty - - // Handle microphone state change if needed - Task { - // Only enable microphone if sensing is also enabled - var actuallyEnabled = micEnabled && self.sensingEnabled - - let glassesHasMic = sgc?.hasMic ?? false - - var useGlassesMic = false - var useOnboardMic = false - - useOnboardMic = self.preferredMic == "phone" - useGlassesMic = self.preferredMic == "glasses" - - if self.onboardMicUnavailable { - useOnboardMic = false - } - - if !glassesHasMic { - useGlassesMic = false - } - - if !useGlassesMic, !useOnboardMic { - // if we have a non-preferred mic, use it: - if glassesHasMic { - useGlassesMic = true - } else if !self.onboardMicUnavailable { - useOnboardMic = true - } - - if !useGlassesMic, !useOnboardMic { - Bridge.log( - "Mentra: no mic to use! falling back to glasses mic!!!!! (this should not happen)" - ) - useGlassesMic = true - } - } - - let appState = UIApplication.shared.applicationState - if appState == .background { - Bridge.log("App is in background - onboard mic unavailable to start!") - if useOnboardMic { - // if we're using the onboard mic and already recording, simply return as we shouldn't interrupt - // the audio session - if PhoneMic.shared.isRecording { - return - } - - // if we want to use the onboard mic but aren't currently recording, switch to using the glasses mic - // instead since we won't be able to start the mic from the background - useGlassesMic = true - useOnboardMic = false - } - } - - // preferred state: - useGlassesMic = actuallyEnabled && useGlassesMic - useOnboardMic = actuallyEnabled && useOnboardMic - - // Core.log( - // "Mentra: MIC: isEnabled: \(isEnabled) sensingEnabled: \(self.sensingEnabled) useOnboardMic: \(useOnboardMic) " + - // "useGlassesMic: \(useGlassesMic) glassesHasMic: \(glassesHasMic) preferredMic: \(self.preferredMic) " + - // "somethingConnected: \(isSomethingConnected()) onboardMicUnavailable: \(self.onboardMicUnavailable)" + - // "actuallyEnabled: \(actuallyEnabled)" - // ) - - // if a g1 is connected, set the mic enabled: - if sgc?.hasMic ?? false, sgc!.ready { - await sgc!.setMicEnabled(useGlassesMic) - } - - setOnboardMicEnabled(useOnboardMic) + func setMicActivationMode(_ mode: String) { + let normalized = MicActivationMode.from(mode) + if micActivationMode == normalized { + return } + micActivationMode = normalized + handle_microphone_state_change(currentRequiredData, bypassVadForPCM) } func onJsonMessage(_ message: [String: Any]) { @@ -609,29 +263,6 @@ struct ViewState { sgc?.stopVideoRecording(requestId: requestId) } - func setOnboardMicEnabled(_ isEnabled: Bool) { - Task { - if isEnabled { - // Just check permissions - we no longer request them directly from Swift - // Permissions should already be granted via React Native UI flow - if !(PhoneMic.shared.checkPermissions()) { - Bridge.log("Microphone permissions not granted. Cannot enable microphone.") - return - } - - let success = PhoneMic.shared.startRecording() - if !success { - // fallback to glasses mic if possible: - if getGlassesHasMic() { - await enableGlassesMic(true) - } - } - } else { - PhoneMic.shared.stopRecording() - } - } - } - // func onDashboardDisplayEvent(_ event: [String: Any]) { // Core.log("got dashboard display event") //// onDisplayEvent?(["event": event, "type": "dashboard"]) @@ -641,201 +272,6 @@ struct ViewState { //// } // } - // send whatever was there before sending something else: - func clearState() { - sendCurrentState(sgc?.isHeadUp ?? false) - } - - func sendCurrentState(_ isDashboard: Bool) { - if isUpdatingScreen { - return - } - - Task { - var currentViewState: ViewState! - if isDashboard { - currentViewState = self.viewStates[1] - } else { - currentViewState = self.viewStates[0] - } - self.isHeadUp = isDashboard - - if isDashboard && !self.contextualDashboard { - return - } - - if self.defaultWearable.contains("Simulated") || self.defaultWearable.isEmpty { - // dont send the event to glasses that aren't there: - return - } - - if !self.isSomethingConnected() { - return - } - - // cancel any pending clear display work item: - sendStateWorkItem?.cancel() - - let layoutType = currentViewState.layoutType - switch layoutType { - case "text_wall": - let text = currentViewState.text - sendText(text) - case "double_text_wall": - let topText = currentViewState.topText - let bottomText = currentViewState.bottomText - sgc?.sendDoubleTextWall(topText, bottomText) - sgc?.sendDoubleTextWall(topText, bottomText) - case "reference_card": - sendText(currentViewState.title + "\n\n" + currentViewState.text) - case "bitmap_view": - Bridge.log("Mentra: Processing bitmap_view layout") - guard let data = currentViewState.data else { - Bridge.log("Mentra: ERROR: bitmap_view missing data field") - return - } - Bridge.log("Mentra: Processing bitmap_view with base64 data, length: \(data.count)") - await sgc?.displayBitmap(base64ImageData: data) - case "clear_view": - Bridge.log("Mentra: Processing clear_view layout - clearing display") - clearDisplay() - default: - Bridge.log("UNHANDLED LAYOUT_TYPE \(layoutType)") - } - } - } - - func parsePlaceholders(_ text: String) -> String { - let dateFormatter = DateFormatter() - dateFormatter.dateFormat = "M/dd, h:mm" - let formattedDate = dateFormatter.string(from: Date()) - - // 12-hour time format (with leading zeros for hours) - let time12Format = DateFormatter() - time12Format.dateFormat = "hh:mm" - let time12 = time12Format.string(from: Date()) - - // 24-hour time format - let time24Format = DateFormatter() - time24Format.dateFormat = "HH:mm" - let time24 = time24Format.string(from: Date()) - - // Current date with format MM/dd - let dateFormat = DateFormatter() - dateFormat.dateFormat = "MM/dd" - let currentDate = dateFormat.string(from: Date()) - - var placeholders: [String: String] = [:] - placeholders["$no_datetime$"] = formattedDate - placeholders["$DATE$"] = currentDate - placeholders["$TIME12$"] = time12 - placeholders["$TIME24$"] = time24 - - if (sgc?.batteryLevel ?? -1) == -1 { - placeholders["$GBATT$"] = "" - } else { - placeholders["$GBATT$"] = "\(sgc!.batteryLevel)%" - } - - // placeholders["$CONNECTION_STATUS$"] = - // WebSocketManager.shared.isConnected() ? "Connected" : "Disconnected" - // TODO: config: - placeholders["$CONNECTION_STATUS$"] = "Connected" - - var result = text - for (key, value) in placeholders { - result = result.replacingOccurrences(of: key, with: value) - } - - return result - } - - func handle_display_text(_ params: [String: Any]) { - guard let text = params["text"] as? String else { - Bridge.log("Mentra: display_text missing text parameter") - return - } - - Bridge.log("Mentra: Displaying text: \(text)") - sendText(text) - } - - func handle_display_event(_ event: [String: Any]) { - guard let view = event["view"] as? String else { - Bridge.log("Mentra: invalid view") - return - } - let isDashboard = view == "dashboard" - - var stateIndex = 0 - if isDashboard { - stateIndex = 1 - } else { - stateIndex = 0 - } - - let layout = event["layout"] as! [String: Any] - let layoutType = layout["layoutType"] as! String - var text = layout["text"] as? String ?? " " - var topText = layout["topText"] as? String ?? " " - var bottomText = layout["bottomText"] as? String ?? " " - var title = layout["title"] as? String ?? " " - var data = layout["data"] as? String ?? "" - - text = parsePlaceholders(text) - topText = parsePlaceholders(topText) - bottomText = parsePlaceholders(bottomText) - title = parsePlaceholders(title) - - var newViewState = ViewState( - topText: topText, bottomText: bottomText, title: title, layoutType: layoutType, - text: text, data: data, animationData: nil - ) - - if layoutType == "bitmap_animation" { - if let frames = layout["frames"] as? [String], - let interval = layout["interval"] as? Double - { - let animationData: [String: Any] = [ - "frames": frames, - "interval": interval, - "repeat": layout["repeat"] as? Bool ?? true, - ] - newViewState.animationData = animationData - Bridge.log( - "Mentra: Parsed bitmap_animation with \(frames.count) frames, interval: \(interval)ms" - ) - } else { - Bridge.log("Mentra: ERROR: bitmap_animation missing frames or interval") - } - } - - let cS = viewStates[stateIndex] - let nS = newViewState - let currentState = - cS.layoutType + cS.text + cS.topText + cS.bottomText + cS.title + (cS.data ?? "") - let newState = - nS.layoutType + nS.text + nS.topText + nS.bottomText + nS.title + (nS.data ?? "") - - if currentState == newState { - // Core.log("Mentra: View state is the same, skipping update") - return - } - - Bridge.log( - "Updating view state \(stateIndex) with \(layoutType) \(text) \(topText) \(bottomText)") - - viewStates[stateIndex] = newViewState - - let headUp = isHeadUp - // send the state we just received if the user is currently in that state: - if stateIndex == 0, !headUp { - sendCurrentState(false) - } else if stateIndex == 1, headUp { - sendCurrentState(true) - } - } - func onRequestSingle(_ dataType: String) { // Handle single data request if dataType == "battery" { @@ -851,32 +287,11 @@ struct ViewState { Bridge.log("Mentra: onRouteChange: reason: \(reason)") Bridge.log("Mentra: onRouteChange: inputs: \(availableInputs)") - // Core.log the available inputs and see if any are an onboard mic: - // for input in availableInputs { - // Core.log("input: \(input.portType)") - // } - - // if availableInputs.isEmpty { - // self.onboardMicUnavailable = true - // self.setOnboardMicEnabled(false) - // handle_microphone_state_change([], false) - // return - // } else { - // self.onboardMicUnavailable = false - // } - - // switch reason { - // case .newDeviceAvailable: - // micManager?.stopRecording() - // micManager?.startRecording() - // case .oldDeviceUnavailable: - // micManager?.stopRecording() - // micManager?.startRecording() - // default: - // break - // } - // TODO: re-enable this: - // handle_microphone_state_change(currentRequiredData, bypassVadForPCM) + // Update onboard mic availability based on inputs present + onboardMicUnavailable = availableInputs.isEmpty + + // Re-evaluate mic selection immediately to adopt the most stable source + handle_microphone_state_change(currentRequiredData, bypassVadForPCM) } func onInterruption(began: Bool) { @@ -886,45 +301,6 @@ struct ViewState { handle_microphone_state_change(currentRequiredData, bypassVadForPCM) } - private func clearDisplay() { - if sgc is G1 { - let g1 = sgc as? G1 - g1?.sendTextWall(" ") - - // clear the screen after 3 seconds if the text is empty or a space: - if powerSavingMode { - sendStateWorkItem?.cancel() - Bridge.log("Mentra: Clearing display after 3 seconds") - // if we're clearing the display, after a delay, send a clear command if not cancelled with another - let workItem = DispatchWorkItem { [weak self] in - guard let self = self else { return } - if self.isHeadUp { - return - } - g1?.clearDisplay() - } - sendStateWorkItem = workItem - sendStateQueue.asyncAfter(deadline: .now() + 3, execute: workItem) - } - } else { - sgc!.clearDisplay() - } - } - - func sendText(_ text: String) { - // Core.log("Mentra: Sending text: \(text)") - if sgc == nil { - return - } - - if text == " " || text.isEmpty { - clearDisplay() - return - } - - sgc?.sendTextWall(text) - } - // command functions: func setAuthCreds(_ token: String, _ userId: String) { Bridge.log("Mentra: Setting core token to: \(token) for user: \(userId)") @@ -933,52 +309,13 @@ struct ViewState { handle_request_status() } - func disconnectWearable() { - sendText(" ") // clear the screen - Task { - connectTask?.cancel() - sgc?.disconnect() - self.isSearching = false - handle_request_status() - } - } - - func forgetSmartGlasses() { - disconnectWearable() - defaultWearable = "" - deviceName = "" - sgc?.forget() - sgc = nil - Bridge.saveSetting("default_wearable", "") - Bridge.saveSetting("device_name", "") - handle_request_status() - } - - func handleSearchForCompatibleDeviceNames(_ modelName: String) { - Bridge.log("Mentra: Searching for compatible device names for: \(modelName)") - if modelName.contains("Simulated") { - defaultWearable = "Simulated Glasses" // there is no pairing process for simulated glasses - handle_request_status() - return - } - if modelName.contains("G1") { - pendingWearable = "Even Realities G1" - } else if modelName.contains("Live") { - pendingWearable = "Mentra Live" - } else if modelName.contains("Mach1") || modelName.contains("Z100") { - pendingWearable = "Mach1" - } - initSGC(pendingWearable) - sgc?.findCompatibleDevices() - } - func enableContextualDashboard(_ enabled: Bool) { contextualDashboard = enabled handle_request_status() // to update the UI } func setPreferredMic(_ mic: String) { - preferredMic = mic + preferredMic = MicPreference.from(mic) handle_microphone_state_change(currentRequiredData, bypassVadForPCM) handle_request_status() // to update the UI } @@ -1015,27 +352,34 @@ struct ViewState { handle_request_status() // to update the UI } - func handleRgbLedControl(requestId: String, - packageName: String?, - action: String, - color: String?, - ontime: Int, - offtime: Int, - count: Int) - { + func handleRgbLedControl( + requestId: String, + packageName: String?, + action: String, + color: String?, + ontime: Int, + offtime: Int, + count: Int + ) { guard let live = sgc as? MentraLive else { - Bridge.log("Mentra: RGB LED control requested but current SGC does not support Mentra Live features") - Bridge.sendRgbLedControlResponse(requestId: requestId, success: false, error: "unsupported_device") + Bridge.log( + "Mentra: RGB LED control requested but current SGC does not support Mentra Live features" + ) + Bridge.sendRgbLedControlResponse( + requestId: requestId, success: false, error: "unsupported_device" + ) return } - live.handleRgbLedControl(requestId: requestId, - packageName: packageName, - action: action, - color: color, - ontime: ontime, - offtime: offtime, - count: count) + live.handleRgbLedControl( + requestId: requestId, + packageName: packageName, + action: action, + color: color, + ontime: ontime, + offtime: offtime, + count: count + ) } func updateGlassesHeadUpAngle(_ value: Int) { @@ -1130,14 +474,6 @@ struct ViewState { handle_microphone_state_change(requiredData, bypassVadForPCM) } - func startBufferRecording() { - sgc?.startBufferRecording() - } - - func stopBufferRecording() { - sgc?.stopBufferRecording() - } - func setBypassAudioEncoding(_ enabled: Bool) { bypassAudioEncoding = enabled } @@ -1203,7 +539,7 @@ struct ViewState { transcriber?.restart() } - private func getGlassesHasMic() -> Bool { + func getGlassesHasMic() -> Bool { if defaultWearable.contains("G1") { return true } @@ -1216,140 +552,8 @@ struct ViewState { return false } - func enableGlassesMic(_: Bool) async { - await sgc?.setMicEnabled(true) - } - - func handle_request_status() { - // construct the status object: - let simulatedConnected = defaultWearable == "Simulated Glasses" - let isGlassesConnected = sgc?.ready ?? false - if isGlassesConnected { - isSearching = false - } - - // also referenced as glasses_info: - var glassesSettings: [String: Any] = [:] - var connectedGlasses: [String: Any] = [:] - - if isGlassesConnected { - connectedGlasses = [ - "model_name": defaultWearable, - "battery_level": sgc?.batteryLevel ?? -1, - "glasses_app_version": sgc?.glassesAppVersion ?? "", - "glasses_build_number": sgc?.glassesBuildNumber ?? "", - "glasses_device_model": sgc?.glassesDeviceModel ?? "", - "glasses_android_version": sgc?.glassesAndroidVersion ?? "", - "glasses_ota_version_url": sgc?.glassesOtaVersionUrl ?? "", - ] - } - - if simulatedConnected { - connectedGlasses["model_name"] = defaultWearable - } - - if sgc is G1 { - connectedGlasses["case_removed"] = sgc?.caseRemoved ?? true - connectedGlasses["case_open"] = sgc?.caseOpen ?? true - connectedGlasses["case_charging"] = sgc?.caseCharging ?? false - connectedGlasses["case_battery_level"] = sgc?.caseBatteryLevel ?? -1 - - if let serialNumber = sgc?.glassesSerialNumber, !serialNumber.isEmpty { - connectedGlasses["glasses_serial_number"] = serialNumber - connectedGlasses["glasses_style"] = sgc?.glassesStyle ?? "" - connectedGlasses["glasses_color"] = sgc?.glassesColor ?? "" - } - } - - if sgc is MentraLive { - if let wifiSsid = sgc?.wifiSsid, !wifiSsid.isEmpty { - connectedGlasses["glasses_wifi_ssid"] = wifiSsid - connectedGlasses["glasses_wifi_connected"] = sgc?.wifiConnected - connectedGlasses["glasses_wifi_local_ip"] = sgc?.wifiLocalIp - } - - // Add hotspot information - always include all fields for consistency - connectedGlasses["glasses_hotspot_enabled"] = sgc?.isHotspotEnabled ?? false - connectedGlasses["glasses_hotspot_ssid"] = sgc?.hotspotSsid ?? "" - connectedGlasses["glasses_hotspot_password"] = sgc?.hotspotPassword ?? "" - connectedGlasses["glasses_hotspot_gateway_ip"] = sgc?.hotspotGatewayIp ?? "" - } - - // Add Bluetooth device name if available - if let bluetoothName = sgc?.getConnectedBluetoothName() { - connectedGlasses["bluetooth_name"] = bluetoothName - } - - glassesSettings = [ - "brightness": brightness, - "auto_brightness": autoBrightness, - "dashboard_height": dashboardHeight, - "dashboard_depth": dashboardDepth, - "head_up_angle": headUpAngle, - "button_mode": buttonPressMode, - "button_photo_size": buttonPhotoSize, - "button_video_settings": [ - "width": buttonVideoWidth, - "height": buttonVideoHeight, - "fps": buttonVideoFps, - ], - "button_max_recording_time_minutes": buttonMaxRecordingTimeMinutes, - "button_camera_led": buttonCameraLed, - ] - - // let cloudConnectionStatus = - // WebSocketManager.shared.isConnected() ? "CONNECTED" : "DISCONNECTED" - - // TODO: config: remove - let coreInfo: [String: Any] = [ - "augmentos_core_version": "Unknown", - "default_wearable": defaultWearable as Any, - "preferred_mic": preferredMic, - // "is_searching": self.isSearching && !self.defaultWearable.isEmpty, - "is_searching": isSearching, - // only on if recording from glasses: - // TODO: this isn't robust: - "is_mic_enabled_for_frontend": micEnabled && (preferredMic == "glasses") - && isSomethingConnected(), - "sensing_enabled": sensingEnabled, - "power_saving_mode": powerSavingMode, - "always_on_status_bar": alwaysOnStatusBar, - "bypass_vad_for_debugging": bypassVad, - "enforce_local_transcription": enforceLocalTranscription, - "bypass_audio_encoding_for_debugging": bypassAudioEncoding, - "core_token": coreToken, - "puck_connected": true, - "metric_system_enabled": metricSystemEnabled, - "contextual_dashboard_enabled": contextualDashboard, - ] - - // hardcoded list of apps: - var apps: [[String: Any]] = [] - - let authObj: [String: Any] = [ - "core_token_owner": coreTokenOwner, - // "core_token_status": - ] - - let statusObj: [String: Any] = [ - "connected_glasses": connectedGlasses, - "glasses_settings": glassesSettings, - "apps": apps, - "core_info": coreInfo, - "auth": authObj, - ] - - lastStatusObj = statusObj - - Bridge.sendStatus(statusObj) - } - - func triggerStatusUpdate() { - Bridge.log("🔄 Triggering immediate status update") - handle_request_status() - } - - private func playStartupSequence() { + // construct the status object: + func playStartupSequence() { Bridge.log("Mentra: playStartupSequence()") // Arrow frames for the animation let arrowFrames = ["↑", "↗", "↑", "↖"] @@ -1401,273 +605,6 @@ struct ViewState { } } - private func isSomethingConnected() -> Bool { - if sgc?.ready == true { - return true - } - if defaultWearable.contains("Simulated") { - return true - } - return false - } - - private func handleDeviceReady() { - guard let sgc else { - Bridge.log("Mentra: SGC is nil, returning") - return - } - Bridge.log("Mentra: handleDeviceReady(): \(sgc.type)") - // send to the server our battery status: - Bridge.sendBatteryStatus(level: sgc.batteryLevel ?? -1, charging: false) - Bridge.sendGlassesConnectionState(modelName: defaultWearable, status: "CONNECTED") - - pendingWearable = "" - defaultWearable = sgc.type - isSearching = false - handle_request_status() - - if defaultWearable.contains("G1") { - handleG1Ready() - } else if defaultWearable.contains("Mach1") { - handleMach1Ready() - } - - // save the default_wearable now that we're connected: - Bridge.saveSetting("default_wearable", defaultWearable) - Bridge.saveSetting("device_name", deviceName) - // Bridge.saveSetting("device_address", deviceAddress) - } - - private func handleG1Ready() { - // load settings and send the animation: - Task { - // give the glasses some extra time to finish booting: - try? await Task.sleep(nanoseconds: 1_000_000_000) // 3 seconds - await sgc?.setSilentMode(false) // turn off silent mode - await sgc?.getBatteryStatus() - - if shouldSendBootingMessage { - sendText("// BOOTING MENTRAOS") - } - - // send loaded settings to glasses: - try? await Task.sleep(nanoseconds: 400_000_000) - sgc?.setHeadUpAngle(headUpAngle) - try? await Task.sleep(nanoseconds: 400_000_000) - sgc?.setBrightness(brightness, autoMode: autoBrightness) - try? await Task.sleep(nanoseconds: 400_000_000) - // self.g1Manager?.RN_setDashboardPosition(self.dashboardHeight, self.dashboardDepth) - // try? await Task.sleep(nanoseconds: 400_000_000) - // playStartupSequence() - if shouldSendBootingMessage { - sendText("// MENTRAOS CONNECTED") - try? await Task.sleep(nanoseconds: 1_000_000_000) // 1 second - sendText(" ") // clear screen - } - - shouldSendBootingMessage = false - - self.handle_request_status() - } - } - - private func handleMach1Ready() { - Task { - // Send startup message - sendText("MENTRAOS CONNECTED") - try? await Task.sleep(nanoseconds: 1_000_000_000) // 1 second - clearDisplay() - - self.handle_request_status() - } - } - - private func handleDeviceDisconnected() { - Bridge.log("Mentra: Device disconnected") - handle_microphone_state_change([], false) - Bridge.sendGlassesConnectionState(modelName: defaultWearable, status: "DISCONNECTED") - handle_request_status() - } - - func handle_connect_wearable(_ deviceName: String, modelName: String? = nil) { - Bridge.log( - "Mentra: Connecting to modelName: \(modelName ?? "nil") deviceName: \(deviceName) defaultWearable: \(defaultWearable) pendingWearable: \(pendingWearable) selfDeviceName: \(self.deviceName)" - ) - - if modelName != nil { - pendingWearable = modelName! - } - - if pendingWearable.contains("Simulated") { - Bridge.log( - "Mentra: Pending wearable is simulated, setting default wearable to Simulated Glasses" - ) - defaultWearable = "Simulated Glasses" - handle_request_status() - return - } - - if pendingWearable.isEmpty, defaultWearable.isEmpty { - Bridge.log("Mentra: No pending or default wearable, returning") - return - } - - if pendingWearable.isEmpty, !defaultWearable.isEmpty { - Bridge.log("Mentra: No pending wearable, using default wearable: \(defaultWearable)") - pendingWearable = defaultWearable - } - - Task { - disconnectWearable() - - try? await Task.sleep(nanoseconds: 100 * 1_000_000) // 100ms - self.isSearching = true - handle_request_status() // update the UI - - if deviceName != "" { - self.deviceName = deviceName - } - - initSGC(self.pendingWearable) - sgc?.connectById(self.deviceName) - } - - // wait for the g1's to be fully ready: - // connectTask?.cancel() - // connectTask = Task { - // while !(connectTask?.isCancelled ?? true) { - // Core.log("checking if g1 is ready... \(self.g1Manager?.g1Ready ?? false)") - // Core.log("leftReady \(self.g1Manager?.leftReady ?? false) rightReady \(self.g1Manager?.rightReady ?? false)") - // if self.g1Manager?.g1Ready ?? false { - // // we actualy don't need this line: - // // handleDeviceReady() - // handle_request_status() - // break - // } else { - // // todo: ios not the cleanest solution here - // self.g1Manager?.RN_startScan() - // } - // - // try? await Task.sleep(nanoseconds: 15_000_000_000) // 15 seconds - // } - // } - } - - func handle_update_settings(_ settings: [String: Any]) { - Bridge.log("Mentra: Received update settings: \(settings)") - - // update our settings with the new values: - if let newPreferredMic = settings["preferred_mic"] as? String, - newPreferredMic != preferredMic - { - setPreferredMic(newPreferredMic) - } - - if let newHeadUpAngle = settings["head_up_angle"] as? Int, newHeadUpAngle != headUpAngle { - updateGlassesHeadUpAngle(newHeadUpAngle) - } - - if let newBrightness = settings["brightness"] as? Int, newBrightness != brightness { - updateGlassesBrightness(newBrightness, autoBrightness: false) - } - - if let newDashboardHeight = settings["dashboard_height"] as? Int, - newDashboardHeight != dashboardHeight - { - updateGlassesHeight(newDashboardHeight) - } - - if let newDashboardDepth = settings["dashboard_depth"] as? Int, - newDashboardDepth != dashboardDepth - { - updateGlassesDepth(newDashboardDepth) - } - - if let newAutoBrightness = settings["auto_brightness"] as? Bool, - newAutoBrightness != autoBrightness - { - updateGlassesBrightness(brightness, autoBrightness: newAutoBrightness) - } - - if let sensingEnabled = settings["sensing_enabled"] as? Bool, - sensingEnabled != self.sensingEnabled - { - enableSensing(sensingEnabled) - } - - if let powerSavingMode = settings["power_saving_mode"] as? Bool, - powerSavingMode != self.powerSavingMode - { - enablePowerSavingMode(powerSavingMode) - } - - if let newAlwaysOnStatusBar = settings["always_on_status_bar_enabled"] as? Bool, - newAlwaysOnStatusBar != alwaysOnStatusBar - { - enableAlwaysOnStatusBar(newAlwaysOnStatusBar) - } - - if let newBypassVad = settings["bypass_vad_for_debugging"] as? Bool, - newBypassVad != bypassVad - { - bypassVad(newBypassVad) - } - - if let newEnforceLocalTranscription = settings["enforce_local_transcription"] as? Bool, - newEnforceLocalTranscription != enforceLocalTranscription - { - enforceLocalTranscription(newEnforceLocalTranscription) - } - - if let newEnableOfflineMode = settings["offline_captions_app_running"] as? Bool, - newEnableOfflineMode != offlineModeEnabled - { - enableOfflineMode(newEnableOfflineMode) - } - - if let newMetricSystemEnabled = settings["metric_system_enabled"] as? Bool, - newMetricSystemEnabled != metricSystemEnabled - { - setMetricSystemEnabled(newMetricSystemEnabled) - } - - if let newContextualDashboard = settings["contextual_dashboard_enabled"] as? Bool, - newContextualDashboard != contextualDashboard - { - enableContextualDashboard(newContextualDashboard) - } - - if let newButtonMode = settings["button_mode"] as? String, newButtonMode != buttonPressMode { - setButtonMode(newButtonMode) - } - - if let newFps = settings["button_video_fps"] as? Int, newFps != buttonVideoFps { - setButtonVideoSettings(width: buttonVideoWidth, height: buttonVideoHeight, fps: newFps) - } - - if let newWidth = settings["button_video_width"] as? Int, newWidth != buttonVideoWidth { - setButtonVideoSettings(width: newWidth, height: buttonVideoHeight, fps: buttonVideoFps) - } - - if let newHeight = settings["button_video_height"] as? Int, newHeight != buttonVideoHeight { - setButtonVideoSettings(width: buttonVideoWidth, height: newHeight, fps: buttonVideoFps) - } - - if let newPhotoSize = settings["button_photo_size"] as? String, - newPhotoSize != buttonPhotoSize - { - setButtonPhotoSize(newPhotoSize) - } - - // get default wearable from core_info: - if let newDefaultWearable = settings["default_wearable"] as? String, - newDefaultWearable != defaultWearable - { - defaultWearable = newDefaultWearable - Bridge.saveSetting("default_wearable", newDefaultWearable) - } - } - // MARK: - Cleanup @objc func cleanup() { diff --git a/mobile/ios/Source/sgcs/MentraNex.swift b/mobile/ios/Source/sgcs/MentraNex.swift index f13bd1491c..cfa430c690 100644 --- a/mobile/ios/Source/sgcs/MentraNex.swift +++ b/mobile/ios/Source/sgcs/MentraNex.swift @@ -44,7 +44,7 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { private var reconnectionAttempts = 0 // TODO: change this private let maxReconnectionAttempts = -1 // -1 for unlimited - private let reconnectionInterval: TimeInterval = 5.0 // 5 seconds + private let reconnectionInterval: TimeInterval = 15.0 // 5 seconds private var peripheralToConnectName: String? // Heartbeat tracking (like Java implementation) @@ -146,7 +146,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } // Custom Bluetooth queue for better performance (like G1) - private static let _bluetoothQueue = DispatchQueue(label: "com.mentra.nex.bluetooth", qos: .background) + private static let _bluetoothQueue = DispatchQueue( + label: "com.mentra.nex.bluetooth", qos: .background + ) static var instance: MentraNexSGC? @@ -239,7 +241,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { Bridge.log("NEX: 📱 Initial Bluetooth State: \(centralManager.state.rawValue)") } - Bridge.log("NEX: 💾 Loaded saved device - Name: \(savedDeviceName ?? "None"), Address: \(savedDeviceAddress ?? "None")") + Bridge.log( + "NEX: 💾 Loaded saved device - Name: \(savedDeviceName ?? "None"), Address: \(savedDeviceAddress ?? "None")" + ) } private func setupCommandQueue() { @@ -256,14 +260,18 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } private func queueChunks(_ chunks: [[UInt8]], waitTimeMs: Int = 0, chunkDelayMs: Int = 8) { - let cmd = BufferedCommand(chunks: chunks, waitTimeMs: waitTimeMs, chunkDelayMs: chunkDelayMs) + let cmd = BufferedCommand( + chunks: chunks, waitTimeMs: waitTimeMs, chunkDelayMs: chunkDelayMs + ) Task { [weak self] in await self?.commandQueue.enqueue(cmd) } } // Enhanced method that uses MTU-optimized chunking - private func queueDataWithOptimalChunking(_ data: Data, packetType: UInt8 = 0x02, waitTimeMs: Int = 0) { + private func queueDataWithOptimalChunking( + _ data: Data, packetType: UInt8 = 0x02, waitTimeMs: Int = 0 + ) { var chunks: [[UInt8]] = [] let effectiveChunkSize = maxChunkSize - 1 // Reserve 1 byte for packet type @@ -280,7 +288,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { offset += chunkSize } - Bridge.log("NEX: 📦 Created \(chunks.count) MTU-optimized chunks (max size: \(effectiveChunkSize) bytes)") + Bridge.log( + "NEX: 📦 Created \(chunks.count) MTU-optimized chunks (max size: \(effectiveChunkSize) bytes)" + ) queueChunks(chunks, waitTimeMs: waitTimeMs) } @@ -301,7 +311,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { // Send each chunk sequentially for (index, chunk) in command.chunks.enumerated() { let data = Data(chunk) - Bridge.log("NEX: 📦 Sending chunk \(index) of \(command.chunks.count) to \(peripheral.name ?? "Unknown")") + Bridge.log( + "NEX: 📦 Sending chunk \(index) of \(command.chunks.count) to \(peripheral.name ?? "Unknown")" + ) Bridge.log("NEX: 📦 Chunk data: \(data.toHexString())") peripheral.writeValue(data, for: writeCharacteristic, type: .withResponse) @@ -324,7 +336,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { savedDeviceAddress = UserDefaults.standard.string(forKey: PREFS_DEVICE_ADDRESS) preferredDeviceId = UserDefaults.standard.string(forKey: PREFS_DEVICE_ID) - Bridge.log("NEX: 💾 Loaded device info - Name: \(savedDeviceName ?? "None"), Address: \(savedDeviceAddress ?? "None"), ID: \(preferredDeviceId ?? "None")") + Bridge.log( + "NEX: 💾 Loaded device info - Name: \(savedDeviceName ?? "None"), Address: \(savedDeviceAddress ?? "None"), ID: \(preferredDeviceId ?? "None")" + ) } private func savePairedDeviceInfo(name: String?, address: String?) { @@ -433,16 +447,20 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { return false } - Bridge.log("NEX-CONN: 🔵 Attempting to retrieve peripheral with stored UUID: \(uuid.uuidString)") + Bridge.log( + "NEX-CONN: 🔵 Attempting to retrieve peripheral with stored UUID: \(uuid.uuidString)") let peripherals = centralManager.retrievePeripherals(withIdentifiers: [uuid]) if let peripheralToConnect = peripherals.first { - Bridge.log("NEX-CONN: 🔵 Found peripheral by UUID: \(peripheralToConnect.name ?? "Unknown"). Initiating connection.") + Bridge.log( + "NEX-CONN: 🔵 Found peripheral by UUID: \(peripheralToConnect.name ?? "Unknown"). Initiating connection." + ) peripheral = peripheralToConnect centralManager.connect(peripheralToConnect, options: nil) return true } else { - Bridge.log("NEX-CONN: 🔵 Could not find peripheral for stored UUID. Will proceed to scan.") + Bridge.log( + "NEX-CONN: 🔵 Could not find peripheral for stored UUID. Will proceed to scan.") return false } } @@ -502,7 +520,8 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } guard centralManager.state == .poweredOn else { - Bridge.log("NEX-CONN: ❌ Bluetooth not powered on. State: \(centralManager.state.rawValue)") + Bridge.log( + "NEX-CONN: ❌ Bluetooth not powered on. State: \(centralManager.state.rawValue)") return } @@ -513,9 +532,17 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } // If that fails, check for already-connected system devices - let connectedPeripherals = centralManager.retrieveConnectedPeripherals(withServices: [MAIN_SERVICE_UUID]) - if let targetName = peripheralToConnectName, let existingPeripheral = connectedPeripherals.first(where: { $0.name?.contains(targetName) == true }) { - Bridge.log("NEX-CONN: 📱 Found already connected peripheral that matches target: \(existingPeripheral.name ?? "Unknown")") + let connectedPeripherals = centralManager.retrieveConnectedPeripherals(withServices: [ + MAIN_SERVICE_UUID, + ]) + if let targetName = peripheralToConnectName, + let existingPeripheral = connectedPeripherals.first(where: { + $0.name?.contains(targetName) == true + }) + { + Bridge.log( + "NEX-CONN: 📱 Found already connected peripheral that matches target: \(existingPeripheral.name ?? "Unknown")" + ) if peripheral == nil { peripheral = existingPeripheral centralManager.connect(existingPeripheral, options: nil) @@ -524,7 +551,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } // Check if we have a saved device name to reconnect to (like MentraLive) - if let savedDeviceName = UserDefaults.standard.string(forKey: PREFS_DEVICE_NAME), !savedDeviceName.isEmpty { + if let savedDeviceName = UserDefaults.standard.string(forKey: PREFS_DEVICE_NAME), + !savedDeviceName.isEmpty + { Bridge.log("NEX-CONN: 🔄 Looking for saved device: \(savedDeviceName)") // This will be handled in didDiscover when the device is found } @@ -543,7 +572,8 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { // Re-emit already discovered peripherals (like MentraLive) for (_, peripheral) in discoveredPeripherals { - Bridge.log("NEX-CONN: 📡 (Re-emitting from cache) peripheral: \(peripheral.name ?? "Unknown")") + Bridge.log( + "NEX-CONN: 📡 (Re-emitting from cache) peripheral: \(peripheral.name ?? "Unknown")") if let name = peripheral.name { emitDiscoveredDevice(name) } @@ -611,7 +641,10 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { Task { if centralManager == nil { - centralManager = CBCentralManager(delegate: self, queue: bluetoothQueue, options: ["CBCentralManagerOptionShowPowerAlertKey": 0]) + centralManager = CBCentralManager( + delegate: self, queue: bluetoothQueue, + options: ["CBCentralManagerOptionShowPowerAlertKey": 0] + ) // wait for the central manager to be fully initialized before we start scanning: try? await Task.sleep(nanoseconds: 100 * 1_000_000) // 100ms } @@ -759,7 +792,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } let protobufData = try! phoneToGlasses.serializedData() - queueDataWithOptimalChunking(protobufData, packetType: PACKET_TYPE_PROTOBUF, waitTimeMs: 100) + queueDataWithOptimalChunking( + protobufData, packetType: PACKET_TYPE_PROTOBUF, waitTimeMs: 100 + ) // Send image chunks sendImageChunks(streamId: streamId, imageData: bmpData) @@ -803,15 +838,17 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { var pixelData = Data(count: width * height * bytesPerPixel) pixelData.withUnsafeMutableBytes { bytes in - guard let context = CGContext( - data: bytes.bindMemory(to: UInt8.self).baseAddress, - width: width, - height: height, - bitsPerComponent: bitsPerComponent, - bytesPerRow: bytesPerRow, - space: CGColorSpaceCreateDeviceRGB(), - bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue - ) else { return } + guard + let context = CGContext( + data: bytes.bindMemory(to: UInt8.self).baseAddress, + width: width, + height: height, + bitsPerComponent: bitsPerComponent, + bytesPerRow: bytesPerRow, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + else { return } context.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height)) } @@ -1207,7 +1244,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } private func processAudioData(_ audioData: Data, sequenceNumber: UInt8) { - Bridge.log("NEX: Received audio data - sequence: \(sequenceNumber), size: \(audioData.count) bytes") + Bridge.log( + "NEX: Received audio data - sequence: \(sequenceNumber), size: \(audioData.count) bytes" + ) // Update @Published property (G1-compatible approach) // Create packet with sequence number prefix like G1 expects @@ -1314,7 +1353,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { aiListening = vadActiveState // Mirror G1's aiListening behavior } - private func handleImageTransferCompleteProtobuf(_ transferComplete: Mentraos_Ble_ImageTransferComplete) { + private func handleImageTransferCompleteProtobuf( + _ transferComplete: Mentraos_Ble_ImageTransferComplete + ) { let status = transferComplete.status let missingChunks = transferComplete.missingChunks @@ -1323,11 +1364,11 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { switch status { case .ok: Bridge.log("NEX: Image transfer completed successfully") - // Clear any pending image chunks + // Clear any pending image chunks case .incomplete: Bridge.log("NEX: Image transfer incomplete - Missing chunks: \(missingChunks)") - // Could implement chunk retransmission here + // Could implement chunk retransmission here default: Bridge.log("NEX: Unknown image transfer status") @@ -1540,7 +1581,8 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { micBeatCount += 1 // Schedule periodic mic beat (like Java lines 1753-1762) - micBeatTimer = Timer.scheduledTimer(withTimeInterval: MICBEAT_INTERVAL_MS, repeats: true) { [weak self] _ in + micBeatTimer = Timer.scheduledTimer(withTimeInterval: MICBEAT_INTERVAL_MS, repeats: true) { + [weak self] _ in guard let self else { return } Bridge.log("NEX: SENDING MIC BEAT") self.setMicrophoneEnabled(self.shouldUseGlassesMic) @@ -1779,7 +1821,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { case .poweredOn: Bridge.log("NEX: ✅ Bluetooth is ready for scanning") - if let savedDeviceName = UserDefaults.standard.string(forKey: PREFS_DEVICE_NAME), !savedDeviceName.isEmpty { + if let savedDeviceName = UserDefaults.standard.string(forKey: PREFS_DEVICE_NAME), + !savedDeviceName.isEmpty + { Bridge.log("NEX: 🔄 Looking for saved device: \(savedDeviceName)") // This will be handled in didDiscover when the device is found startScan() @@ -1834,7 +1878,10 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } } - func centralManager(_: CBCentralManager, didDiscover peripheral: CBPeripheral, advertisementData _: [String: Any], rssi RSSI: NSNumber) { + func centralManager( + _: CBCentralManager, didDiscover peripheral: CBPeripheral, + advertisementData _: [String: Any], rssi RSSI: NSNumber + ) { guard let deviceName = peripheral.name else { // Bridge.log("NEX-CONN: 🚫 Ignoring device with no name") return @@ -1886,11 +1933,15 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { private func connectToFoundDevice(_ peripheral: CBPeripheral, reason: String) { guard self.peripheral == nil else { - Bridge.log("NEX-CONN: ⚠️ Already connected/connecting to a device, ignoring new connect request for '\(peripheral.name ?? "Unknown")'") + Bridge.log( + "NEX-CONN: ⚠️ Already connected/connecting to a device, ignoring new connect request for '\(peripheral.name ?? "Unknown")'" + ) return } - Bridge.log("NEX-CONN: 🔗 Connecting to device '\(peripheral.name ?? "Unknown")' - Reason: \(reason)") + Bridge.log( + "NEX-CONN: 🔗 Connecting to device '\(peripheral.name ?? "Unknown")' - Reason: \(reason)" + ) // Stop scanning since we found our target if _isScanning { @@ -1942,8 +1993,12 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { Bridge.log("NEX-CONN: 🔄 Reset reconnection attempts counter") } - func centralManager(_: CBCentralManager, didFailToConnect peripheral: CBPeripheral, error: Error?) { - Bridge.log("NEX-CONN: ❌ Failed to connect to peripheral \(peripheral.name ?? "Unknown"). Error: \(error?.localizedDescription ?? "unknown")") + func centralManager( + _: CBCentralManager, didFailToConnect peripheral: CBPeripheral, error: Error? + ) { + Bridge.log( + "NEX-CONN: ❌ Failed to connect to peripheral \(peripheral.name ?? "Unknown"). Error: \(error?.localizedDescription ?? "unknown")" + ) isConnecting = false connectionState = .disconnected self.peripheral = nil // Reset peripheral on failure to allow reconnection @@ -1953,8 +2008,12 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { } } - func centralManager(_: CBCentralManager, didDisconnectPeripheral disconnectedPeripheral: CBPeripheral, error: Error?) { - Bridge.log("NEX-CONN: 🔌 Disconnected from peripheral: \(disconnectedPeripheral.name ?? "Unknown")") + func centralManager( + _: CBCentralManager, didDisconnectPeripheral disconnectedPeripheral: CBPeripheral, + error: Error? + ) { + Bridge.log( + "NEX-CONN: 🔌 Disconnected from peripheral: \(disconnectedPeripheral.name ?? "Unknown")") if let error { Bridge.log("NEX-CONN: ⚠️ Disconnect error: \(error.localizedDescription)") @@ -2008,7 +2067,9 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { startReconnectionTimer() } else { - Bridge.log("NEX-CONN: ✅ Intentional disconnect (isDisconnecting: \(isDisconnecting), isKilled: \(isKilled))") + Bridge.log( + "NEX-CONN: ✅ Intentional disconnect (isDisconnecting: \(isDisconnecting), isKilled: \(isKilled))" + ) if isDisconnecting { // Don't clear device info on intentional disconnect - user might reconnect later @@ -2114,7 +2175,8 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { // 2. Restore previous microphone state (Java lines 657-665) DispatchQueue.main.asyncAfter(deadline: .now() + 0.02) { // 20ms delay - let shouldRestoreMic = UserDefaults.standard.bool(forKey: "microphoneStateBeforeDisconnection") + let shouldRestoreMic = UserDefaults.standard.bool( + forKey: "microphoneStateBeforeDisconnection") Bridge.log("NEX: 🎤 Restoring microphone state to: \(shouldRestoreMic)") if shouldRestoreMic { @@ -2177,14 +2239,19 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { for service in services { if service.uuid == MAIN_SERVICE_UUID { Bridge.log("NEX-CONN: ✅ Found main service. Discovering characteristics...") - peripheral.discoverCharacteristics([WRITE_CHAR_UUID, NOTIFY_CHAR_UUID], for: service) + peripheral.discoverCharacteristics( + [WRITE_CHAR_UUID, NOTIFY_CHAR_UUID], for: service + ) } } } - func peripheral(_ peripheral: CBPeripheral, didDiscoverCharacteristicsFor service: CBService, error: Error?) { + func peripheral( + _ peripheral: CBPeripheral, didDiscoverCharacteristicsFor service: CBService, error: Error? + ) { if let error { - Bridge.log("NEX-CONN: ❌ Error discovering characteristics: \(error.localizedDescription)") + Bridge.log( + "NEX-CONN: ❌ Error discovering characteristics: \(error.localizedDescription)") return } @@ -2197,21 +2264,26 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { Bridge.log("NEX-CONN: ✅ Found write characteristic.") writeCharacteristic = characteristic } else if characteristic.uuid == NOTIFY_CHAR_UUID { - Bridge.log("NEX-CONN: ✅ Found notify characteristic. Subscribing for notifications.") + Bridge.log( + "NEX-CONN: ✅ Found notify characteristic. Subscribing for notifications.") notifyCharacteristic = characteristic peripheral.setNotifyValue(true, for: characteristic) } } if writeCharacteristic != nil, notifyCharacteristic != nil { - Bridge.log("NEX-CONN: ✅ All required characteristics discovered. Proceeding to MTU negotiation.") + Bridge.log( + "NEX-CONN: ✅ All required characteristics discovered. Proceeding to MTU negotiation." + ) // Start MTU negotiation like Java implementation requestOptimalMTU(for: peripheral) } } - func peripheral(_: CBPeripheral, didUpdateValueFor characteristic: CBCharacteristic, error: Error?) { + func peripheral( + _: CBPeripheral, didUpdateValueFor characteristic: CBCharacteristic, error: Error? + ) { if let error { Bridge.log("NEX-CONN: ❌ Error on updating value: \(error.localizedDescription)") return @@ -2227,25 +2299,38 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate { processReceivedData(data) } - func peripheral(_: CBPeripheral, didWriteValueFor characteristic: CBCharacteristic, error: Error?) { + func peripheral( + _: CBPeripheral, didWriteValueFor characteristic: CBCharacteristic, error: Error? + ) { if let error { - Bridge.log("NEX-CONN: ❌ Error writing value to \(characteristic.uuid): \(error.localizedDescription)") + Bridge.log( + "NEX-CONN: ❌ Error writing value to \(characteristic.uuid): \(error.localizedDescription)" + ) return } // This log can be very noisy, so it's commented out. // Bridge.log("NEX-CONN: 📤 Successfully wrote value to \(characteristic.uuid).") } - func peripheral(_: CBPeripheral, didUpdateNotificationStateFor characteristic: CBCharacteristic, error: Error?) { + func peripheral( + _: CBPeripheral, didUpdateNotificationStateFor characteristic: CBCharacteristic, + error: Error? + ) { if let error { - Bridge.log("NEX-CONN: ❌ Error changing notification state for \(characteristic.uuid): \(error.localizedDescription)") + Bridge.log( + "NEX-CONN: ❌ Error changing notification state for \(characteristic.uuid): \(error.localizedDescription)" + ) return } if characteristic.isNotifying { - Bridge.log("NEX-CONN: ✅ Successfully subscribed to notifications for characteristic \(characteristic.uuid.uuidString).") + Bridge.log( + "NEX-CONN: ✅ Successfully subscribed to notifications for characteristic \(characteristic.uuid.uuidString)." + ) } else { - Bridge.log("NEX-CONN: unsubscribed from notifications for characteristic \(characteristic.uuid.uuidString).") + Bridge.log( + "NEX-CONN: unsubscribed from notifications for characteristic \(characteristic.uuid.uuidString)." + ) } } } diff --git a/mobile/src/app/settings/privacy.tsx b/mobile/src/app/settings/privacy.tsx index 1b80c4ed2d..231b3dd262 100644 --- a/mobile/src/app/settings/privacy.tsx +++ b/mobile/src/app/settings/privacy.tsx @@ -1,18 +1,5 @@ -import React, {useEffect, useState} from "react" -import { - View, - Text, - StyleSheet, - Switch, - Platform, - ScrollView, - AppState, - NativeModules, - Linking, - ViewStyle, - TextStyle, -} from "react-native" -import {useCoreStatus} from "@/contexts/CoreStatusProvider" +import {useEffect, useState} from "react" +import {Platform, ScrollView, AppState} from "react-native" import bridge from "@/bridge/MantleBridge" import {requestFeaturePermissions, PermissionFeatures, checkFeaturePermissions} from "@/utils/PermissionsUtils" import { @@ -20,16 +7,16 @@ import { checkAndRequestNotificationAccessSpecialPermission, } from "@/utils/NotificationServiceUtils" // import {NotificationService} from '@/utils/NotificationServiceUtils'; -import showAlert from "@/utils/AlertUtils" import {Header, Screen} from "@/components/ignite" -import {spacing, ThemedStyle} from "@/theme" import {useAppTheme} from "@/utils/useAppTheme" import ToggleSetting from "@/components/settings/ToggleSetting" +import SelectSetting from "@/components/settings/SelectSetting" import {translate} from "@/i18n" import {Spacer} from "@/components/misc/Spacer" import {useNavigationHistory} from "@/contexts/NavigationHistoryContext" import PermissionButton from "@/components/settings/PermButton" -import {SETTINGS_KEYS, useSetting, useSettingsStore} from "@/stores/settings" +import {SETTINGS_KEYS, useSetting} from "@/stores/settings" +import mantle from "@/managers/MantleManager" export default function PrivacySettingsScreen() { const [notificationsEnabled, setNotificationsEnabled] = useState(true) @@ -39,9 +26,10 @@ export default function PrivacySettingsScreen() { const [locationPermissionPending, setLocationPermissionPending] = useState(false) const [appState, setAppState] = useState(AppState.currentState) const {theme} = useAppTheme() - const {goBack, push} = useNavigationHistory() + const {goBack} = useNavigationHistory() const [sensingEnabled, setSensingEnabled] = useSetting(SETTINGS_KEYS.sensing_enabled) - const setSetting = useSettingsStore(state => state.setSetting) + const [micActivationMode, setMicActivationMode] = useSetting(SETTINGS_KEYS.mic_activation_mode) + const [locationUpdatesMode, setLocationUpdatesMode] = useSetting(SETTINGS_KEYS.location_updates_mode) // Check permissions when screen loads useEffect(() => { @@ -124,6 +112,16 @@ export default function PrivacySettingsScreen() { } } + const micActivationOptions = [ + {label: translate("settings:micActivationModeHeadUp"), value: "head_up"}, + {label: translate("settings:micActivationModeAlwaysOn"), value: "always_on"}, + ] + + const locationUpdateOptions = [ + {label: translate("settings:locationUpdatesModeHeadUp"), value: "head_up"}, + {label: translate("settings:locationUpdatesModeAlwaysOn"), value: "always_on"}, + ] + // Monitor app state to detect when user returns from settings useEffect(() => { const subscription = AppState.addEventListener("change", nextAppState => { @@ -269,6 +267,27 @@ export default function PrivacySettingsScreen() { value={sensingEnabled} onValueChange={toggleSensing} /> + + { + void setMicActivationMode(value) + }} + /> + + { + void setLocationUpdatesMode(value) + void mantle.setLocationUpdatesMode(value) + }} + /> ) diff --git a/mobile/src/bridge/MantleBridge.tsx b/mobile/src/bridge/MantleBridge.tsx index 42ec5882b5..a091a148e1 100644 --- a/mobile/src/bridge/MantleBridge.tsx +++ b/mobile/src/bridge/MantleBridge.tsx @@ -441,9 +441,12 @@ export class MantleBridge extends EventEmitter { case "save_setting": await useSettingsStore.getState().setSetting(data.key, data.value, false) break - case "head_up": - socketComms.sendHeadPosition(data.position) + case "head_up": { + const isHeadUp = !!data.position + void mantle.handleHeadPosition(isHeadUp) + socketComms.sendHeadPosition(isHeadUp) break + } // TODO: config: remove (this is legacy/android only) case "transcription_result": mantle.handleLocalTranscription(data) @@ -468,10 +471,9 @@ export class MantleBridge extends EventEmitter { for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i) } + socketComms.sendBinary(bytes) if (livekitManager.isRoomConnected()) { livekitManager.addPcm(bytes) - } else { - socketComms.sendBinary(bytes) } break case "rtmp_stream_status": diff --git a/mobile/src/contexts/AppletStatusProvider.tsx b/mobile/src/contexts/AppletStatusProvider.tsx index 0d3a510c75..1e550628ba 100644 --- a/mobile/src/contexts/AppletStatusProvider.tsx +++ b/mobile/src/contexts/AppletStatusProvider.tsx @@ -37,6 +37,7 @@ export const AppStatusProvider = ({children}: {children: ReactNode}) => { const pendingOperations = useRef<{[packageName: string]: "start" | "stop"}>({}) // Keep track of refresh timeouts to cancel them const refreshTimeouts = useRef<{[packageName: string]: NodeJS.Timeout}>({}) + const foregroundStateSent = useRef(null) const refreshAppStatus = useCallback(async () => { console.log("AppStatusProvider: refreshAppStatus called - user exists:", !!user, "user email:", user?.email) @@ -459,14 +460,53 @@ export const AppStatusProvider = ({children}: {children: ReactNode}) => { } }, [appStatus, optimisticallyStartApp]) - // refresh app status until loaded: + // Refresh app status lazily with exponential backoff until loaded; also react to CORE_TOKEN_SET useEffect(() => { if (appStatus.length > 0) return - const interval = setInterval(() => { - refreshAppStatus() - }, 2000) - return () => clearInterval(interval) - }, [appStatus.length]) + + let cancelled = false + let timeout: NodeJS.Timeout | null = null + let delay = 5000 // start at 5s, back off to reduce radio wakeups + + const tryRefresh = async () => { + if (cancelled) return + await refreshAppStatus() + if (cancelled || appStatus.length > 0) return + delay = Math.min(delay * 2, 30000) // cap at 30s + timeout = setTimeout(tryRefresh, delay) + } + + // Initial attempt + timeout = setTimeout(tryRefresh, delay) + + const onCoreTokenSet = () => { + if (cancelled) return + // Reset delay and try immediately when token becomes available + delay = 5000 + if (timeout) clearTimeout(timeout) + tryRefresh() + } + + // @ts-ignore + GlobalEventEmitter.on("CORE_TOKEN_SET", onCoreTokenSet) + + return () => { + cancelled = true + if (timeout) clearTimeout(timeout) + // @ts-ignore + GlobalEventEmitter.off("CORE_TOKEN_SET", onCoreTokenSet) + } + }, [appStatus.length, refreshAppStatus]) + + // Notify native only when the foreground-open state changes + useEffect(() => { + const anyForegroundOpen = appStatus.some(app => (app.type === "standard" || !app.type) && app.is_running) + if (foregroundStateSent.current !== anyForegroundOpen) { + foregroundStateSent.current = anyForegroundOpen + // Handled by Bridge.swift -> MentraManager.setForegroundAppOpen + bridge.sendCommand("set_foreground_app_open", {active: anyForegroundOpen}) + } + }, [appStatus]) // Watch camera app state and send gallery mode updates to glasses (Android only) useEffect(() => { diff --git a/mobile/src/devtools/ReactotronConfig.ts b/mobile/src/devtools/ReactotronConfig.ts index b5dabc21ed..e8ad16353c 100644 --- a/mobile/src/devtools/ReactotronConfig.ts +++ b/mobile/src/devtools/ReactotronConfig.ts @@ -59,7 +59,8 @@ reactotron.onCustomCommand<[{name: "route"; type: ArgType.String}]>({ const {route} = args ?? {} if (route) { Reactotron.log(`Navigating to: ${route}`) - router.push(route) + // Cast for dev utility to satisfy Expo Router's typed paths + router.push(route as any) } else { Reactotron.log("Could not navigate. No route provided.") } @@ -126,4 +127,6 @@ declare global { /** * Now that we've setup all our Reactotron configuration, let's connect! */ -reactotron.connect() +if (__DEV__) { + reactotron.connect() +} diff --git a/mobile/src/i18n/en.ts b/mobile/src/i18n/en.ts index b05f1c780c..8fecabd556 100644 --- a/mobile/src/i18n/en.ts +++ b/mobile/src/i18n/en.ts @@ -200,11 +200,19 @@ const en = { calendarSubtitle: "Display calendar events on your smart glasses.", locationLabel: "Location Access", locationSubtitle: "Display navigation and weather information on your smart glasses.", + locationUpdatesModeLabel: "Location Updates", + locationUpdatesModeSubtitle: "Choose when Mentra collects your location.", + locationUpdatesModeHeadUp: "Only when looking up", + locationUpdatesModeAlwaysOn: "Always on", autoBrightnessLabel: "Auto Brightness", autoBrightnessSubtitle: "Automatically adjust the brightness of your smart glasses based on the ambient light.", notificationsLabel: "Notifications Access", notificationsSubtitle: "Allow Mentra to forward your phone notifications to your smart glasses.", selectMic: "Select which microphone to use", + micActivationModeLabel: "Microphone Activation", + micActivationModeSubtitle: "Choose when the microphone turns on.", + micActivationModeHeadUp: "Only when looking up", + micActivationModeAlwaysOn: "Always on", simulatedGlassesNote: "This setting has no effect when using Simulated Glasses", profileSettings: "Profile Settings", privacySettings: "Permissions and Privacy", diff --git a/mobile/src/managers/MantleManager.ts b/mobile/src/managers/MantleManager.ts index a1a048a0f4..22ce801391 100644 --- a/mobile/src/managers/MantleManager.ts +++ b/mobile/src/managers/MantleManager.ts @@ -9,22 +9,38 @@ import bridge from "@/bridge/MantleBridge" const LOCATION_TASK_NAME = "handleLocationUpdates" -TaskManager.defineTask(LOCATION_TASK_NAME, ({data: {locations}, error}) => { - if (error) { - // check `error.message` for more details. - console.error("Error handling location updates", error) - return - } - const locs = locations as Location.LocationObject[] - if (locs.length === 0) { - console.log("Mantle: LOCATION: No locations received") - return - } +TaskManager.defineTask( + LOCATION_TASK_NAME, + async ({data, error}: TaskManager.TaskManagerTaskBody<{locations: Location.LocationObject[]}>) => { + if (error) { + // check `error.message` for more details. + console.error("Error handling location updates", error) + return + } + const locs = (data?.locations ?? []) as Location.LocationObject[] + if (!locs || locs.length === 0) { + console.log("Mantle: LOCATION: No locations received") + return + } - console.log("Received new locations", locations) - const first = locs[0]! - socketComms.sendLocationUpdate(first.coords.latitude, first.coords.longitude, first.coords.accuracy ?? undefined) -}) + console.log("Received new locations", data?.locations) + const first = locs[0]! + + const mm = MantleManager.getInstance() + // Update cache if we got a good fix + mm.updateLocationCacheIfGood(first) + + // Choose the best location to send (prefer cached good fix within TTL) + const best = mm.getBestLocationForSend(first) + if (!best) { + console.log("Mantle: LOCATION: No best location available to send") + return + } + + const {coords} = best + socketComms.sendLocationUpdate(coords.latitude, coords.longitude, coords.accuracy ?? undefined) + }, +) class MantleManager { private static instance: MantleManager | null = null @@ -34,6 +50,17 @@ class MantleManager { private clearTextTimeout: NodeJS.Timeout | null = null private readonly MAX_CHARS_PER_LINE = 30 private readonly MAX_LINES = 3 + private locationUpdatesActive = false + private locationUpdatesStarting = false + private locationUpdatesStopping = false + private wantLocationUpdates = false + private isHeadUp = false + private cachedLocationTier: string | null = null + private locationUpdatesMode: "head_up" | "always_on" = "head_up" + // Cache a good GPS lock for 5 minutes to avoid degrading to worse fixes + private lastGoodLocation: {latitude: number; longitude: number; accuracy?: number; timestamp: number} | null = null + private readonly GOOD_ACCURACY_METERS = 50 + private readonly CACHE_TTL_MS = 5 * 60 * 1000 public static getInstance(): MantleManager { if (!MantleManager.instance) { @@ -66,7 +93,9 @@ class MantleManager { clearInterval(this.calendarSyncTimer) this.calendarSyncTimer = null } - Location.stopLocationUpdatesAsync(LOCATION_TASK_NAME) + this.wantLocationUpdates = false + this.isHeadUp = false + void this.stopLocationUpdatesIfNeeded() this.transcriptProcessor.clear() } @@ -80,13 +109,30 @@ class MantleManager { 60 * 60 * 1000, ) // 1 hour try { - let locationAccuracy = await useSettingsStore.getState().loadSetting(SETTINGS_KEYS.location_tier) - let properAccuracy = this.getLocationAccuracy(locationAccuracy) - Location.startLocationUpdatesAsync(LOCATION_TASK_NAME, { - accuracy: properAccuracy, - }) + const storedTier = await useSettingsStore.getState().loadSetting(SETTINGS_KEYS.location_tier) + this.cachedLocationTier = typeof storedTier === "string" ? storedTier : null } catch (error) { - console.error("Mantle: Error starting location updates", error) + console.error("Mantle: Error loading location tier", error) + this.cachedLocationTier = null + } + + try { + const hasStarted = await Location.hasStartedLocationUpdatesAsync(LOCATION_TASK_NAME) + if (hasStarted) { + await Location.stopLocationUpdatesAsync(LOCATION_TASK_NAME) + } + this.locationUpdatesActive = false + this.wantLocationUpdates = false + } catch (error) { + console.error("Mantle: Error stopping existing location updates", error) + } + + try { + const storedMode = await useSettingsStore.getState().loadSetting(SETTINGS_KEYS.location_updates_mode) + await this.applyLocationUpdatesMode(typeof storedMode === "string" ? storedMode : null) + } catch (error) { + console.error("Mantle: Error applying location updates mode", error) + await this.applyLocationUpdatesMode(null) } } @@ -112,6 +158,165 @@ class MantleManager { // socketComms.sendLocationUpdate(location) } + private async resolveLocationTier(): Promise { + if (this.cachedLocationTier === null) { + try { + const storedTier = await useSettingsStore.getState().loadSetting(SETTINGS_KEYS.location_tier) + this.cachedLocationTier = typeof storedTier === "string" ? storedTier : null + } catch (error) { + console.error("Mantle: Error resolving location tier", error) + this.cachedLocationTier = null + } + } + + return this.cachedLocationTier + } + + private async buildLocationTaskOptions(): Promise { + const tier = await this.resolveLocationTier() + const accuracy = this.getLocationAccuracy(tier ?? "") + return { + accuracy, + pausesUpdatesAutomatically: false, + } + } + + // Location cache helpers + private isGoodAccuracy(accuracy?: number | null): boolean { + return typeof accuracy === "number" && accuracy > 0 && accuracy <= this.GOOD_ACCURACY_METERS + } + + private isCacheFresh(): boolean { + return this.lastGoodLocation !== null && Date.now() - this.lastGoodLocation.timestamp <= this.CACHE_TTL_MS + } + + public getCachedLocation(): {latitude: number; longitude: number; accuracy?: number} | null { + if (this.lastGoodLocation && this.isCacheFresh()) { + const {latitude, longitude, accuracy} = this.lastGoodLocation + return {latitude, longitude, accuracy} + } + return null + } + + public updateLocationCacheIfGood(loc?: Location.LocationObject | null): void { + if (!loc) return + const acc = loc.coords.accuracy + if (this.isGoodAccuracy(acc)) { + this.lastGoodLocation = { + latitude: loc.coords.latitude, + longitude: loc.coords.longitude, + accuracy: acc ?? undefined, + timestamp: Date.now(), + } + } + } + + // Prefer a fresh cached good fix over a newly delivered poor fix. + // Fallback order: good new fix -> fresh cached good fix -> any new fix -> null + public getBestLocationForSend( + newLoc?: Location.LocationObject | null, + ): {coords: {latitude: number; longitude: number; accuracy?: number}; fromCache: boolean} | null { + if (newLoc && this.isGoodAccuracy(newLoc.coords.accuracy ?? undefined)) { + return { + coords: { + latitude: newLoc.coords.latitude, + longitude: newLoc.coords.longitude, + accuracy: newLoc.coords.accuracy ?? undefined, + }, + fromCache: false, + } + } + + const cached = this.getCachedLocation() + if (cached) { + return {coords: cached, fromCache: true} + } + + if (newLoc) { + // No fresh good cache and new fix is not "good", but still send something if available + return { + coords: { + latitude: newLoc.coords.latitude, + longitude: newLoc.coords.longitude, + accuracy: newLoc.coords.accuracy ?? undefined, + }, + fromCache: false, + } + } + + return null + } + + private async applyLocationUpdatesMode(mode: string | null) { + const normalized: "head_up" | "always_on" = mode === "always_on" ? "always_on" : "head_up" + this.locationUpdatesMode = normalized + const shouldWantUpdates = normalized === "always_on" || (normalized === "head_up" && this.isHeadUp) + this.wantLocationUpdates = shouldWantUpdates + + if (shouldWantUpdates) { + await this.startLocationUpdatesIfNeeded() + } else { + await this.stopLocationUpdatesIfNeeded() + } + } + + private async startLocationUpdatesIfNeeded() { + if (!this.wantLocationUpdates) { + return + } + if (this.locationUpdatesActive || this.locationUpdatesStarting) { + return + } + + this.locationUpdatesStarting = true + try { + const options = await this.buildLocationTaskOptions() + if (!this.wantLocationUpdates) { + return + } + + await Location.startLocationUpdatesAsync(LOCATION_TASK_NAME, options) + this.locationUpdatesActive = true + } catch (error) { + console.error("Mantle: Error starting location updates", error) + this.locationUpdatesActive = false + } finally { + this.locationUpdatesStarting = false + } + } + + private async stopLocationUpdatesIfNeeded() { + if (this.locationUpdatesStopping) { + return + } + + this.locationUpdatesStopping = true + try { + const hasStarted = await Location.hasStartedLocationUpdatesAsync(LOCATION_TASK_NAME) + if (hasStarted) { + await Location.stopLocationUpdatesAsync(LOCATION_TASK_NAME) + } + this.locationUpdatesActive = false + } catch (error) { + console.error("Mantle: Error stopping location updates", error) + } finally { + this.locationUpdatesStopping = false + this.locationUpdatesStarting = false + } + } + + private async restartLocationUpdatesIfNeeded() { + if (!this.wantLocationUpdates) { + return + } + await this.stopLocationUpdatesIfNeeded() + await this.startLocationUpdatesIfNeeded() + } + + public async setLocationUpdatesMode(mode: string) { + await this.applyLocationUpdatesMode(mode) + } + public getLocationAccuracy(accuracy: string) { switch (accuracy) { case "realtime": @@ -134,28 +339,32 @@ class MantleManager { public async setLocationTier(tier: string) { console.log("Mantle: setLocationTier()", tier) - // restComms.sendLocationData({tier}) - try { - const accuracy = this.getLocationAccuracy(tier) - await Location.stopLocationUpdatesAsync(LOCATION_TASK_NAME) - await Location.startLocationUpdatesAsync(LOCATION_TASK_NAME, { - accuracy: accuracy, - pausesUpdatesAutomatically: false, - }) - } catch (error) { - console.error("Mantle: Error setting location tier", error) + this.cachedLocationTier = tier + if (this.wantLocationUpdates) { + await this.restartLocationUpdatesIfNeeded() } } public async requestSingleLocation(accuracy: string, correlationId: string) { console.log("Mantle: requestSingleLocation()") - // restComms.sendLocationData({tier}) try { + // If we have a fresh good fix cached, use it immediately + const cached = this.getCachedLocation() + if (cached) { + socketComms.sendLocationUpdate(cached.latitude, cached.longitude, cached.accuracy ?? undefined, correlationId) + return + } + + // Otherwise fetch a new fix, update cache if good, then choose best to send const location = await Location.getCurrentPositionAsync({accuracy: this.getLocationAccuracy(accuracy)}) + this.updateLocationCacheIfGood(location) + const best = this.getBestLocationForSend(location) + if (!best) return + socketComms.sendLocationUpdate( - location.coords.latitude, - location.coords.longitude, - location.coords.accuracy ?? undefined, + best.coords.latitude, + best.coords.longitude, + best.coords.accuracy ?? undefined, correlationId, ) } catch (error) { @@ -163,6 +372,22 @@ class MantleManager { } } + public async handleHeadPosition(isUp: boolean) { + this.isHeadUp = isUp + if (this.locationUpdatesMode === "always_on") { + this.wantLocationUpdates = true + await this.startLocationUpdatesIfNeeded() + return + } + + this.wantLocationUpdates = isUp + if (isUp) { + await this.startLocationUpdatesIfNeeded() + } else { + await this.stopLocationUpdatesIfNeeded() + } + } + public async handleLocalTranscription(data: any) { // TODO: performance! const offlineStt = await useSettingsStore.getState().loadSetting(SETTINGS_KEYS.offline_captions_app_running) diff --git a/mobile/src/managers/SocketComms.ts b/mobile/src/managers/SocketComms.ts index 68e385e774..6088d91f57 100644 --- a/mobile/src/managers/SocketComms.ts +++ b/mobile/src/managers/SocketComms.ts @@ -372,8 +372,7 @@ class SocketComms { } private handle_app_state_change(msg: any) { - // console.log("SocketCommsTS: app state change", msg) - // this.parse_app_list(msg) + // Forward to RN layer; UI will derive and notify native about foreground state GlobalEventEmitter.emit("APP_STATE_CHANGE", msg) } @@ -629,11 +628,11 @@ class SocketComms { break case "start_buffer_recording": - this.handle_start_buffer_recording(msg) + this.handle_start_buffer_recording() break case "stop_buffer_recording": - this.handle_stop_buffer_recording(msg) + this.handle_stop_buffer_recording() break case "save_buffer_video": diff --git a/mobile/src/managers/WebSocketManager.ts b/mobile/src/managers/WebSocketManager.ts index 352cd1755c..98570d1128 100644 --- a/mobile/src/managers/WebSocketManager.ts +++ b/mobile/src/managers/WebSocketManager.ts @@ -112,7 +112,7 @@ class WebSocketManager extends EventEmitter { if (store.status === WebSocketStatus.CONNECTED) { clearInterval(this.reconnectInterval) } - }, 5000) + }, 15000) } handleReconnect() { diff --git a/mobile/src/stores/settings.ts b/mobile/src/stores/settings.ts index e1efd9d9a7..0f8e739da3 100644 --- a/mobile/src/stores/settings.ts +++ b/mobile/src/stores/settings.ts @@ -33,6 +33,7 @@ export const SETTINGS_KEYS = { default_wearable: "default_wearable", device_name: "device_name", preferred_mic: "preferred_mic", + mic_activation_mode: "mic_activation_mode", contextual_dashboard_enabled: "contextual_dashboard_enabled", head_up_angle: "head_up_angle", brightness: "brightness", @@ -49,6 +50,7 @@ export const SETTINGS_KEYS = { time_zone: "time_zone", time_zone_override: "time_zone_override", location_tier: "location_tier", + location_updates_mode: "location_updates_mode", offline_captions_app_running: "offline_captions_app_running", camera_app_running: "camera_app_running", SHOW_ADVANCED_SETTINGS: "SHOW_ADVANCED_SETTINGS", @@ -91,6 +93,7 @@ const DEFAULT_SETTINGS: Record = { [SETTINGS_KEYS.default_wearable]: null, [SETTINGS_KEYS.device_name]: "", [SETTINGS_KEYS.preferred_mic]: "phone", + [SETTINGS_KEYS.mic_activation_mode]: "head_up", [SETTINGS_KEYS.contextual_dashboard_enabled]: true, [SETTINGS_KEYS.head_up_angle]: 45, [SETTINGS_KEYS.brightness]: 50, @@ -102,6 +105,7 @@ const DEFAULT_SETTINGS: Record = { [SETTINGS_KEYS.time_zone]: null, [SETTINGS_KEYS.time_zone_override]: null, [SETTINGS_KEYS.location_tier]: null, + [SETTINGS_KEYS.location_updates_mode]: "head_up", [SETTINGS_KEYS.offline_captions_app_running]: false, [SETTINGS_KEYS.camera_app_running]: false, [SETTINGS_KEYS.default_button_action_enabled]: true, @@ -128,6 +132,7 @@ const CORE_SETTINGS_KEYS = [ SETTINGS_KEYS.dashboard_depth, SETTINGS_KEYS.button_mode, SETTINGS_KEYS.button_photo_size, + SETTINGS_KEYS.mic_activation_mode, SETTINGS_KEYS.offline_captions_app_running, ]