diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java index f53d86a1f6..deb72002de 100755 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/AugmentosService.java @@ -1770,12 +1770,12 @@ public void onRequestSingle(String dataType) { } @Override - public void onPhotoRequest(String requestId, String appId, String webhookUrl, String size) { - Log.d(TAG, "Photo request received: requestId=" + requestId + ", appId=" + appId + ", webhookUrl=" + webhookUrl + ", size=" + size); + public void onPhotoRequest(String requestId, String packageName, String webhookUrl, String size) { + Log.d(TAG, "Photo request received: requestId=" + requestId + ", packageName=" + packageName + ", webhookUrl=" + webhookUrl + ", size=" + size); // Forward the request to the smart glasses manager if (smartGlassesManager != null) { - boolean requestSent = smartGlassesManager.requestPhoto(requestId, appId, webhookUrl, size); + boolean requestSent = smartGlassesManager.requestPhoto(requestId, packageName, webhookUrl, size); if (!requestSent) { Log.e(TAG, "Failed to send photo request to glasses"); } @@ -1943,7 +1943,7 @@ public void onAudioPlayRequest(JSONObject audioRequest) { public void onAudioStopRequest(JSONObject audioStopRequest) { // Extract the audio stop request parameters String sessionId = audioStopRequest.optString("sessionId", ""); - String appId = audioStopRequest.optString("appId", ""); + String packageName = audioStopRequest.optString("packageName", ""); // Send the audio stop request as a message to the AugmentOS Manager via BLE if (blePeripheral != null) { @@ -1951,11 +1951,11 @@ public void onAudioStopRequest(JSONObject audioStopRequest) { JSONObject message = new JSONObject(); message.put("type", "audio_stop_request"); message.put("sessionId", sessionId); - message.put("appId", appId); + message.put("packageName", packageName); // Send to AugmentOS Manager blePeripheral.sendDataToAugmentOsManager(message.toString()); - Log.d(TAG, "🔇 Forwarded audio stop request to manager from app: " + appId); + Log.d(TAG, "🔇 Forwarded audio stop request to manager from app: " + packageName); } catch (JSONException e) { Log.e(TAG, "Error creating audio stop request message for manager", e); diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerComms.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerComms.java index e65fa105e4..767a703bd2 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerComms.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerComms.java @@ -622,18 +622,18 @@ public void sendPhotoResponse(String requestId, String photoUrl) { /** * Sends a video stream response message to the server * - * @param appId The ID of the app requesting the stream + * @param packageName The ID of the app requesting the stream * @param streamUrl URL of the video stream */ - public void sendVideoStreamResponse(String appId, String streamUrl) { + public void sendVideoStreamResponse(String packageName, String streamUrl) { try { JSONObject event = new JSONObject(); event.put("type", "video_stream_response"); - event.put("appId", appId); + event.put("packageName", packageName); event.put("streamUrl", streamUrl); event.put("timestamp", System.currentTimeMillis()); wsManager.sendText(event.toString()); - Log.d(TAG, "Sent video stream response for appId: " + appId); + Log.d(TAG, "Sent video stream response for packageName: " + packageName); } catch (JSONException e) { Log.e(TAG, "Error building video_stream_response JSON", e); } @@ -737,14 +737,14 @@ private void handleIncomingMessage(JSONObject msg) { case "photo_request": String requestId = msg.optString("requestId"); - String appId = msg.optString("appId"); + String packageName = msg.optString("packageName"); String webhookUrl = msg.optString("webhookUrl", ""); String size = msg.optString("size", "medium"); - Log.d(TAG, "Received photo_request, requestId: " + requestId + ", appId: " + appId + ", webhookUrl: " + webhookUrl + ", size: " + size); - if (serverCommsCallback != null && !requestId.isEmpty() && !appId.isEmpty()) { - serverCommsCallback.onPhotoRequest(requestId, appId, webhookUrl, size); + Log.d(TAG, "Received photo_request, requestId: " + requestId + ", packageName: " + packageName + ", webhookUrl: " + webhookUrl + ", size: " + size); + if (serverCommsCallback != null && !requestId.isEmpty() && !packageName.isEmpty()) { + serverCommsCallback.onPhotoRequest(requestId, packageName, webhookUrl, size); } else { - Log.e(TAG, "Invalid photo request: missing requestId or appId"); + Log.e(TAG, "Invalid photo request: missing requestId or packageName"); } break; diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerCommsCallback.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerCommsCallback.java index e48ec2f30b..16d030fea6 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerCommsCallback.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/augmentos_backend/ServerCommsCallback.java @@ -20,11 +20,11 @@ public interface ServerCommsCallback { * Called when the server requests a photo to be taken * * @param requestId Unique ID for this photo request - * @param appId ID of the app requesting the photo + * @param packageName ID of the app requesting the photo * @param webhookUrl The webhook URL associated with the photo request * @param size Requested photo size (small|medium|large) */ - void onPhotoRequest(String requestId, String appId, String webhookUrl, String size); + void onPhotoRequest(String requestId, String packageName, String webhookUrl, String size); /** * Called when the server requests an RTMP stream diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/SmartGlassesManager.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/SmartGlassesManager.java index 7803ffca29..b9a145c3c2 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/SmartGlassesManager.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/SmartGlassesManager.java @@ -966,20 +966,20 @@ public boolean sendCustomCommand(String commandJson) { * Request a photo from the connected smart glasses * * @param requestId The unique ID for this photo request - * @param appId The ID of the app requesting the photo + * @param packageName The ID of the app requesting the photo * @param webhookUrl The webhook URL where the photo should be uploaded directly * @param size Requested photo size (small|medium|large) * @return true if request was sent, false if glasses not connected */ - public boolean requestPhoto(String requestId, String appId, String webhookUrl, String size) { + public boolean requestPhoto(String requestId, String packageName, String webhookUrl, String size) { if (smartGlassesRepresentative != null && smartGlassesRepresentative.smartGlassesCommunicator != null && smartGlassesRepresentative.getConnectionState() == SmartGlassesConnectionState.CONNECTED) { - Log.d(TAG, "Requesting photo from glasses, requestId: " + requestId + ", appId: " + appId + ", webhookUrl: " + webhookUrl + ", size=" + size); + Log.d(TAG, "Requesting photo from glasses, requestId: " + requestId + ", packageName: " + packageName + ", webhookUrl: " + webhookUrl + ", size=" + size); // Pass the request to the smart glasses communicator - smartGlassesRepresentative.smartGlassesCommunicator.requestPhoto(requestId, appId, webhookUrl, size); + smartGlassesRepresentative.smartGlassesCommunicator.requestPhoto(requestId, packageName, webhookUrl, size); return true; } else { Log.e(TAG, "Cannot request photo - glasses not connected"); diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/MentraLiveSGC.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/MentraLiveSGC.java index 83c15b2274..8a7cabce0d 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/MentraLiveSGC.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/MentraLiveSGC.java @@ -1562,14 +1562,14 @@ private void processJsonMessage(JSONObject json) { case "photo_response": // Process photo response (success or failure) String requestId = json.optString("requestId", ""); - String appId = json.optString("appId", ""); + String packageName = json.optString("packageName", ""); boolean photoSuccess = json.optBoolean("success", false); if (!photoSuccess) { // Handle failed photo response String errorMsg = json.optString("error", "Unknown error"); Log.d(TAG, "Photo request failed - requestId: " + requestId + - ", appId: " + appId + ", error: " + errorMsg); + ", packageName: " + packageName + ", error: " + errorMsg); } else { // Handle successful photo (in future implementation) Log.d(TAG, "Photo request succeeded - requestId: " + requestId); @@ -2332,14 +2332,14 @@ public void changeSmartGlassesMicrophoneState(boolean enable) { } @Override - public void requestPhoto(String requestId, String appId, String webhookUrl, String size) { - Log.d(TAG, "Requesting photo: " + requestId + " for app: " + appId + " with webhookUrl: " + webhookUrl + ", size=" + size); + public void requestPhoto(String requestId, String packageName, String webhookUrl, String size) { + Log.d(TAG, "Requesting photo: " + requestId + " for app: " + packageName + " with webhookUrl: " + webhookUrl + ", size=" + size); try { JSONObject json = new JSONObject(); json.put("type", "take_photo"); json.put("requestId", requestId); - json.put("appId", appId); + json.put("packageName", packageName); if (webhookUrl != null && !webhookUrl.isEmpty()) { json.put("webhookUrl", webhookUrl); } @@ -2373,7 +2373,7 @@ public void requestRtmpStreamStart(JSONObject message) { // try { JSONObject json = message; json.remove("timestamp"); - json.remove("appId"); + json.remove("packageName"); json.remove("video"); json.remove("audio"); //String rtmpUrl=json.getString("rtmpUrl"); diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/SmartGlassesCommunicator.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/SmartGlassesCommunicator.java index 594aa58a90..1597a83bfe 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/SmartGlassesCommunicator.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/SmartGlassesCommunicator.java @@ -176,10 +176,10 @@ public void sendCustomCommand(String commandJson) { * Requests the smart glasses to take a photo * * @param requestId The unique ID for this photo request - * @param appId The ID of the app requesting the photo + * @param packageName The ID of the app requesting the photo * @param webhookUrl The webhook URL where the photo should be uploaded directly */ - public void requestPhoto(String requestId, String appId, String webhookUrl, String size) { + public void requestPhoto(String requestId, String packageName, String webhookUrl, String size) { // Default implementation does nothing Log.d("SmartGlassesCommunicator", "Photo request (with size) not implemented for this device"); } @@ -189,9 +189,9 @@ public void requestPhoto(String requestId, String appId, String webhookUrl, Stri * Default implementation does nothing - specific communicators should override * * @param requestId The unique ID for this photo request - * @param appId The ID of the app requesting the photo + * @param packageName The ID of the app requesting the photo */ - public void requestPhoto(String requestId, String appId) { + public void requestPhoto(String requestId, String packageName) { // Default implementation does nothing Log.d("SmartGlassesCommunicator", "Photo request not implemented for this device"); } diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/MessageTypes.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/MessageTypes.java index 729b610c9b..b3e995f7a5 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/MessageTypes.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/MessageTypes.java @@ -50,7 +50,7 @@ public class MessageTypes { public static final String PHOTO_RESPONSE = "photo_response"; public static final String PHOTO_REQUEST_ID = "requestId"; public static final String PHOTO_URL = "photoUrl"; - public static final String PHOTO_APP_ID = "appId"; + public static final String PHOTO_APP_ID = "packageName"; //VIDEO STREAM REQUEST diff --git a/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/commands/CommandSystem.java b/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/commands/CommandSystem.java index 552d7f55ed..000a60bd4e 100644 --- a/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/commands/CommandSystem.java +++ b/android_core/app/src/main/java/com/augmentos/augmentos_core/tpa/commands/CommandSystem.java @@ -51,11 +51,11 @@ public class CommandSystem { //command timeout class AppPrivilegeTimeout { - public String appId; + public String packageName; public long timeStart; - AppPrivilegeTimeout(String appId, long timeStart) { - this.appId = appId; + AppPrivilegeTimeout(String packageName, long timeStart) { + this.packageName = packageName; this.timeStart = timeStart; } } @@ -261,7 +261,7 @@ private boolean checkAppHasPrivilege(String eventId, String sendingPackage){ // } // // //if the app responded in good time (and it's the same app that was actually triggered, then allow the request -// if (sendingPackage.equals(appPrivilegeTimeout.appId)) { +// if (sendingPackage.equals(appPrivilegeTimeout.packageName)) { // if ((System.currentTimeMillis() - appPrivilegeTimeout.timeStart) < commandResponseWindowTime) { // return true; // } diff --git a/asg_client/app/src/main/java/com/augmentos/asg_client/camera/PhotoCaptureService.java b/asg_client/app/src/main/java/com/augmentos/asg_client/camera/PhotoCaptureService.java new file mode 100644 index 0000000000..4529d55352 --- /dev/null +++ b/asg_client/app/src/main/java/com/augmentos/asg_client/camera/PhotoCaptureService.java @@ -0,0 +1,339 @@ +package com.augmentos.asg_client.camera; + +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import androidx.annotation.NonNull; +import androidx.preference.PreferenceManager; + +import com.augmentos.augmentos_core.utils.ServerConfigUtil; + +import org.json.JSONException; +import org.json.JSONObject; + +import com.augmentos.augmentos_core.smarterglassesmanager.camera.PhotoUploadService; + +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.UUID; + +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * Service that handles photo capturing and uploading functionality. + * Extracts this logic from AsgClientService to improve modularity. + */ +public class PhotoCaptureService { + private static final String TAG = "PhotoCaptureService"; + + private final Context mContext; + private final PhotoQueueManager mPhotoQueueManager; + private PhotoCaptureListener mPhotoCaptureListener; + + /** + * Interface for listening to photo capture and upload events + */ + public interface PhotoCaptureListener { + void onPhotoCapturing(String requestId); + void onPhotoCaptured(String requestId, String filePath); + void onPhotoUploading(String requestId); + void onPhotoUploaded(String requestId, String url); + void onPhotoError(String requestId, String error); + } + + /** + * Constructor + * + * @param context Application context + * @param photoQueueManager PhotoQueueManager instance + */ + public PhotoCaptureService(@NonNull Context context, @NonNull PhotoQueueManager photoQueueManager) { + mContext = context.getApplicationContext(); + mPhotoQueueManager = photoQueueManager; + } + + /** + * Set a listener for photo capture events + */ + public void setPhotoCaptureListener(PhotoCaptureListener listener) { + this.mPhotoCaptureListener = listener; + } + + /** + * Handles the photo button press by sending a request to the cloud server + * If connected, makes REST API call to server + * If disconnected or server error, takes photo locally + */ + public void handlePhotoButtonPress() { + // Get core token for authentication + String coreToken = PreferenceManager.getDefaultSharedPreferences(mContext) + .getString("core_token", ""); + + // Get device ID for hardware identification + String deviceId = android.os.Build.MODEL + "_" + android.os.Build.SERIAL; + + if (coreToken == null || coreToken.isEmpty()) { + Log.e(TAG, "No core token available, taking photo locally"); + takePhotoLocally(); + return; + } + + // Prepare REST API call + try { + // Get the button press URL from the central config utility + String buttonPressUrl = ServerConfigUtil.getButtonPressUrl(mContext); + + // Create payload for button press event + JSONObject buttonPressPayload = new JSONObject(); + buttonPressPayload.put("buttonId", "photo"); + buttonPressPayload.put("pressType", "short"); + buttonPressPayload.put("deviceId", deviceId); + + Log.d(TAG, "Sending button press event to server: " + buttonPressUrl); + + // Make REST API call with timeout + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(5, java.util.concurrent.TimeUnit.SECONDS) + .writeTimeout(5, java.util.concurrent.TimeUnit.SECONDS) + .readTimeout(5, java.util.concurrent.TimeUnit.SECONDS) + .build(); + + RequestBody requestBody = RequestBody.create( + MediaType.parse("application/json"), + buttonPressPayload.toString() + ); + + Request request = new Request.Builder() + .url(buttonPressUrl) + .header("Authorization", "Bearer " + coreToken) // Use header() for consistency + .post(requestBody) + .build(); + + // Execute request asynchronously + client.newCall(request).enqueue(new Callback() { + @Override + public void onFailure(Call call, IOException e) { + Log.e(TAG, "Failed to send button press event", e); + // Connection failed, take photo locally + takePhotoLocally(); + } + + @Override + public void onResponse(Call call, Response response) { + try { + if (!response.isSuccessful()) { + Log.e(TAG, "Server returned error: " + response.code()); + // Server error, take photo locally + takePhotoLocally(); + return; + } + + // Parse response + String responseBody = response.body().string(); + Log.d(TAG, "Server response: " + responseBody); + JSONObject jsonResponse = new JSONObject(responseBody); + + // Check if we need to take a photo + if ("take_photo".equals(jsonResponse.optString("action"))) { + String requestId = jsonResponse.optString("requestId"); + boolean saveToGallery = jsonResponse.optBoolean("saveToGallery", true); + String packageName = jsonResponse.optString("packageName", "system"); + + Log.d(TAG, "Server requesting photo with requestId: " + requestId); + + // Take photo and upload directly to server + String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(new Date()); + String photoFilePath = mContext.getExternalFilesDir(null) + File.separator + "IMG_" + timeStamp + ".jpg"; + takePhotoAndUpload(photoFilePath, requestId, packageName); + } else { + Log.d(TAG, "Button press handled by server, no photo needed"); + } + } catch (Exception e) { + Log.e(TAG, "Error processing server response", e); + takePhotoLocally(); + } finally { + response.close(); + } + } + }); + } catch (Exception e) { + Log.e(TAG, "Error preparing button press request", e); + // Something went wrong, take photo locally + takePhotoLocally(); + } + } + + /** + * Takes a photo locally when offline or when server communication fails + */ + private void takePhotoLocally() { + // Check storage availability before taking photo + if (!isExternalStorageAvailable()) { + Log.e(TAG, "External storage is not available for photo capture"); + return; + } + + String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(new Date()); + String photoFilePath = mContext.getExternalFilesDir(null) + File.separator + "IMG_" + timeStamp + ".jpg"; + + // Generate a temporary requestId + String requestId = "local_" + timeStamp; + + // For offline mode, take photo and queue it for later upload + CameraNeo.takePictureWithCallback( + mContext, + photoFilePath, + new CameraNeo.PhotoCaptureCallback() { + @Override + public void onPhotoCaptured(String filePath) { + Log.d(TAG, "Offline photo captured successfully at: " + filePath); + + // Queue the photo for later upload + mPhotoQueueManager.queuePhoto(filePath, requestId, "system"); + + // Notify the user about offline mode + Log.d(TAG, "Photo queued for later upload (offline mode)"); + + // Notify through standard capture listener if set up + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoCaptured(requestId, filePath); + mPhotoCaptureListener.onPhotoUploading(requestId); + } + } + + @Override + public void onPhotoError(String errorMessage) { + Log.e(TAG, "Failed to capture offline photo: " + errorMessage); + + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoError(requestId, errorMessage); + } + } + } + ); + } + + /** + * Take a photo and upload it to AugmentOS Cloud + */ + public void takePhotoAndUpload(String photoFilePath, String requestId, String packageName) { + // Notify that we're about to take a photo + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoCapturing(requestId); + } + + try { + // Use CameraNeo for photo capture + CameraNeo.takePictureWithCallback( + mContext, + photoFilePath, + new CameraNeo.PhotoCaptureCallback() { + @Override + public void onPhotoCaptured(String filePath) { + Log.d(TAG, "Photo captured successfully at: " + filePath); + + // Notify that we've captured the photo + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoCaptured(requestId, filePath); + mPhotoCaptureListener.onPhotoUploading(requestId); + } + + // Upload the photo to AugmentOS Cloud + uploadPhotoToCloud(filePath, requestId, packageName); + } + + @Override + public void onPhotoError(String errorMessage) { + Log.e(TAG, "Failed to capture photo: " + errorMessage); + sendPhotoErrorResponse(requestId, packageName, errorMessage); + + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoError(requestId, errorMessage); + } + } + } + ); + } catch (Exception e) { + Log.e(TAG, "Error taking photo", e); + sendPhotoErrorResponse(requestId, packageName, "Error taking photo: " + e.getMessage()); + + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoError(requestId, "Error taking photo: " + e.getMessage()); + } + } + } + + /** + * Upload photo to AugmentOS Cloud + */ + private void uploadPhotoToCloud(String photoFilePath, String requestId, String packageName) { + // Upload the photo to AugmentOS Cloud + PhotoUploadService.uploadPhoto( + mContext, + photoFilePath, + requestId, + new PhotoUploadService.UploadCallback() { + @Override + public void onSuccess(String url) { + Log.d(TAG, "Photo uploaded successfully: " + url); + sendPhotoSuccessResponse(requestId, packageName, url); + + // Notify listener about successful upload + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoUploaded(requestId, url); + } + } + + @Override + public void onFailure(String errorMessage) { + Log.e(TAG, "Photo upload failed: " + errorMessage); + sendPhotoErrorResponse(requestId, packageName, errorMessage); + + // Notify listener about error + if (mPhotoCaptureListener != null) { + mPhotoCaptureListener.onPhotoError(requestId, "Upload failed: " + errorMessage); + } + } + } + ); + } + + /** + * Send a success response for a photo request + * This should be overridden by the service that uses this class + */ + protected void sendPhotoSuccessResponse(String requestId, String packageName, String photoUrl) { + // Default implementation is empty + // This should be overridden by the service that uses this class + } + + /** + * Send an error response for a photo request + * This should be overridden by the service that uses this class + */ + protected void sendPhotoErrorResponse(String requestId, String packageName, String errorMessage) { + // Default implementation is empty + // This should be overridden by the service that uses this class + } + + /** + * Check if external storage is available for read/write + */ + private boolean isExternalStorageAvailable() { + String state = android.os.Environment.getExternalStorageState(); + return android.os.Environment.MEDIA_MOUNTED.equals(state); + } +} \ No newline at end of file diff --git a/asg_client/app/src/main/java/com/augmentos/asg_client/camera/upload/MediaUploadService.java b/asg_client/app/src/main/java/com/augmentos/asg_client/camera/upload/MediaUploadService.java new file mode 100644 index 0000000000..d9ad3bef41 --- /dev/null +++ b/asg_client/app/src/main/java/com/augmentos/asg_client/camera/upload/MediaUploadService.java @@ -0,0 +1,599 @@ +package com.augmentos.asg_client.camera.upload; + +import android.annotation.SuppressLint; +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.PendingIntent; +import android.app.Service; +import android.content.Context; +import android.content.Intent; +import android.content.SharedPreferences; +import android.net.ConnectivityManager; +import android.net.Network; +import android.net.NetworkCapabilities; +import android.os.Binder; +import android.os.Build; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.PowerManager; +import android.util.Log; + +import androidx.preference.PreferenceManager; + +import com.augmentos.augmentos_core.utils.ServerConfigUtil; + +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import androidx.core.app.NotificationCompat; +import androidx.core.app.NotificationManagerCompat; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.IOException; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.augmentos.asg_client.MainActivity; +import com.augmentos.asg_client.R; +import com.augmentos.asg_client.camera.MediaUploadQueueManager; // Updated import + +/** + * Foreground service that manages media (photo/video) uploads in the background. + * Handles processing the media upload queue, retry logic, and user notifications. + */ +public class MediaUploadService extends Service { // Renamed class + + private static final String TAG = "MediaUploadService"; // Renamed TAG + + // Notification constants + private static final String CHANNEL_ID = "media_upload_channel"; // Renamed channel ID + private static final int NOTIFICATION_ID = 1001; + private static final String NOTIFICATION_CHANNEL_NAME = "Media Uploads"; // Updated channel name + private static final String NOTIFICATION_CHANNEL_DESC = "Notifications about media uploads"; // Updated channel desc + + // Actions (remain largely the same, but reflect general media) + public static final String ACTION_START_SERVICE = "com.augmentos.asg_client.action.START_MEDIA_UPLOAD_SERVICE"; + public static final String ACTION_STOP_SERVICE = "com.augmentos.asg_client.action.STOP_MEDIA_UPLOAD_SERVICE"; + public static final String ACTION_PROCESS_QUEUE = "com.augmentos.asg_client.action.PROCESS_MEDIA_QUEUE"; + public static final String ACTION_UPLOAD_STATUS = "com.augmentos.asg_client.action.MEDIA_UPLOAD_STATUS"; + public static final String EXTRA_REQUEST_ID = "request_id"; + public static final String EXTRA_SUCCESS = "success"; + public static final String EXTRA_URL = "url"; + public static final String EXTRA_ERROR = "error"; + public static final String EXTRA_MEDIA_TYPE = "media_type"; // Added for context in notifications/callbacks + + // Queue processing settings + private static final long QUEUE_PROCESSING_INTERVAL = 60000; // 1 minute + private static final int MAX_RETRY_COUNT = 3; + + // Binder for clients + private final IBinder mBinder = new LocalBinder(); + + // Service state + private AtomicBoolean mIsProcessing = new AtomicBoolean(false); + private MediaUploadQueueManager mMediaQueueManager; // Updated type + private Timer mQueueProcessingTimer; + private int mSuccessCount = 0; + private int mFailureCount = 0; + private PowerManager.WakeLock mWakeLock; + + /** + * Class for clients to access the service + */ + public class LocalBinder extends Binder { + public MediaUploadService getService() { // Updated return type + return MediaUploadService.this; + } + } + + /** + * Factory method to start the service with appropriate action + * + * @param context Application context + */ + public static void startService(Context context) { + Intent intent = new Intent(context, MediaUploadService.class); + intent.setAction(ACTION_START_SERVICE); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + context.startForegroundService(intent); + } else { + context.startService(intent); + } + } + + /** + * Factory method to stop the service + * + * @param context Application context + */ + public static void stopService(Context context) { + Intent intent = new Intent(context, MediaUploadService.class); + intent.setAction(ACTION_STOP_SERVICE); + context.startService(intent); + } + + /** + * Factory method to trigger queue processing manually + * + * @param context Application context + */ + public static void processQueue(Context context) { + Intent intent = new Intent(context, MediaUploadService.class); + intent.setAction(ACTION_PROCESS_QUEUE); + context.startService(intent); + } + + /** + * Static method to initiate an upload (used by MediaUploadQueueManager) + */ + public static void uploadMedia(Context context, String filePath, String requestId, int mediaType, UploadCallback callback) { + // Get authentication token from SharedPreferences + String coreToken = PreferenceManager.getDefaultSharedPreferences(context) + .getString("core_token", ""); + + if (coreToken.isEmpty()) { + callback.onFailure("No authentication token available"); + return; + } + + // Create file object and verify it exists + File mediaFile = new File(filePath); + if (!mediaFile.exists()) { + callback.onFailure("Media file does not exist: " + filePath); + return; + } + + // Get device ID + String deviceId = android.os.Build.MODEL + "_" + android.os.Build.SERIAL; + + // Get appropriate upload URL based on media type + String uploadUrl; + MediaType mediaContentType; + + if (mediaType == MediaUploadQueueManager.MEDIA_TYPE_PHOTO) { + uploadUrl = ServerConfigUtil.getPhotoUploadUrl(context); + mediaContentType = MediaType.parse("image/jpeg"); + } else if (mediaType == MediaUploadQueueManager.MEDIA_TYPE_VIDEO) { + uploadUrl = ServerConfigUtil.getVideoUploadUrl(context); + mediaContentType = MediaType.parse("video/mp4"); + } else { + callback.onFailure("Invalid media type: " + mediaType); + return; + } + + uploadUrl = "https://dev.augmentos.org:443/api/photos/upload"; + + Log.d(TAG, "Uploading media to: " + uploadUrl); + + try { + // Create HTTP client with appropriate timeouts + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(30, java.util.concurrent.TimeUnit.SECONDS) + .writeTimeout(60, java.util.concurrent.TimeUnit.SECONDS) + .readTimeout(60, java.util.concurrent.TimeUnit.SECONDS) + .retryOnConnectionFailure(true) // Enable retries + .build(); + + // Log network state + ConnectivityManager connectivityManager = + (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE); + Network activeNetwork = connectivityManager.getActiveNetwork(); + if (activeNetwork != null) { + NetworkCapabilities capabilities = + connectivityManager.getNetworkCapabilities(activeNetwork); + if (capabilities != null) { + boolean hasInternet = capabilities.hasCapability( + NetworkCapabilities.NET_CAPABILITY_INTERNET); + boolean validatedInternet = capabilities.hasCapability( + NetworkCapabilities.NET_CAPABILITY_VALIDATED); + Log.d(TAG, "Network state - Internet: " + hasInternet + + ", Validated: " + validatedInternet); + } + } + + // Build JSON metadata + JSONObject metadata = new JSONObject(); + metadata.put("requestId", requestId); + metadata.put("deviceId", deviceId); + metadata.put("timestamp", System.currentTimeMillis()); + metadata.put("mediaType", mediaType == MediaUploadQueueManager.MEDIA_TYPE_PHOTO ? "photo" : "video"); + metadata.put("packageName", "asg_client"); // Add packageName + + // Create multipart request + RequestBody requestBody = new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("file", mediaFile.getName(), + RequestBody.create(mediaContentType, mediaFile)) + .addFormDataPart("metadata", metadata.toString()) + .build(); + + // Build the request + Request request = new Request.Builder() + .url(uploadUrl) + .header("Authorization", "Bearer " + coreToken) + .post(requestBody) + .build(); + +// Log.d(TAG, "Prepared upload request for: " + filePath); + + // Log detailed request information + StringBuilder requestLog = new StringBuilder(); + requestLog.append("\n=== Request Details ===\n"); + requestLog.append("URL: ").append(request.url()).append("\n"); + requestLog.append("Method: ").append(request.method()).append("\n"); + requestLog.append("Headers:\n"); + request.headers().forEach(header -> + requestLog.append(" ").append(header.getFirst()).append(": ") + .append(header.getSecond()) + .append("\n") + ); + requestLog.append("Metadata: ").append(metadata.toString()).append("\n"); + requestLog.append("File name: ").append(mediaFile.getName()).append("\n"); + requestLog.append("File size: ").append(mediaFile.length()).append(" bytes\n"); + requestLog.append("Media type: ").append(mediaContentType).append("\n"); + requestLog.append("===================="); + + Log.d(TAG, requestLog.toString()); + + // Execute the request + client.newCall(request).enqueue(new Callback() { + @Override + public void onFailure(Call call, IOException e) { + String errorMsg = "Network error during upload: " + e.getMessage(); + Log.e(TAG, errorMsg); + callback.onFailure(errorMsg); + } + + @Override + public void onResponse(Call call, Response response) { + try { + if (!response.isSuccessful()) { + String errorMsg = "Server error: " + response.code(); + Log.e(TAG, errorMsg); + callback.onFailure(errorMsg); + return; + } + + // Parse the response + String responseBody = response.body().string(); + JSONObject jsonResponse = new JSONObject(responseBody); + + // Check if response contains URL + if (jsonResponse.has("url")) { + String url = jsonResponse.getString("url"); + Log.d(TAG, "Media upload successful, URL: " + url); + callback.onSuccess(url); + } else { + Log.e(TAG, "Invalid server response - missing URL"); + callback.onFailure("Invalid server response - missing URL"); + } + } catch (Exception e) { + String errorMsg = "Error processing server response: " + e.getMessage(); + Log.e(TAG, errorMsg); + callback.onFailure(errorMsg); + } finally { + response.close(); + } + } + }); + } catch (Exception e) { + String errorMsg = "Error preparing upload request: " + e.getMessage(); + Log.e(TAG, errorMsg); + callback.onFailure(errorMsg); + } + } + + // Callback interface for upload results + public interface UploadCallback { + void onSuccess(String url); + void onFailure(String errorMessage); + } + + @Override + public void onCreate() { + super.onCreate(); + Log.d(TAG, "Service created"); + + // Initialize the MediaUploadQueueManager + mMediaQueueManager = new MediaUploadQueueManager(getApplicationContext()); // Updated instantiation + + // Set up queue callback + setupQueueCallbacks(); + + // Create notification channel + createNotificationChannel(); + + // Acquire wake lock to ensure service keeps running + PowerManager powerManager = (PowerManager) getSystemService(POWER_SERVICE); + if (powerManager != null) { + mWakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, + "AugmentOS:MediaUploadWakeLock"); // Updated wake lock tag + } else { + Log.e(TAG, "PowerManager not available"); + } + } + + @Override + public int onStartCommand(Intent intent, int flags, int startId) { + if (intent == null || intent.getAction() == null) { + Log.e(TAG, "Service started with null intent or action"); + return START_STICKY; + } + + String action = intent.getAction(); + Log.d(TAG, "Service received action: " + action); + + switch (action) { + case ACTION_START_SERVICE: + startForeground(NOTIFICATION_ID, createNotification("Starting media upload service...")); + startQueueProcessing(); + break; + + case ACTION_STOP_SERVICE: + stopQueueProcessing(); + stopForeground(true); + stopSelf(); + break; + + case ACTION_PROCESS_QUEUE: + processQueueNow(); + break; + + case ACTION_UPLOAD_STATUS: + handleUploadStatus(intent); + break; + } + + return START_STICKY; + } + + @Override + public IBinder onBind(Intent intent) { + return mBinder; + } + + @Override + public void onDestroy() { + Log.d(TAG, "Service destroyed"); + stopQueueProcessing(); + + // Release wake lock if held + if (mWakeLock != null && mWakeLock.isHeld()) { + mWakeLock.release(); + } + + super.onDestroy(); + } + + /** + * Create the notification channel for Android O and above + */ + private void createNotificationChannel() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + NotificationChannel channel = new NotificationChannel( + CHANNEL_ID, + NOTIFICATION_CHANNEL_NAME, + NotificationManager.IMPORTANCE_LOW); + channel.setDescription(NOTIFICATION_CHANNEL_DESC); + + NotificationManager notificationManager = getSystemService(NotificationManager.class); + if (notificationManager != null) { + notificationManager.createNotificationChannel(channel); + } + } + } + + /** + * Create notification with current status + */ + private Notification createNotification(String contentText) { + // Create an intent to open the app when notification is tapped + Intent intent = new Intent(this, MainActivity.class); + PendingIntent pendingIntent = PendingIntent.getActivity( + this, 0, intent, PendingIntent.FLAG_IMMUTABLE); + + // Get queue stats + JSONObject stats = mMediaQueueManager.getQueueStats(); + int totalCount = stats.optInt("totalCount", 0); + int queuedCount = stats.optInt("queuedCount", 0); + int uploadingCount = stats.optInt("uploadingCount", 0); + int failedCount = stats.optInt("failedCount", 0); + + String statusContent; + if (totalCount == 0) { + statusContent = "No media items in queue"; + } else { + statusContent = String.format("Queue: %d items (%d waiting, %d in progress, %d failed)", + totalCount, queuedCount, uploadingCount, failedCount); + } + + // Create the notification + return new NotificationCompat.Builder(this, CHANNEL_ID) + .setContentTitle("Media Upload Service") // Updated title + .setContentText(contentText) + .setStyle(new NotificationCompat.BigTextStyle() + .bigText(contentText + "\n" + statusContent)) + .setSmallIcon(R.drawable.ic_launcher_foreground) + .setPriority(NotificationCompat.PRIORITY_LOW) + .setContentIntent(pendingIntent) + .setOngoing(true) + .build(); + } + + /** + * Update the notification with new content + */ + private void updateNotification(String contentText) { + Notification notification = createNotification(contentText); + + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(this); + notificationManager.notify(NOTIFICATION_ID, notification); + } + + /** + * Set up callbacks for the MediaUploadQueueManager + */ + private void setupQueueCallbacks() { + mMediaQueueManager.setMediaQueueCallback(new MediaUploadQueueManager.MediaQueueCallback() { + @Override + public void onMediaQueued(String requestId, String filePath, int mediaType) { + Log.d(TAG, "Media queued: " + requestId + " (type: " + mediaType + ")"); + updateNotification("Media queued: " + requestId); + } + + @Override + public void onMediaUploaded(String requestId, String url, int mediaType) { + Log.d(TAG, "Media uploaded: " + requestId + ", URL: " + url + " (type: " + mediaType + ")"); + mSuccessCount++; + + // Send status broadcast + Intent statusIntent = new Intent(ACTION_UPLOAD_STATUS); + statusIntent.putExtra(EXTRA_REQUEST_ID, requestId); + statusIntent.putExtra(EXTRA_SUCCESS, true); + statusIntent.putExtra(EXTRA_URL, url); + statusIntent.putExtra(EXTRA_MEDIA_TYPE, mediaType); + sendBroadcast(statusIntent); // Permission check might be needed here for some Android versions + + updateNotification("Media uploaded successfully"); + } + + @SuppressLint("MissingPermission") + // Assuming internal broadcast, otherwise add permission check + @Override + public void onMediaUploadFailed(String requestId, String error, int mediaType) { + Log.e(TAG, "Media upload failed: " + requestId + ", error: " + error + " (type: " + mediaType + ")"); + mFailureCount++; + + Intent statusIntent = new Intent(ACTION_UPLOAD_STATUS); + statusIntent.putExtra(EXTRA_REQUEST_ID, requestId); + statusIntent.putExtra(EXTRA_SUCCESS, false); + statusIntent.putExtra(EXTRA_ERROR, error); + statusIntent.putExtra(EXTRA_MEDIA_TYPE, mediaType); + sendBroadcast(statusIntent); + + updateNotification("Media upload failed: " + error); + } + }); + } + + /** + * Start the periodic queue processing + */ + private void startQueueProcessing() { + if (mQueueProcessingTimer != null) { + mQueueProcessingTimer.cancel(); + } + + mQueueProcessingTimer = new Timer(true); + mQueueProcessingTimer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + processQueueNow(); + } + }, 0, QUEUE_PROCESSING_INTERVAL); + + // Acquire wake lock + if (mWakeLock != null && !mWakeLock.isHeld()) { + mWakeLock.acquire(24 * 60 * 60 * 1000L); // 24 hours max + } + + updateNotification("Media upload service is running"); + } + + /** + * Stop the periodic queue processing + */ + private void stopQueueProcessing() { + if (mQueueProcessingTimer != null) { + mQueueProcessingTimer.cancel(); + mQueueProcessingTimer = null; + } + + // Release wake lock if held + if (mWakeLock != null && mWakeLock.isHeld()) { + mWakeLock.release(); + } + } + + /** + * Process the queue now + */ + private void processQueueNow() { + // Only process if not already processing and there are queued media items + if (mIsProcessing.compareAndSet(false, true)) { + new Handler(Looper.getMainLooper()).post(() -> { + updateNotification("Processing media upload queue..."); + }); + + // If there are failed uploads, retry them + mMediaQueueManager.retryFailedUploads(MAX_RETRY_COUNT); + + // Process the queue + mMediaQueueManager.processQueue(); + + // Reset processing flag + mIsProcessing.set(false); + } + } + + /** + * Handle upload status from intent + */ + private void handleUploadStatus(Intent intent) { + String requestId = intent.getStringExtra(EXTRA_REQUEST_ID); + boolean success = intent.getBooleanExtra(EXTRA_SUCCESS, false); + int mediaType = intent.getIntExtra(EXTRA_MEDIA_TYPE, MediaUploadQueueManager.MEDIA_TYPE_PHOTO); + + if (success) { + String url = intent.getStringExtra(EXTRA_URL); + Log.d(TAG, "Upload succeeded for request: " + requestId + ", URL: " + url + " (type: " + mediaType + ")"); + updateNotification("Media uploaded successfully: " + requestId); + } else { + String error = intent.getStringExtra(EXTRA_ERROR); + Log.e(TAG, "Upload failed for request: " + requestId + ", error: " + error + " (type: " + mediaType + ")"); + updateNotification("Media upload failed: " + requestId + " (" + error + ")"); + } + } + + /** + * Get statistics about uploads and the queue + */ + public JSONObject getStatistics() { + JSONObject stats = new JSONObject(); + try { + // Get queue stats + JSONObject queueStats = mMediaQueueManager.getQueueStats(); + + // Add our own tracking stats + stats.put("queueStats", queueStats); + stats.put("successCount", mSuccessCount); + stats.put("failureCount", mFailureCount); + stats.put("isProcessing", mIsProcessing.get()); + + } catch (JSONException e) { + Log.e(TAG, "Error creating statistics JSON", e); + } + + return stats; + } + + /** + * Get the media queue manager instance + */ + public MediaUploadQueueManager getMediaQueueManager() { // Updated return type + return mMediaQueueManager; + } +} diff --git a/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/MediaUploadQueueManager.java b/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/MediaUploadQueueManager.java index fefa62cfda..2a23d2569c 100644 --- a/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/MediaUploadQueueManager.java +++ b/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/MediaUploadQueueManager.java @@ -232,7 +232,7 @@ public void processQueue() { // Only process queued media in this pass if (STATUS_QUEUED.equals(status)) { String requestId = media.getString("requestId"); - String appId = media.getString("appId"); + String packageName = media.getString("packageName"); String queuedPath = media.getString("queuedPath"); int mediaType = media.getInt("mediaType"); // Get mediaType @@ -242,7 +242,7 @@ public void processQueue() { updateMediaInManifest(i, media); // Attempt to upload the media - uploadMedia(queuedPath, requestId, appId, mediaType, i); + uploadMedia(queuedPath, requestId, packageName, mediaType, i); processed++; } @@ -261,7 +261,7 @@ public void processQueue() { /** * Upload a media item from the queue */ - private void uploadMedia(String queuedPath, String requestId, String appId, int mediaType, int index) { + private void uploadMedia(String queuedPath, String requestId, String packageName, int mediaType, int index) { MediaUploadService.uploadMedia( mContext, queuedPath, @@ -582,7 +582,7 @@ public boolean rebuildManifest() { // Create a new entry for this file JSONObject mediaEntry = new JSONObject(); mediaEntry.put("requestId", requestId); - mediaEntry.put("appId", "system"); + mediaEntry.put("packageName", "system"); mediaEntry.put("originalPath", ""); mediaEntry.put("queuedPath", file.getAbsolutePath()); mediaEntry.put("mediaType", mediaType); diff --git a/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/PhotoQueueManager.java b/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/PhotoQueueManager.java index 654c065773..6bdc13683b 100644 --- a/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/PhotoQueueManager.java +++ b/asg_client/app/src/main/java/com/augmentos/asg_client/io/media/managers/PhotoQueueManager.java @@ -187,10 +187,10 @@ public boolean queuePhoto(String photoFilePath, String requestId) { * * @param photoFilePath Path to the photo file * @param requestId Request ID associated with this photo - * @param appId App ID that requested the photo + * @param packageName App ID that requested the photo * @return true if successfully queued, false otherwise */ - public boolean queuePhoto(String photoFilePath, String requestId, String appId) { + public boolean queuePhoto(String photoFilePath, String requestId, String packageName) { File photoFile = new File(photoFilePath); // Check if file exists @@ -210,7 +210,7 @@ public boolean queuePhoto(String photoFilePath, String requestId, String appId) // Add to manifest JSONObject photoEntry = new JSONObject(); photoEntry.put("requestId", requestId); - photoEntry.put("appId", appId); + photoEntry.put("packageName", packageName); photoEntry.put("originalPath", photoFilePath); photoEntry.put("queuedPath", queuedFile.getAbsolutePath()); photoEntry.put("status", STATUS_QUEUED); @@ -266,7 +266,7 @@ public void processQueue() { // Only process queued photos in this pass if (STATUS_QUEUED.equals(status)) { String requestId = photo.getString("requestId"); - String appId = photo.getString("appId"); + String packageName = photo.getString("packageName"); String queuedPath = photo.getString("queuedPath"); // Update status to uploading @@ -275,7 +275,7 @@ public void processQueue() { updatePhotoInManifest(i, photo); // Attempt to upload the photo - uploadPhoto(queuedPath, requestId, appId, i); + uploadPhoto(queuedPath, requestId, packageName, i); processed++; } @@ -294,7 +294,7 @@ public void processQueue() { /** * Upload a photo from the queue */ - private void uploadPhoto(String queuedPath, String requestId, String appId, int index) { + private void uploadPhoto(String queuedPath, String requestId, String packageName, int index) { PhotoUploadService.uploadPhoto( mContext, queuedPath, @@ -603,7 +603,7 @@ public boolean rebuildManifest() { // Create a new entry for this file JSONObject photoEntry = new JSONObject(); photoEntry.put("requestId", requestId); - photoEntry.put("appId", "system"); + photoEntry.put("packageName", "system"); photoEntry.put("originalPath", ""); photoEntry.put("queuedPath", file.getAbsolutePath()); photoEntry.put("status", STATUS_QUEUED); diff --git a/asg_client/app/src/test/java/com/augmentos/asg_client/camera/MediaUploadTest.java b/asg_client/app/src/test/java/com/augmentos/asg_client/camera/MediaUploadTest.java index bf886b5dde..24a27680bf 100644 --- a/asg_client/app/src/test/java/com/augmentos/asg_client/camera/MediaUploadTest.java +++ b/asg_client/app/src/test/java/com/augmentos/asg_client/camera/MediaUploadTest.java @@ -64,7 +64,7 @@ public void testPhotoUpload() throws Exception { metadata.put("deviceId", "K900_unknown"); metadata.put("timestamp", System.currentTimeMillis()); metadata.put("mediaType", "photo"); - metadata.put("appId", "asg_client"); + metadata.put("packageName", "asg_client"); System.out.println("Metadata: " + metadata.toString()); diff --git a/cloud/developer-portal/src/components/dialogs/ApiKeyDialog.tsx b/cloud/developer-portal/src/components/dialogs/ApiKeyDialog.tsx index 972547d0e4..ccc3d2a641 100644 --- a/cloud/developer-portal/src/components/dialogs/ApiKeyDialog.tsx +++ b/cloud/developer-portal/src/components/dialogs/ApiKeyDialog.tsx @@ -32,7 +32,7 @@ const ApiKeyDialog: React.FC = ({ const [error, setError] = useState(null); const [success, setSuccess] = useState(null); const [lastRegenerated, setLastRegenerated] = useState(new Date()); - const [currentAppId, setCurrentAppId] = useState(null); + const [currentPackageName, setCurrentPackageName] = useState(null); // Format API key to be partially masked const formatApiKey = (key: string): string => { @@ -99,11 +99,11 @@ const ApiKeyDialog: React.FC = ({ // Complete reset of dialog state when App changes useEffect(() => { if (app) { - const appId = app.packageName; + const packageName = app.packageName; // Only reset state if App has changed - if (currentAppId !== appId) { - console.log(`App changed from ${currentAppId} to ${appId}, resetting dialog state`); + if (currentPackageName !== packageName) { + console.log(`App changed from ${currentPackageName} to ${packageName}, resetting dialog state`); // Reset all state setApiKey(''); @@ -112,11 +112,11 @@ const ApiKeyDialog: React.FC = ({ setShowConfirmation(false); setIsCopied(false); - // Update current App ID tracker - setCurrentAppId(appId); + // Update current Package Name tracker + setCurrentPackageName(packageName); } } - }, [app, currentAppId]); + }, [app, currentPackageName]); // Update local state when apiKey prop changes (only if it's a real key) useEffect(() => { diff --git a/cloud/package.json b/cloud/package.json index 081b2996bd..2599aa4e0f 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -17,6 +17,7 @@ "dev:setup-network": "docker network create augmentos-network-dev", "dev:clean": "docker compose -f docker-compose.dev.yml -p dev down -v && docker system prune -f", "ngrok:isaiah": "ngrok http --url=isaiah.augmentos.cloud 8002", + "ngrok:aryan": " ngrok http --url=aryan-cloud.ngrok.app 8002", "prod": "docker compose -f docker-compose.yml -p prod up -d", "prod:stop": "docker compose -f docker-compose.yml -p prod down", "prod:deploy": "bun run prod:stop && bun run prod", diff --git a/cloud/packages/cloud/src/models/gallery-photo.model.ts b/cloud/packages/cloud/src/models/gallery-photo.model.ts index d86bbbce39..1960ae3dd6 100644 --- a/cloud/packages/cloud/src/models/gallery-photo.model.ts +++ b/cloud/packages/cloud/src/models/gallery-photo.model.ts @@ -8,7 +8,7 @@ export interface GalleryPhotoDocument extends Document { filename: string; // Filename in storage photoUrl: string; // URL to access the photo requestId: string; // Original request ID that triggered the photo - appId: string; // App that requested the photo + packageName: string; // App that requested the photo uploadDate: Date; // When the photo was uploaded metadata?: { originalFilename?: string; @@ -41,7 +41,7 @@ const GalleryPhotoSchema = new Schema({ type: String, required: true }, - appId: { + packageName: { type: String, required: true }, diff --git a/cloud/packages/cloud/src/services/core/photo-request.service.ts b/cloud/packages/cloud/src/services/core/photo-request.service.ts index c381d01159..57a43c125f 100644 --- a/cloud/packages/cloud/src/services/core/photo-request.service.ts +++ b/cloud/packages/cloud/src/services/core/photo-request.service.ts @@ -32,7 +32,7 @@ export interface PendingPhotoRequest { userId: string; // User ID who initiated the request timestamp: number; // When the request was created origin: PhotoRequestOrigin; // Whether this is from system or App - appId?: string; // App ID (required for App requests, optional for system) + packageName?: string; // App ID (required for App requests, optional for system) ws?: WebSocket; // WebSocket connection (only for App requests) saveToGallery: boolean; // Whether to save to gallery (defaults true for system, configurable for App) } @@ -105,14 +105,14 @@ class PhotoRequestService { * Create a new App-initiated photo request * * @param userId User ID who initiated the request - * @param appId App ID that requested the photo + * @param packageName App ID that requested the photo * @param ws WebSocket connection to send the response to * @param config Optional configuration options * @returns The request ID for the new photo request */ createAppPhotoRequest( userId: string, - appId: string, + packageName: string, ws: WebSocket, config?: PhotoRequestConfig ): string { @@ -127,12 +127,12 @@ class PhotoRequestService { userId, timestamp, origin: 'app', - appId, + packageName, ws, saveToGallery }); - logger.info(`[PhotoRequestService] Created App photo request: ${requestId} for app ${appId}, user ${userId}`); + logger.info(`[PhotoRequestService] Created App photo request: ${requestId} for package ${packageName}, user ${userId}`); // Set timeout to clean up if not used this.setRequestTimeout(requestId, timeoutMs); @@ -195,7 +195,7 @@ class PhotoRequestService { }; pendingRequest.ws.send(JSON.stringify(photoResponse)); - logger.info(`[PhotoRequestService] Photo response sent to App ${pendingRequest.appId}, requestId: ${requestId}`); + logger.info(`[PhotoRequestService] Photo response sent to App ${pendingRequest.packageName}, requestId: ${requestId}`); } else if (pendingRequest.origin === 'system') { // For system requests, we don't need to forward anything, just log it logger.info(`[PhotoRequestService] System photo request completed: ${requestId}`); diff --git a/cloud/packages/cloud/src/services/core/stream-tracker.service.ts b/cloud/packages/cloud/src/services/core/stream-tracker.service.ts index 6e12eeb677..fd855aac6b 100644 --- a/cloud/packages/cloud/src/services/core/stream-tracker.service.ts +++ b/cloud/packages/cloud/src/services/core/stream-tracker.service.ts @@ -13,7 +13,7 @@ import crypto from 'crypto'; interface StreamInfo { streamId: string; sessionId: string; - appId: string; + packageName: string; rtmpUrl: string; status: 'initializing' | 'active' | 'stopping' | 'stopped' | 'timeout'; startTime: Date; @@ -33,8 +33,8 @@ export class StreamTrackerService { /** * Start tracking a new stream */ - public startTracking(streamId: string, sessionId: string, appId: string, rtmpUrl: string): void { - logger.info(`[StreamTracker]: Starting tracking for streamId: ${streamId}, app: ${appId}`); + public startTracking(streamId: string, sessionId: string, packageName: string, rtmpUrl: string): void { + logger.info(`[StreamTracker]: Starting tracking for streamId: ${streamId}, packageName: ${packageName}`); // Cancel existing stream if any this.stopTracking(streamId); @@ -43,7 +43,7 @@ export class StreamTrackerService { const streamInfo: StreamInfo = { streamId, sessionId, - appId, + packageName, rtmpUrl, status: 'initializing', startTime: now, @@ -115,8 +115,8 @@ export class StreamTrackerService { /** * Get all active streams for an app */ - public getStreamsForApp(appId: string): StreamInfo[] { - return Array.from(this.streams.values()).filter(stream => stream.appId === appId); + public getStreamsForApp(packageName: string): StreamInfo[] { + return Array.from(this.streams.values()).filter(stream => stream.packageName === packageName); } /** diff --git a/cloud/packages/cloud/src/services/debug/debug-service.ts b/cloud/packages/cloud/src/services/debug/debug-service.ts index a497208825..96a9710527 100644 --- a/cloud/packages/cloud/src/services/debug/debug-service.ts +++ b/cloud/packages/cloud/src/services/debug/debug-service.ts @@ -69,7 +69,7 @@ type DebuggerEvent = | { type: 'SESSION_UPDATE'; sessionId: string; data: Partial } | { type: 'SESSION_DISCONNECTED'; sessionId: string; timestamp: string } | { type: 'SESSION_CONNECTED'; sessionId: string; timestamp: string } - | { type: 'APP_STATE_CHANGE'; sessionId: string; appId: string; state: any } + | { type: 'APP_STATE_CHANGE'; sessionId: string; packageName: string; state: any } | { type: 'DISPLAY_UPDATE'; sessionId: string; display: any } | { type: 'TRANSCRIPTION_UPDATE'; sessionId: string; transcript: any } | { type: 'SYSTEM_STATS_UPDATE'; stats: SystemStats }; @@ -334,13 +334,13 @@ export class DebugService { } } - public updateAppState(sessionId: string, appId: string, state: any) { + public updateAppState(sessionId: string, packageName: string, state: any) { if (!this.isActive) return; // Skip if debug service is not active this.broadcastEvent({ type: 'APP_STATE_CHANGE', sessionId, - appId, + packageName, state }); } diff --git a/cloud/packages/cloud/src/services/session/PhotoManager.ts b/cloud/packages/cloud/src/services/session/PhotoManager.ts index 10a73149db..ae3fed6616 100644 --- a/cloud/packages/cloud/src/services/session/PhotoManager.ts +++ b/cloud/packages/cloud/src/services/session/PhotoManager.ts @@ -127,7 +127,7 @@ export class PhotoManager { type: CloudToGlassesMessageType.PHOTO_REQUEST, sessionId: this.userSession.sessionId, requestId, - appId: packageName, // Glasses expect `appId` + packageName: packageName, // Glasses expect `packageName` webhookUrl, // Use custom webhookUrl if provided, otherwise default size, // Propagate desired size timestamp: new Date(), diff --git a/cloud/packages/cloud/src/services/session/VideoManager.ts b/cloud/packages/cloud/src/services/session/VideoManager.ts index 62a907d445..6c6a99dfad 100644 --- a/cloud/packages/cloud/src/services/session/VideoManager.ts +++ b/cloud/packages/cloud/src/services/session/VideoManager.ts @@ -121,7 +121,7 @@ export class VideoManager { type: CloudToGlassesMessageType.START_RTMP_STREAM, sessionId: this.userSession.sessionId, rtmpUrl, - appId: packageName, + packageName: packageName, streamId, video: video || {}, audio: audio || {}, @@ -469,7 +469,7 @@ export class VideoManager { const stopMessage: StopRtmpStream = { type: CloudToGlassesMessageType.STOP_RTMP_STREAM, sessionId: this.userSession.sessionId, - appId: packageName, + packageName: packageName, streamId: streamId || '', timestamp: new Date(), }; @@ -504,7 +504,7 @@ export class VideoManager { status, // The SDK status string errorDetails, stats, - appId: packageName, // Clarify which app this status pertains to + packageName: packageName, // Clarify which app this status pertains to timestamp: new Date(), }; @@ -543,7 +543,7 @@ export class VideoManager { streamId, status, errorDetails, - appId: packageName, + packageName: packageName, stats, timestamp: new Date(), }; diff --git a/cloud/packages/cloud/src/services/streaming/ManagedStreamingExtension.ts b/cloud/packages/cloud/src/services/streaming/ManagedStreamingExtension.ts index 0babf43ec6..491f47fbec 100644 --- a/cloud/packages/cloud/src/services/streaming/ManagedStreamingExtension.ts +++ b/cloud/packages/cloud/src/services/streaming/ManagedStreamingExtension.ts @@ -136,7 +136,7 @@ export class ManagedStreamingExtension { // Add viewer to existing stream const managedStream = this.stateManager.createOrJoinManagedStream({ userId, - appId: packageName, + packageName: packageName, liveInput: { liveInputId: existingStream.cfLiveInputId, rtmpUrl: existingStream.cfIngestUrl, @@ -205,7 +205,7 @@ export class ManagedStreamingExtension { ); const managedStream = this.stateManager.createOrJoinManagedStream({ userId, - appId: packageName, + packageName: packageName, liveInput, }); @@ -228,7 +228,7 @@ export class ManagedStreamingExtension { type: CloudToGlassesMessageType.START_RTMP_STREAM, sessionId: userSession.sessionId, rtmpUrl: liveInput.rtmpUrl, // Cloudflare ingest URL - appId: "MANAGED_STREAM", // Special app ID for managed streams + packageName: "MANAGED_STREAM", // Special package name for managed streams streamId: managedStream.streamId, video: video || {}, audio: audio || {}, @@ -373,10 +373,10 @@ export class ManagedStreamingExtension { } // Send status to all viewers - for (const appId of stream.activeViewers) { + for (const packageName of stream.activeViewers) { await this.sendManagedStreamStatus( userSession, - appId, + packageName, stream.streamId, mappedStatus, ); @@ -692,10 +692,10 @@ export class ManagedStreamingExtension { if (!userSession) return; // Send updated status to all viewers - for (const appId of stream.activeViewers) { + for (const packageName of stream.activeViewers) { await this.sendManagedStreamStatus( userSession, - appId, + packageName, stream.streamId, "active", "Outputs updated", @@ -763,10 +763,10 @@ export class ManagedStreamingExtension { } // Send status update to all apps viewing this stream - for (const appId of managedStream.activeViewers) { + for (const packageName of managedStream.activeViewers) { await this.sendManagedStreamStatus( userSession, - appId, + packageName, managedStream.streamId, "active", "Stream is now live", @@ -1061,7 +1061,7 @@ export class ManagedStreamingExtension { const stopMessage: StopRtmpStream = { type: CloudToGlassesMessageType.STOP_RTMP_STREAM, sessionId: userSession.sessionId, - appId: "MANAGED_STREAM", // Same special app ID used when starting + packageName: "MANAGED_STREAM", // Same special app ID used when starting streamId: stream.streamId, timestamp: new Date(), }; diff --git a/cloud/packages/cloud/src/services/streaming/StreamStateManager.ts b/cloud/packages/cloud/src/services/streaming/StreamStateManager.ts index b6c8ebb103..16c87b8d44 100644 --- a/cloud/packages/cloud/src/services/streaming/StreamStateManager.ts +++ b/cloud/packages/cloud/src/services/streaming/StreamStateManager.ts @@ -43,7 +43,7 @@ export interface ManagedStreamState extends BaseStreamState { export interface UnmanagedStreamState extends BaseStreamState { type: "unmanaged"; rtmpUrl: string; - requestingAppId: string; + requestingPackageName: string; streamId: string; } @@ -57,7 +57,7 @@ export type StreamState = ManagedStreamState | UnmanagedStreamState; */ export interface CreateManagedStreamOptions { userId: string; - appId: string; + packageName: string; liveInput: LiveInputResult; } @@ -148,20 +148,20 @@ export class StreamStateManager { createOrJoinManagedStream( options: CreateManagedStreamOptions, ): ManagedStreamState { - const { userId, appId, liveInput } = options; + const { userId, packageName, liveInput } = options; // Check if user already has a managed stream const existingStream = this.userStreams.get(userId); if (existingStream && existingStream.type === "managed") { // Add viewer to existing stream - existingStream.activeViewers.add(appId); + existingStream.activeViewers.add(packageName); existingStream.lastActivity = new Date(); this.logger.info( { userId, - appId, + packageName, viewerCount: existingStream.activeViewers.size, }, "Added viewer to existing managed stream", @@ -180,7 +180,7 @@ export class StreamStateManager { hlsUrl: liveInput.hlsUrl, dashUrl: liveInput.dashUrl, webrtcUrl: liveInput.webrtcUrl, - activeViewers: new Set([appId]), + activeViewers: new Set([packageName]), streamId, createdAt: new Date(), lastActivity: new Date(), @@ -188,7 +188,7 @@ export class StreamStateManager { cfOutputId: output.uid, url: output.url, name: undefined, // Will be set later if needed - addedBy: appId, // Initial outputs are owned by the app that created the stream + addedBy: packageName, // Initial outputs are owned by the app that created the stream status: output, })), }; @@ -201,7 +201,7 @@ export class StreamStateManager { this.logger.info( { userId, - appId, + packageName, streamId, cfLiveInputId: liveInput.liveInputId, }, @@ -214,20 +214,20 @@ export class StreamStateManager { /** * Remove a viewer from a managed stream */ - removeViewerFromManagedStream(userId: string, appId: string): boolean { + removeViewerFromManagedStream(userId: string, packageName: string): boolean { const stream = this.userStreams.get(userId); if (!stream || stream.type !== "managed") { return false; } - stream.activeViewers.delete(appId); + stream.activeViewers.delete(packageName); stream.lastActivity = new Date(); this.logger.info( { userId, - appId, + packageName, remainingViewers: stream.activeViewers.size, }, "Removed viewer from managed stream", @@ -242,7 +242,7 @@ export class StreamStateManager { */ createUnmanagedStream( userId: string, - appId: string, + packageName: string, rtmpUrl: string, ): UnmanagedStreamState { const streamId = this.generateStreamId(); @@ -251,7 +251,7 @@ export class StreamStateManager { userId, type: "unmanaged", rtmpUrl, - requestingAppId: appId, + requestingPackageName: packageName, streamId, createdAt: new Date(), lastActivity: new Date(), @@ -264,7 +264,7 @@ export class StreamStateManager { this.logger.info( { userId, - appId, + packageName, streamId, }, "Created unmanaged stream", diff --git a/cloud/packages/cloud/src/services/websocket/websocket-app.service.ts b/cloud/packages/cloud/src/services/websocket/websocket-app.service.ts index ba0e33ab17..b4a6a48243 100644 --- a/cloud/packages/cloud/src/services/websocket/websocket-app.service.ts +++ b/cloud/packages/cloud/src/services/websocket/websocket-app.service.ts @@ -480,7 +480,7 @@ export class AppWebSocketService { const glassesAudioStopRequest = { type: CloudToGlassesMessageType.AUDIO_STOP_REQUEST, sessionId: userSession.sessionId, - appId: audioStopMsg.packageName, + packageName: audioStopMsg.packageName, timestamp: new Date(), }; diff --git a/cloud/packages/sdk/src/app/session/modules/camera.ts b/cloud/packages/sdk/src/app/session/modules/camera.ts index 121f554e56..f739bd130d 100644 --- a/cloud/packages/sdk/src/app/session/modules/camera.ts +++ b/cloud/packages/sdk/src/app/session/modules/camera.ts @@ -586,7 +586,7 @@ export class CameraModule { streamId: message.streamId, status: message.status, errorDetails: message.errorDetails, - appId: message.appId, + packageName: message.packageName, stats: message.stats, timestamp: message.timestamp || new Date(), }; diff --git a/cloud/packages/sdk/src/types/messages/cloud-to-glasses.ts b/cloud/packages/sdk/src/types/messages/cloud-to-glasses.ts index 9a367305ea..60dd7f1338 100644 --- a/cloud/packages/sdk/src/types/messages/cloud-to-glasses.ts +++ b/cloud/packages/sdk/src/types/messages/cloud-to-glasses.ts @@ -79,7 +79,7 @@ export interface PhotoRequestToGlasses extends BaseMessage { type: CloudToGlassesMessageType.PHOTO_REQUEST; userSession: Partial; requestId: string; - appId: string; + packageName: string; saveToGallery?: boolean; webhookUrl?: string; // URL where ASG should send the photo directly /** Desired capture size to guide device resolution selection */ @@ -116,7 +116,7 @@ export interface SettingsUpdate extends BaseMessage { export interface StartRtmpStream extends BaseMessage { type: CloudToGlassesMessageType.START_RTMP_STREAM; rtmpUrl: string; - appId: string; + packageName: string; streamId?: string; video?: any; // Video configuration audio?: any; // Audio configuration @@ -128,7 +128,7 @@ export interface StartRtmpStream extends BaseMessage { */ export interface StopRtmpStream extends BaseMessage { type: CloudToGlassesMessageType.STOP_RTMP_STREAM; - appId: string; + packageName: string; streamId?: string; } @@ -177,7 +177,7 @@ export interface AudioPlayRequestToGlasses extends BaseMessage { type: CloudToGlassesMessageType.AUDIO_PLAY_REQUEST; userSession: Partial; requestId: string; - appId: string; + packageName: string; audioUrl: string; // URL to audio file for download and play volume?: number; // Volume level 0.0-1.0, defaults to 1.0 stopOtherAudio?: boolean; // Whether to stop other audio playback, defaults to true @@ -189,7 +189,7 @@ export interface AudioPlayRequestToGlasses extends BaseMessage { export interface AudioStopRequestToGlasses extends BaseMessage { type: CloudToGlassesMessageType.AUDIO_STOP_REQUEST; userSession: Partial; - appId: string; + packageName: string; } /** diff --git a/cloud/packages/sdk/src/types/messages/glasses-to-cloud.ts b/cloud/packages/sdk/src/types/messages/glasses-to-cloud.ts index bb1d03c140..d410a3897e 100644 --- a/cloud/packages/sdk/src/types/messages/glasses-to-cloud.ts +++ b/cloud/packages/sdk/src/types/messages/glasses-to-cloud.ts @@ -1,9 +1,13 @@ // src/messages/glasses-to-cloud.ts -import { BaseMessage } from './base'; -import { GlassesToCloudMessageType, ControlActionTypes, EventTypes } from '../message-types'; -import { StreamType } from '../streams'; -import { PhotoRequest } from './app-to-cloud'; +import { BaseMessage } from "./base"; +import { + GlassesToCloudMessageType, + ControlActionTypes, + EventTypes, +} from "../message-types"; +import { StreamType } from "../streams"; +import { PhotoRequest } from "./app-to-cloud"; //=========================================================== // Control actions @@ -64,7 +68,7 @@ export interface OpenDashboard extends BaseMessage { export interface ButtonPress extends BaseMessage { type: GlassesToCloudMessageType.BUTTON_PRESS; buttonId: string; - pressType: 'short' | 'long'; + pressType: "short" | "long"; } /** @@ -72,7 +76,7 @@ export interface ButtonPress extends BaseMessage { */ export interface HeadPosition extends BaseMessage { type: GlassesToCloudMessageType.HEAD_POSITION; - position: 'up' | 'down'; + position: "up" | "down"; } /** @@ -80,9 +84,9 @@ export interface HeadPosition extends BaseMessage { */ export interface GlassesBatteryUpdate extends BaseMessage { type: GlassesToCloudMessageType.GLASSES_BATTERY_UPDATE; - level: number; // 0-100 + level: number; // 0-100 charging: boolean; - timeRemaining?: number; // minutes + timeRemaining?: number; // minutes } /** @@ -90,9 +94,9 @@ export interface GlassesBatteryUpdate extends BaseMessage { */ export interface PhoneBatteryUpdate extends BaseMessage { type: GlassesToCloudMessageType.PHONE_BATTERY_UPDATE; - level: number; // 0-100 + level: number; // 0-100 charging: boolean; - timeRemaining?: number; // minutes + timeRemaining?: number; // minutes } /** @@ -170,7 +174,7 @@ export interface PhoneNotification extends BaseMessage { app: string; title: string; content: string; - priority: 'low' | 'normal' | 'high'; + priority: "low" | "normal" | "high"; } /** @@ -204,15 +208,14 @@ export interface CoreStatusUpdate extends BaseMessage { details?: Record; } - // =========================================================== // Mentra Live // =========================================================== export interface PhotoResponse extends BaseMessage { type: GlassesToCloudMessageType.PHOTO_RESPONSE; - requestId: string; // Unique ID for the photo request - photoUrl: string; // URL of the uploaded photo - savedToGallery: boolean; // Whether the photo was saved to gallery + requestId: string; // Unique ID for the photo request + photoUrl: string; // URL of the uploaded photo + savedToGallery: boolean; // Whether the photo was saved to gallery } /** @@ -220,10 +223,20 @@ export interface PhotoResponse extends BaseMessage { */ export interface RtmpStreamStatus extends BaseMessage { type: GlassesToCloudMessageType.RTMP_STREAM_STATUS; - streamId?: string; // Unique identifier for the stream - status: "initializing" | "connecting" | "reconnecting" | "streaming" | "error" | "stopped" | "active" | "stopping" | "disconnected" | "timeout"; + streamId?: string; // Unique identifier for the stream + status: + | "initializing" + | "connecting" + | "reconnecting" + | "streaming" + | "error" + | "stopped" + | "active" + | "stopping" + | "disconnected" + | "timeout"; errorDetails?: string; - appId?: string; // ID of the app that requested the stream + packageName?: string; // ID of the app that requested the stream stats?: { bitrate: number; fps: number; @@ -237,8 +250,8 @@ export interface RtmpStreamStatus extends BaseMessage { */ export interface KeepAliveAck extends BaseMessage { type: GlassesToCloudMessageType.KEEP_ALIVE_ACK; - streamId: string; // ID of the stream being kept alive - ackId: string; // Acknowledgment ID that was sent by cloud + streamId: string; // ID of the stream being kept alive + ackId: string; // Acknowledgment ID that was sent by cloud } /** @@ -305,15 +318,21 @@ export function isEvent(message: GlassesToCloudMessage): boolean { } // Individual type guards -export function isConnectionInit(message: GlassesToCloudMessage): message is ConnectionInit { +export function isConnectionInit( + message: GlassesToCloudMessage, +): message is ConnectionInit { return message.type === GlassesToCloudMessageType.CONNECTION_INIT; } -export function isRequestSettings(message: GlassesToCloudMessage): message is RequestSettings { +export function isRequestSettings( + message: GlassesToCloudMessage, +): message is RequestSettings { return message.type === GlassesToCloudMessageType.REQUEST_SETTINGS; } -export function isStartApp(message: GlassesToCloudMessage): message is StartApp { +export function isStartApp( + message: GlassesToCloudMessage, +): message is StartApp { return message.type === GlassesToCloudMessageType.START_APP; } @@ -321,31 +340,45 @@ export function isStopApp(message: GlassesToCloudMessage): message is StopApp { return message.type === GlassesToCloudMessageType.STOP_APP; } -export function isButtonPress(message: GlassesToCloudMessage): message is ButtonPress { +export function isButtonPress( + message: GlassesToCloudMessage, +): message is ButtonPress { return message.type === GlassesToCloudMessageType.BUTTON_PRESS; } -export function isHeadPosition(message: GlassesToCloudMessage): message is HeadPosition { +export function isHeadPosition( + message: GlassesToCloudMessage, +): message is HeadPosition { return message.type === GlassesToCloudMessageType.HEAD_POSITION; } -export function isGlassesBatteryUpdate(message: GlassesToCloudMessage): message is GlassesBatteryUpdate { +export function isGlassesBatteryUpdate( + message: GlassesToCloudMessage, +): message is GlassesBatteryUpdate { return message.type === GlassesToCloudMessageType.GLASSES_BATTERY_UPDATE; } -export function isPhoneBatteryUpdate(message: GlassesToCloudMessage): message is PhoneBatteryUpdate { +export function isPhoneBatteryUpdate( + message: GlassesToCloudMessage, +): message is PhoneBatteryUpdate { return message.type === GlassesToCloudMessageType.PHONE_BATTERY_UPDATE; } -export function isGlassesConnectionState(message: GlassesToCloudMessage): message is GlassesConnectionState { +export function isGlassesConnectionState( + message: GlassesToCloudMessage, +): message is GlassesConnectionState { return message.type === GlassesToCloudMessageType.GLASSES_CONNECTION_STATE; } -export function isLocationUpdate(message: GlassesToCloudMessage): message is LocationUpdate { +export function isLocationUpdate( + message: GlassesToCloudMessage, +): message is LocationUpdate { return message.type === GlassesToCloudMessageType.LOCATION_UPDATE; } -export function isCalendarEvent(message: GlassesToCloudMessage): message is CalendarEvent { +export function isCalendarEvent( + message: GlassesToCloudMessage, +): message is CalendarEvent { return message.type === GlassesToCloudMessageType.CALENDAR_EVENT; } @@ -353,34 +386,52 @@ export function isVad(message: GlassesToCloudMessage): message is Vad { return message.type === GlassesToCloudMessageType.VAD; } -export function isPhoneNotification(message: GlassesToCloudMessage): message is PhoneNotification { +export function isPhoneNotification( + message: GlassesToCloudMessage, +): message is PhoneNotification { return message.type === GlassesToCloudMessageType.PHONE_NOTIFICATION; } -export function isPhoneNotificationDismissed(message: GlassesToCloudMessage): message is PhoneNotificationDismissed { - return message.type === GlassesToCloudMessageType.PHONE_NOTIFICATION_DISMISSED; +export function isPhoneNotificationDismissed( + message: GlassesToCloudMessage, +): message is PhoneNotificationDismissed { + return ( + message.type === GlassesToCloudMessageType.PHONE_NOTIFICATION_DISMISSED + ); } -export function isRtmpStreamStatus(message: GlassesToCloudMessage): message is RtmpStreamStatus { +export function isRtmpStreamStatus( + message: GlassesToCloudMessage, +): message is RtmpStreamStatus { return message.type === GlassesToCloudMessageType.RTMP_STREAM_STATUS; } -export function isPhotoResponse(message: GlassesToCloudMessage): message is PhotoResponse { +export function isPhotoResponse( + message: GlassesToCloudMessage, +): message is PhotoResponse { return message.type === GlassesToCloudMessageType.PHOTO_RESPONSE; } -export function isKeepAliveAck(message: GlassesToCloudMessage): message is KeepAliveAck { +export function isKeepAliveAck( + message: GlassesToCloudMessage, +): message is KeepAliveAck { return message.type === GlassesToCloudMessageType.KEEP_ALIVE_ACK; } -export function isPhotoTaken(message: GlassesToCloudMessage): message is PhotoTaken { +export function isPhotoTaken( + message: GlassesToCloudMessage, +): message is PhotoTaken { return message.type === GlassesToCloudMessageType.PHOTO_TAKEN; } -export function isAudioPlayResponse(message: GlassesToCloudMessage): message is AudioPlayResponse { +export function isAudioPlayResponse( + message: GlassesToCloudMessage, +): message is AudioPlayResponse { return message.type === GlassesToCloudMessageType.AUDIO_PLAY_RESPONSE; } -export function isLocalTranscription(message: GlassesToCloudMessage): message is LocalTranscription { +export function isLocalTranscription( + message: GlassesToCloudMessage, +): message is LocalTranscription { return message.type === GlassesToCloudMessageType.LOCAL_TRANSCRIPTION; -} \ No newline at end of file +} diff --git a/cloud/websites/debugger/src/components/DebuggerEvents.tsx b/cloud/websites/debugger/src/components/DebuggerEvents.tsx index ccf4bb99ab..2775176462 100644 --- a/cloud/websites/debugger/src/components/DebuggerEvents.tsx +++ b/cloud/websites/debugger/src/components/DebuggerEvents.tsx @@ -86,7 +86,7 @@ type DebuggerEvent = | { type: 'SESSION_UPDATE'; sessionId: string; data: Partial } | { type: 'SESSION_DISCONNECTED'; sessionId: string; timestamp: string } | { type: 'SESSION_CONNECTED'; sessionId: string; timestamp: string } - | { type: 'APP_STATE_CHANGE'; sessionId: string; appId: string; state: any } + | { type: 'APP_STATE_CHANGE'; sessionId: string; packageName: string; state: any } | { type: 'DISPLAY_UPDATE'; sessionId: string; display: any } | { type: 'TRANSCRIPTION_UPDATE'; sessionId: string; transcript: any } | { type: 'SYSTEM_STATS_UPDATE'; stats: SystemStats }; diff --git a/mobile/.maestro/config.yaml b/mobile/.maestro/config.yaml index e43a0e96f5..5f1dc9372f 100644 --- a/mobile/.maestro/config.yaml +++ b/mobile/.maestro/config.yaml @@ -1,7 +1,7 @@ # Maestro Configuration for MentraOS # Global test configuration -appId: com.mentra.mentra +packageName: com.mentra.mentra name: "MentraOS E2E Tests" # Test tags for organizing test runs diff --git a/mobile/.maestro/flows/01-auth-flow-simple.yaml b/mobile/.maestro/flows/01-auth-flow-simple.yaml index 4166e8d9b1..11bf6da1e9 100644 --- a/mobile/.maestro/flows/01-auth-flow-simple.yaml +++ b/mobile/.maestro/flows/01-auth-flow-simple.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: Authentication Flow (Simplified) # Description: Tests login flow - assumes app starts logged out diff --git a/mobile/.maestro/flows/01-auth-flow.yaml b/mobile/.maestro/flows/01-auth-flow.yaml index 622a6f6722..28c9b9aa38 100644 --- a/mobile/.maestro/flows/01-auth-flow.yaml +++ b/mobile/.maestro/flows/01-auth-flow.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: Authentication Flow # Description: Tests login flow for existing users diff --git a/mobile/.maestro/flows/02-tab-navigation.yaml b/mobile/.maestro/flows/02-tab-navigation.yaml index f258e67778..8cfaf7e099 100644 --- a/mobile/.maestro/flows/02-tab-navigation.yaml +++ b/mobile/.maestro/flows/02-tab-navigation.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: Tab Navigation # Description: Tests bottom tab navigation between all main screens diff --git a/mobile/.maestro/flows/03-app-store.yaml b/mobile/.maestro/flows/03-app-store.yaml index e14c0f036b..0d9b1a7f81 100644 --- a/mobile/.maestro/flows/03-app-store.yaml +++ b/mobile/.maestro/flows/03-app-store.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: App Store Browse and Install # Description: Tests browsing the MentraOS Store and installing an app diff --git a/mobile/.maestro/flows/04-simulated-glasses-pairing.yaml b/mobile/.maestro/flows/04-simulated-glasses-pairing.yaml index 29dd4b0ec9..2b6059a62b 100644 --- a/mobile/.maestro/flows/04-simulated-glasses-pairing.yaml +++ b/mobile/.maestro/flows/04-simulated-glasses-pairing.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: Simulated Glasses Pairing # Description: Tests pairing flow with simulated glasses (requires BT permission but doesn't use actual BT) diff --git a/mobile/.maestro/flows/05-no-internet-connection.yaml b/mobile/.maestro/flows/05-no-internet-connection.yaml index efdfcd05ca..7a5435f216 100644 --- a/mobile/.maestro/flows/05-no-internet-connection.yaml +++ b/mobile/.maestro/flows/05-no-internet-connection.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: No Internet Connection # Description: Tests app behavior when launched without internet connection diff --git a/mobile/.maestro/flows/06-launch-app-simulated-glasses.yaml b/mobile/.maestro/flows/06-launch-app-simulated-glasses.yaml index 66732331d9..f2c412708d 100644 --- a/mobile/.maestro/flows/06-launch-app-simulated-glasses.yaml +++ b/mobile/.maestro/flows/06-launch-app-simulated-glasses.yaml @@ -1,4 +1,4 @@ -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- # Test: Launch App with Simulated Glasses # Description: Tests launching Mira app on simulated glasses and verifying "Starting App" status diff --git a/mobile/.maestro/flows/FavoritePodcast.yaml b/mobile/.maestro/flows/FavoritePodcast.yaml index df29fa013c..a37c22b740 100644 --- a/mobile/.maestro/flows/FavoritePodcast.yaml +++ b/mobile/.maestro/flows/FavoritePodcast.yaml @@ -1,6 +1,6 @@ # flow: run the login flow and then navigate to the demo podcast list screen, favorite a podcast, and then switch the list to only be favorites. -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} env: FAVORITES_TEXT: "Switch on to only show favorites" onFlowStart: diff --git a/mobile/.maestro/flows/Login.yaml b/mobile/.maestro/flows/Login.yaml index 74a3ab061a..c4e6310cd0 100644 --- a/mobile/.maestro/flows/Login.yaml +++ b/mobile/.maestro/flows/Login.yaml @@ -3,7 +3,7 @@ # Open up our app and use the default credentials to login # and navigate to the demo screen -appId: ${MAESTRO_APP_ID} # the app id of the app we want to test +packageName: ${MAESTRO_APP_ID} # the app id of the app we want to test # You can find the appId of an Ignite app in the `app.json` file # as the "package" under the "android" section and "bundleIdentifier" under the "ios" section onFlowStart: diff --git a/mobile/.maestro/shared/_Login.yaml b/mobile/.maestro/shared/_Login.yaml index c7dd7daf92..4ec2624eb0 100644 --- a/mobile/.maestro/shared/_Login.yaml +++ b/mobile/.maestro/shared/_Login.yaml @@ -1,6 +1,6 @@ #flow: Shared _Login #intent: shared login flow for any flow that needs to start with a log in. -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- - assertVisible: "Log In" - tapOn: diff --git a/mobile/.maestro/shared/_OpenAppClearStateAndKeychain.yaml b/mobile/.maestro/shared/_OpenAppClearStateAndKeychain.yaml index c2f7ab6884..7e0339070d 100644 --- a/mobile/.maestro/shared/_OpenAppClearStateAndKeychain.yaml +++ b/mobile/.maestro/shared/_OpenAppClearStateAndKeychain.yaml @@ -1,5 +1,5 @@ --- -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- - launchApp: clearState: true diff --git a/mobile/.maestro/shared/_OpenWithDevClient.yaml b/mobile/.maestro/shared/_OpenWithDevClient.yaml index bacb786367..072e2260d6 100644 --- a/mobile/.maestro/shared/_OpenWithDevClient.yaml +++ b/mobile/.maestro/shared/_OpenWithDevClient.yaml @@ -1,7 +1,7 @@ # Common subflow for opening the app with a clear state, clear keychain, and getting past the Expo Dev Client prompts. # Use this for local development flows. --- -appId: ${MAESTRO_APP_ID} +packageName: ${MAESTRO_APP_ID} --- - runFlow: file: "./_OpenAppClearStateAndKeychain.yaml" diff --git a/mobile/ios/BleManager/AOSManager.swift b/mobile/ios/BleManager/AOSManager.swift index c9920ae7bd..449b19d7ea 100644 --- a/mobile/ios/BleManager/AOSManager.swift +++ b/mobile/ios/BleManager/AOSManager.swift @@ -337,9 +337,9 @@ struct ViewState { guard let self = self else { return } self.serverComms.sendPhotoResponse(requestId: requestId, photoUrl: photoUrl) } - liveManager!.onVideoStreamResponse = { [weak self] (appId: String, streamUrl: String) in + liveManager!.onVideoStreamResponse = { [weak self] (packageName: String, streamUrl: String) in guard let self = self else { return } - self.serverComms.sendVideoStreamResponse(appId: appId, streamUrl: streamUrl) + self.serverComms.sendVideoStreamResponse(packageName: packageName, streamUrl: streamUrl) } } @@ -650,9 +650,9 @@ struct ViewState { liveManager?.sendJson(message) } - func onPhotoRequest(_ requestId: String, _ appId: String, _ webhookUrl: String, _ size: String) { - CoreCommsService.log("AOS: onPhotoRequest: \(requestId), \(appId), \(webhookUrl), size=\(size)") - liveManager?.requestPhoto(requestId, appId: appId, webhookUrl: webhookUrl.isEmpty ? nil : webhookUrl, size: size) + func onPhotoRequest(_ requestId: String, _ packageName: String, _ webhookUrl: String, _ size: String) { + CoreCommsService.log("AOS: onPhotoRequest: \(requestId), \(packageName), \(webhookUrl), size=\(size)") + liveManager?.requestPhoto(requestId, packageName: packageName, webhookUrl: webhookUrl.isEmpty ? nil : webhookUrl, size: size) } func onRtmpStreamStartRequest(_ message: [String: Any]) { diff --git a/mobile/ios/BleManager/MentraLiveManager.swift b/mobile/ios/BleManager/MentraLiveManager.swift index 1a31f7b0cc..7e30856b84 100644 --- a/mobile/ios/BleManager/MentraLiveManager.swift +++ b/mobile/ios/BleManager/MentraLiveManager.swift @@ -926,13 +926,13 @@ typealias JSONObject = [String: Any] sendJson(json, wakeUp: true) } - @objc func requestPhoto(_ requestId: String, appId: String, webhookUrl: String?, size: String?) { - CoreCommsService.log("Requesting photo: \(requestId) for app: \(appId)") + @objc func requestPhoto(_ requestId: String, packageName: String, webhookUrl: String?, size: String?) { + CoreCommsService.log("Requesting photo: \(requestId) for app: \(packageName)") var json: [String: Any] = [ "type": "take_photo", "requestId": requestId, - "appId": appId, + "packageName": packageName, ] // Always generate BLE ID for potential fallback diff --git a/mobile/ios/BleManager/ServerComms.swift b/mobile/ios/BleManager/ServerComms.swift index 596fc2490d..aef70ab5eb 100644 --- a/mobile/ios/BleManager/ServerComms.swift +++ b/mobile/ios/BleManager/ServerComms.swift @@ -20,7 +20,7 @@ protocol ServerCommsCallback { func onAppStarted(_ packageName: String) func onAppStopped(_ packageName: String) func onJsonMessage(_ message: [String: Any]) - func onPhotoRequest(_ requestId: String, _ appId: String, _ webhookUrl: String, _ size: String) + func onPhotoRequest(_ requestId: String, _ packageName: String, _ webhookUrl: String, _ size: String) func onRtmpStreamStartRequest(_ message: [String: Any]) func onRtmpStreamStop() func onRtmpStreamKeepAlive(_ message: [String: Any]) @@ -436,11 +436,11 @@ class ServerComms { } } - func sendVideoStreamResponse(appId: String, streamUrl: String) { + func sendVideoStreamResponse(packageName: String, streamUrl: String) { do { let event: [String: Any] = [ "type": "video_stream_response", - "appId": appId, + "packageName": packageName, "streamUrl": streamUrl, "timestamp": Int(Date().timeIntervalSince1970 * 1000), ] @@ -603,14 +603,14 @@ class ServerComms { case "photo_request": let requestId = msg["requestId"] as? String ?? "" - let appId = msg["appId"] as? String ?? "" + let packageName = msg["packageName"] as? String ?? "" let webhookUrl = msg["webhookUrl"] as? String ?? "" let size = (msg["size"] as? String) ?? "medium" - CoreCommsService.log("Received photo_request, requestId: \(requestId), appId: \(appId), webhookUrl: \(webhookUrl), size: \(size)") - if !requestId.isEmpty, !appId.isEmpty { - serverCommsCallback?.onPhotoRequest(requestId, appId, webhookUrl, size) + CoreCommsService.log("Received photo_request, requestId: \(requestId), packageName: \(packageName), webhookUrl: \(webhookUrl), size: \(size)") + if !requestId.isEmpty, !packageName.isEmpty { + serverCommsCallback?.onPhotoRequest(requestId, packageName, webhookUrl, size) } else { - CoreCommsService.log("Invalid photo request: missing requestId or appId") + CoreCommsService.log("Invalid photo request: missing requestId or packageName") } case "start_rtmp_stream": diff --git a/mobile/src/app/mirror/gallery.tsx b/mobile/src/app/mirror/gallery.tsx index 1035bcb089..785d9baf93 100644 --- a/mobile/src/app/mirror/gallery.tsx +++ b/mobile/src/app/mirror/gallery.tsx @@ -36,7 +36,7 @@ interface GalleryPhoto { id: string photoUrl: string uploadDate: string - appId: string + packageName: string userId: string } @@ -51,7 +51,7 @@ interface MediaItem { formattedDate?: string // Pre-formatted date for display formattedTime?: string // Pre-formatted time for display metadata: { - appId?: string // Source app for cloud photos + packageName?: string // Source app for cloud photos fileName?: string // For local recordings // Other metadata as needed } @@ -156,7 +156,7 @@ const GlassesRecordingsGallery: React.FC = ({isDa formattedDate, formattedTime, metadata: { - appId: photo.appId, + packageName: photo.packageName, }, } } @@ -513,7 +513,7 @@ const GlassesRecordingsGallery: React.FC = ({isDa id: item.id, photoUrl: item.contentUrl, uploadDate: new Date(item.timestamp).toISOString(), - appId: item.metadata.appId || "Unknown", + packageName: item.metadata.packageName || "Unknown", }} isDarkTheme={isDarkTheme} onViewPhoto={viewPhoto} @@ -546,7 +546,7 @@ const GlassesRecordingsGallery: React.FC = ({isDa {new Date(selectedPhoto.uploadDate).toLocaleString()} - From app: {selectedPhoto.appId} + From app: {selectedPhoto.packageName} diff --git a/mobile/src/components/misc/PhotoItem.tsx b/mobile/src/components/misc/PhotoItem.tsx index 4832192a52..5d496ad7eb 100644 --- a/mobile/src/components/misc/PhotoItem.tsx +++ b/mobile/src/components/misc/PhotoItem.tsx @@ -6,7 +6,7 @@ interface PhotoItemProps { photo: { photoUrl: string uploadDate: string - appId: string + packageName: string id: string } isDarkTheme: boolean @@ -55,7 +55,7 @@ const PhotoItem: React.FC = ({ {/* App source */} - From: {photo.appId} + From: {photo.packageName} {/* Action Buttons */} diff --git a/mobile/src/components/misc/types.ts b/mobile/src/components/misc/types.ts index 97e97e0841..987aafaf3b 100644 --- a/mobile/src/components/misc/types.ts +++ b/mobile/src/components/misc/types.ts @@ -21,7 +21,7 @@ export type RootStackParamList = { GlassesMirrorFullscreen: undefined GlassesRecordingsGallery: undefined VideoPlayerScreen: {filePath: string; fileName?: string} - Reviews: {appId: string; appName: string} + Reviews: {packageName: string; appName: string} ConnectingToPuck: undefined PhoneNotificationSettings: undefined PrivacySettingsScreen: undefined