From 565cac50a09583970bbd3a1984de75896f2cf1fd Mon Sep 17 00:00:00 2001 From: Vincent Tran Date: Fri, 20 Sep 2024 14:04:24 -0700 Subject: [PATCH 1/2] feat(liveness): Add support for configuring the back camera for the no light challenge --- .../ui/liveness/camera/LivenessCoordinator.kt | 37 +++++++++++--- .../ui/liveness/ml/FaceDetector.kt | 1 - .../ui/liveness/state/LivenessState.kt | 1 + .../ui/liveness/ui/FaceLivenessDetector.kt | 49 +++++++++++++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt index a7ca5cf..a20faf1 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt @@ -47,6 +47,8 @@ import com.amplifyframework.ui.liveness.model.FaceLivenessDetectionException import com.amplifyframework.ui.liveness.model.LivenessCheckState import com.amplifyframework.ui.liveness.state.AttemptCounter import com.amplifyframework.ui.liveness.state.LivenessState +import com.amplifyframework.ui.liveness.ui.Camera +import com.amplifyframework.ui.liveness.ui.ChallengeOptions import com.amplifyframework.ui.liveness.util.WebSocketCloseCode import java.util.Date import java.util.concurrent.Executors @@ -68,11 +70,12 @@ internal typealias OnFreshnessColorDisplayed = ( @SuppressLint("UnsafeOptInUsageError") internal class LivenessCoordinator( val context: Context, - lifecycleOwner: LifecycleOwner, + private val lifecycleOwner: LifecycleOwner, private val sessionId: String, private val region: String, private val credentialsProvider: AWSCredentialsProvider?, private val disableStartView: Boolean, + private val challengeOptions: ChallengeOptions, private val onChallengeComplete: OnChallengeComplete, val onChallengeFailed: Consumer ) { @@ -142,6 +145,14 @@ internal class LivenessCoordinator( init { startLivenessSession() + if (challengeOptions.hasOneCameraConfigured()) { + launchCamera(challengeOptions.faceMovementAndLight.camera) + } else { + livenessState.loadingCameraPreview = true + } + } + + private fun launchCamera(camera: Camera) { MainScope().launch { delay(5_000) if (!previewTextureView.hasReceivedUpdate) { @@ -156,20 +167,28 @@ internal class LivenessCoordinator( getCameraProvider(context).apply { if (lifecycleOwner.lifecycle.currentState != Lifecycle.State.DESTROYED) { unbindAll() - if (this.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA)) { + + val (chosenCamera, orientation) = when (camera) { + Camera.Front -> Pair(CameraSelector.DEFAULT_FRONT_CAMERA, "front") + Camera.Back -> Pair(CameraSelector.DEFAULT_BACK_CAMERA, "back") + } + + if (this.hasCamera(chosenCamera)) { bindToLifecycle( lifecycleOwner, - CameraSelector.DEFAULT_FRONT_CAMERA, + chosenCamera, preview, analysis ) } else { + livenessState.loadingCameraPreview = false val faceLivenessException = FaceLivenessDetectionException( - "A front facing camera is required but no front facing camera detected.", - "Enable a front facing camera." + "A $orientation facing camera is required but no $orientation facing camera detected.", + "Enable a $orientation facing camera." ) processSessionError(faceLivenessException, true) } + livenessState.loadingCameraPreview = false } } } @@ -200,7 +219,13 @@ internal class LivenessCoordinator( faceLivenessSessionInformation, faceLivenessSessionOptions, BuildConfig.LIVENESS_VERSION_NAME, - { livenessState.onLivenessSessionReady(it) }, + { + livenessState.onLivenessSessionReady(it) + if (!challengeOptions.hasOneCameraConfigured()) { + val foundChallenge = challengeOptions.getOptions(it.challengeType) + launchCamera(foundChallenge.camera) + } + }, { disconnectEventReceived = true onChallengeComplete() diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/ml/FaceDetector.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/ml/FaceDetector.kt index 68cf13f..0f4fc8c 100755 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/ml/FaceDetector.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/ml/FaceDetector.kt @@ -18,7 +18,6 @@ package com.amplifyframework.ui.liveness.ml import android.content.Context import android.graphics.RectF import androidx.annotation.VisibleForTesting -import com.amplifyframework.predictions.aws.models.FaceTargetChallenge import com.amplifyframework.predictions.aws.models.FaceTargetMatchingParameters import com.amplifyframework.ui.liveness.R import com.amplifyframework.ui.liveness.camera.LivenessCoordinator.Companion.TARGET_HEIGHT diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/state/LivenessState.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/state/LivenessState.kt index b835f42..2ae36ca 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/state/LivenessState.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/state/LivenessState.kt @@ -60,6 +60,7 @@ internal data class LivenessState( var initialLocalFaceFound by mutableStateOf(false) var showingStartView by mutableStateOf(!disableStartView) + var loadingCameraPreview by mutableStateOf(false) private var initialStreamFace: InitialStreamFace? = null @VisibleForTesting diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt index aa574e5..046f3d3 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt @@ -27,6 +27,7 @@ import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width import androidx.compose.material3.Button +import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface @@ -72,6 +73,7 @@ import kotlinx.coroutines.launch * @param region AWS region to stream the video to. Current supported regions are listed in [add link here] * @param credentialsProvider to provide custom CredentialsProvider for authentication. Default uses initialized Amplify.Auth CredentialsProvider * @param disableStartView to bypass warmup screen. + * @param challengeOptions is the list of ChallengeOptions that are to be overridden from the default configuration * @param onComplete callback notifying a completed challenge * @param onError callback containing exception for cause */ @@ -81,6 +83,7 @@ fun FaceLivenessDetector( region: String, credentialsProvider: AWSCredentialsProvider? = null, disableStartView: Boolean = false, + challengeOptions: ChallengeOptions = ChallengeOptions(), onComplete: Action, onError: Consumer ) { @@ -124,6 +127,7 @@ fun FaceLivenessDetector( region, credentialsProvider = credentialsProvider, disableStartView, + challengeOptions = challengeOptions, onChallengeComplete = { scope.launch { // if we are already finished, we already provided a result in complete or failed @@ -156,6 +160,7 @@ internal fun ChallengeView( region: String, credentialsProvider: AWSCredentialsProvider?, disableStartView: Boolean, + challengeOptions: ChallengeOptions, onChallengeComplete: OnChallengeComplete, onChallengeFailed: Consumer ) { @@ -176,6 +181,7 @@ internal fun ChallengeView( region, credentialsProvider, disableStartView, + challengeOptions, onChallengeComplete = { currentOnChallengeComplete() }, onChallengeFailed = { currentOnChallengeFailed.accept(it) } ) @@ -232,6 +238,15 @@ internal fun ChallengeView( if (livenessState.showingStartView) { + if (livenessState.loadingCameraPreview) { + CircularProgressIndicator( + color = MaterialTheme.colorScheme.primary, + modifier = Modifier + .align(Alignment.Center), + strokeWidth = 2.dp, + ) + } + FaceGuide( modifier = Modifier .fillMaxSize() @@ -402,6 +417,40 @@ internal fun ChallengeView( } } +data class ChallengeOptions( + val faceMovementAndLight: LivenessChallenge.FaceMovementAndLight = LivenessChallenge.FaceMovementAndLight, + val faceMovement: LivenessChallenge.FaceMovement = LivenessChallenge.FaceMovement() +) { + fun getOptions(challengeType: FaceLivenessChallengeType): LivenessChallenge = + when (challengeType) { + FaceLivenessChallengeType.FaceMovementAndLightChallenge -> faceMovementAndLight + FaceLivenessChallengeType.FaceMovementChallenge -> faceMovement + } + + /** + * @return true if all of the challenge options are configured to use the same camera configuration + */ + fun hasOneCameraConfigured(): Boolean = + listOf( + faceMovementAndLight, + faceMovement + ).all { it.camera == faceMovementAndLight.camera } +} + +sealed class LivenessChallenge( + val camera: Camera = Camera.Front +) { + class FaceMovement(camera: Camera = Camera.Front) : LivenessChallenge( + camera = camera + ) + object FaceMovementAndLight : LivenessChallenge() +} + +sealed class Camera { + object Front : Camera() + object Back : Camera() +} + private fun FaceLivenessSession?.isFaceMovementAndLightChallenge(): Boolean = this?.challengeType == FaceLivenessChallengeType.FaceMovementAndLightChallenge From fbd15b2bebeb735a8fca47582297e88cde1d3f28 Mon Sep 17 00:00:00 2001 From: Vincent Tran Date: Wed, 26 Mar 2025 14:01:42 -0700 Subject: [PATCH 2/2] feat(liveness): Throw a new exception when the app is paused and stop liveness --- .../ui/liveness/camera/LivenessCoordinator.kt | 1 + .../model/FaceLivenessDetectionException.kt | 6 ++++++ .../ui/liveness/ui/FaceLivenessDetector.kt | 19 +++++++++++++++++++ .../ui/liveness/util/WebSocketCloseCode.kt | 1 + .../ui/sample/liveness/ui/LivenessScreen.kt | 5 ++++- 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt index a20faf1..22d5fd3 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt @@ -269,6 +269,7 @@ internal class LivenessCoordinator( val webSocketCloseCode = when (faceLivenessException) { is FaceLivenessDetectionException.UserCancelledException -> WebSocketCloseCode.CANCELED is FaceLivenessDetectionException.FaceInOvalMatchExceededTimeLimitException -> WebSocketCloseCode.TIMEOUT + is FaceLivenessDetectionException.LostFocusException -> WebSocketCloseCode.LOST_FOCUS else -> WebSocketCloseCode.RUNTIME_ERROR } livenessState.onError(stopLivenessSession, webSocketCloseCode) diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException.kt index c440aac..cff65c3 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException.kt @@ -58,6 +58,12 @@ open class FaceLivenessDetectionException( throwable: Throwable? = null ) : FaceLivenessDetectionException(message, recoverySuggestion, throwable) + class LostFocusException( + message: String = "Face liveness check was cancelled because the app lost focus.", + recoverySuggestion: String = "Retry the face liveness check.", + throwable: Throwable? = null + ) : FaceLivenessDetectionException(message, recoverySuggestion, throwable) + /** * This is not an error we have determined to publicly expose. * The error will come to the customer in onError, but only instance checked as FaceLivenessDetectionException. diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt index 046f3d3..0e277c4 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt @@ -52,6 +52,8 @@ import androidx.compose.ui.platform.LocalLifecycleOwner import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp import androidx.compose.ui.viewinterop.AndroidView +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleEventObserver import com.amplifyframework.auth.AWSCredentials import com.amplifyframework.auth.AWSCredentialsProvider import com.amplifyframework.core.Action @@ -194,6 +196,23 @@ internal fun ChallengeView( ) } + + val observer = LifecycleEventObserver { _, event -> + // If the app ever gets paused while the liveness check is in progress, + // send a cancelled event to the backend and stop the session. + if (event == Lifecycle.Event.ON_PAUSE) { + val isActionable = coordinator?.livenessState?.livenessCheckState?.isActionable + if (isActionable != null && isActionable) { + coordinator?.processSessionError( + FaceLivenessDetectionException.LostFocusException(), + true + ) + } + } + } + + lifecycleOwner.lifecycle.addObserver(observer) + onDispose { coordinator?.destroy(context) } diff --git a/liveness/src/main/java/com/amplifyframework/ui/liveness/util/WebSocketCloseCode.kt b/liveness/src/main/java/com/amplifyframework/ui/liveness/util/WebSocketCloseCode.kt index a795a8b..09bf4f7 100644 --- a/liveness/src/main/java/com/amplifyframework/ui/liveness/util/WebSocketCloseCode.kt +++ b/liveness/src/main/java/com/amplifyframework/ui/liveness/util/WebSocketCloseCode.kt @@ -3,6 +3,7 @@ package com.amplifyframework.ui.liveness.util internal enum class WebSocketCloseCode(val code: Int) { TIMEOUT(4001), CANCELED(4003), + LOST_FOCUS(4004), RUNTIME_ERROR(4005), DISPOSED(4008) } diff --git a/samples/liveness/app/src/main/java/com/amplifyframework/ui/sample/liveness/ui/LivenessScreen.kt b/samples/liveness/app/src/main/java/com/amplifyframework/ui/sample/liveness/ui/LivenessScreen.kt index 175bf5b..3822efc 100644 --- a/samples/liveness/app/src/main/java/com/amplifyframework/ui/sample/liveness/ui/LivenessScreen.kt +++ b/samples/liveness/app/src/main/java/com/amplifyframework/ui/sample/liveness/ui/LivenessScreen.kt @@ -29,7 +29,10 @@ fun LivenessScreen( onChallengeComplete() }, onError = { - if (it is FaceLivenessDetectionException.UserCancelledException) { + if ( + it is FaceLivenessDetectionException.UserCancelledException || + it is FaceLivenessDetectionException.LostFocusException + ) { onBack() } else { viewModel.reportErrorResult(it)