Multiplatform ImageViewer desktop camera support

experimental/examples/imageviewer/.run/desktopApp.run.xml

@@ -4,7 +4,7 @@
       <option name="executionName" />
       <option name="externalProjectPath" value="$PROJECT_DIR$/desktopApp" />
       <option name="externalSystemIdString" value="GRADLE" />
-      <option name="scriptParameters" value="" />
+      <option name="scriptParameters" value="--stacktrace" />
       <option name="taskDescriptions">
         <list />
       </option>

experimental/examples/imageviewer/build.gradle.kts

@@ -15,5 +15,6 @@ allprojects {
         mavenCentral()
         maven("https://maven.pkg.jetbrains.space/public/p/compose/dev")
         mavenLocal()
+        maven("https://jitpack.io")
     }
 }

experimental/examples/imageviewer/desktopApp/build.gradle.kts

@@ -30,7 +30,14 @@ compose.desktop {
 
             val iconsRoot = project.file("../common/src/desktopMain/resources/images")
             macOS {
-                iconFile.set(iconsRoot.resolve("icon-mac.icns"))
+                //iconFile.set(iconsRoot.resolve("icon-mac.icns")) does not exist
+                runtimeEntitlementsFile.set(project.file("runtime-entitlements.plist"))
+                infoPlist {
+                    extraKeysRawXml = """
+                        <key>NSCameraUsageDescription</key>
+                        <string>This app uses camera for capturing photos</string>
+                    """.trimIndent()
+                }
             }
             windows {
                 iconFile.set(iconsRoot.resolve("icon-windows.ico"))

experimental/examples/imageviewer/desktopApp/runtime-entitlements.plist

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>com.apple.security.app-sandbox</key>
+    <true/>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
+    <true/>
+    <key>com.apple.security.cs.debugger</key>
+    <true/>
+    <key>com.apple.security.device.audio-input</key>
+    <true/>
+    <key>com.apple.security.device.camera</key>
+    <true/>
+</dict>
+</plist>

experimental/examples/imageviewer/shared/build.gradle.kts

@@ -65,6 +65,10 @@ kotlin {
                 implementation(compose.desktop.common)
                 implementation("org.jetbrains.kotlinx:kotlinx-coroutines-swing:1.6.4")
                 implementation("io.ktor:ktor-client-cio:$ktorVersion")
+
+                implementation("com.github.sarxos:webcam-capture:0.3.12")
+                implementation("com.github.sarxos:webcam-capture-driver-gstreamer:0.3.12") // for Linux
+                implementation("com.github.eduramiba:webcam-capture-driver-native:-SNAPSHOT") // for MacOS and Windows
             }
         }
     }

experimental/examples/imageviewer/shared/src/desktopMain/kotlin/example/imageviewer/utils/Webcam.kt

@@ -0,0 +1,226 @@
+package example.imageviewer.utils
+
+import androidx.compose.foundation.*
+import androidx.compose.foundation.layout.*
+import androidx.compose.material.*
+import androidx.compose.runtime.*
+import androidx.compose.ui.graphics.*
+import com.github.eduramiba.webcamcapture.drivers.NativeDriver
+import com.github.sarxos.webcam.*
+import com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import kotlinx.coroutines.yield
+import org.jetbrains.skia.Bitmap
+import org.jetbrains.skia.ColorAlphaType
+import org.jetbrains.skia.ImageInfo
+import java.awt.Dimension
+import java.awt.image.*
+import kotlin.math.max
+import kotlin.time.ExperimentalTime
+import kotlin.time.measureTimedValue
+
+private var driverInitialized = false
+
+@Composable
+internal fun rememberWebcamListState(): WebcamListState {
+    val webcamListState = remember { WebcamListState() }
+    webcamListState.setup()
+
+    return webcamListState
+}
+
+@Stable
+internal class WebcamListState {
+    private val webcamsMutable = mutableStateListOf<Webcam>()
+    val webcams: List<Webcam> = webcamsMutable
+
+    var isLoading by mutableStateOf(true)
+        private set
+    var defaultWebcam: Webcam? by mutableStateOf(null) // null when there is no camera.
+        private set
+
+    @Composable
+    fun setup() {
+        LaunchedEffect(Unit) {
+            isLoading = true
+            withContext(Dispatchers.IO) {
+                // check if running on ARM/MacOS and apply the native driver.
+                if(driverInitialized.not()) {
+                    val os = System.getProperty("os.name")
+                    val driver = if(os.contains("linux", ignoreCase = true)) {
+                        GStreamerDriver()
+                    } else {
+                        // mac and windows
+                        NativeDriver()
+                    }
+
+                    Webcam.setDriver(driver)
+                    driverInitialized = true
+                }
+
+                Webcam.getWebcams().forEach { webcam ->
+                    webcamsMutable.removeIf { it.name == webcam.name }
+                    webcamsMutable.add(webcam)
+                }
+
+                defaultWebcam = Webcam.getDefault()
+            }
+            isLoading = false
+        }
+
+        DisposableEffect(Unit) {
+            val listener = object : WebcamDiscoveryListener {
+                override fun webcamFound(event: WebcamDiscoveryEvent) {
+                    webcamsMutable.removeIf { it.name == event.webcam.name }
+                    webcamsMutable.add(event.webcam)
+                }
+
+                override fun webcamGone(event: WebcamDiscoveryEvent) {
+                    webcamsMutable.removeIf { it.name == event.webcam.name }
+                }
+            }
+            Webcam.addDiscoveryListener(listener)
+
+            onDispose {
+                Webcam.removeDiscoveryListener(listener)
+            }
+        }
+    }
+}
+
+@Composable
+internal fun rememberWebcamState(webcam: Webcam): WebcamState {
+    val state = remember {
+        webcam.viewSize = Dimension(1280, 720)
+        WebcamState(webcam)
+    }
+    state.setup()
+
+    return state
+}
+
+@Stable
+internal class WebcamState(webcam: Webcam) {
+    var webcam by mutableStateOf(webcam)
+    var resolutions by mutableStateOf(webcam.viewSizes.toList())
+        private set
+
+    var currentWebcamResolution by mutableStateOf(webcam.viewSize)
+
+    var lastFrame by mutableStateOf<ImageBitmap?>(null)
+        private set
+
+    var fpsLimitation: Int? by mutableStateOf(30) // if null, it will try the most FPS the computer can take.
+
+    var timeSpentPerFrame: Long by mutableStateOf(0L)
+        private set
+
+    @OptIn(ExperimentalTime::class)
+    @Composable
+    fun setup() {
+        val coroutineScope = rememberCoroutineScope()
+        DisposableEffect(webcam) {
+            coroutineScope.launch(Dispatchers.IO) {
+                webcam.lock.disable()
+                webcam.open(true)
+            }
+            onDispose {
+                webcam.close()
+            }
+        }
+
+        LaunchedEffect(webcam) {
+            resolutions = webcam.viewSizes.toList()
+        }
+
+        LaunchedEffect(currentWebcamResolution) {
+            requiredClose(webcam) {
+                viewSize = currentWebcamResolution
+            }
+        }
+
+        LaunchedEffect(webcam) {
+            withContext(Dispatchers.IO) {
+                while (true) {
+                    val (imageBitmap, measure) = measureTimedValue {
+                        webcam.imageBitmap()
+                    }
+                    if (imageBitmap != null) {
+                        lastFrame = imageBitmap
+                        timeSpentPerFrame = measure.inWholeMilliseconds
+
+                        if(fpsLimitation != null) {
+                            delay(max(fpsLimitation!! - timeSpentPerFrame, 1))
+                        }
+                    } else {
+                        delay(50) // delay until pickup camera again
+                    }
+                    yield()
+                }
+            }
+        }
+    }
+}
+
+private fun requiredClose(webcam: Webcam, apply: Webcam.() -> Unit) {
+    val listener = object : WebcamListener {
+        override fun webcamOpen(we: WebcamEvent) {
+            we.source.removeWebcamListener(this)
+        }
+
+        override fun webcamClosed(we: WebcamEvent) {
+            we.source.apply()
+            we.source.open()
+        }
+
+        override fun webcamDisposed(we: WebcamEvent?) {}
+
+        override fun webcamImageObtained(we: WebcamEvent?) {}
+
+    }
+
+    webcam.addWebcamListener(listener)
+    webcam.close()
+}
+
+private fun Webcam.imageBitmap(): ImageBitmap? {
+    val buffer = imageBytes ?: return null
+    val arrayBuffer = ByteArray(buffer.capacity())
+
+
+    val width = viewSize.width
+    val height = viewSize.height
+
+    val bytesPerPixel = 4
+    val pixels = ByteArray(width * height * bytesPerPixel)
+
+    buffer.mark()
+    buffer.position(0)
+    buffer.get(arrayBuffer, 0, buffer.capacity())
+    buffer.reset()
+
+    // d15f35 -> 3558c7ff
+    // to not go through BufferedImage.toComposeImageBitmap
+    // instead we map directly de sRGB from Webcam Image Buffer
+    // this is way faster, probably there is faster approach.
+    var k = 0
+    for (i in 0 until buffer.capacity() step 3) {
+        val r = arrayBuffer.get(i)
+        val g = arrayBuffer.get(i + 1)
+        val b = arrayBuffer.get(i + 2)
+        pixels[k++] = b
+        pixels[k++] = g
+        pixels[k++] = r
+        pixels[k++] = 0xff.toByte()
+    }
+
+    val bitmap = Bitmap()
+
+    bitmap.allocPixels(ImageInfo.makeS32(width, height, ColorAlphaType.UNPREMUL))
+    bitmap.installPixels(pixels)
+
+    return bitmap.asComposeImageBitmap()
+}