Write YOLO Detector and Drawer

.circleci/config.yml

@@ -1,9 +1,9 @@
 version: 2.1
 orbs:
-  elixir: membraneframework/elixir@1
+  elixir: membraneframework/docker_membrane@1
 
 workflows:
-  version: 2
+  version: 3
   build:
     jobs:
       - elixir/build_test:

.formatter.exs

@@ -1,6 +1,6 @@
 [
   inputs: [
-    "{lib,test,config}/**/*.{ex,exs}",
+    "{lib,test,config,examples}/**/*.{ex,exs}",
     ".formatter.exs",
     "*.exs"
   ],

.gitignore

@@ -1,3 +1,6 @@
+examples/models/*.onnx
+examples/outputs/*
+
 compile_commands.json
 .gdb_history
 bundlex.sh

README.md

@@ -1,35 +1,37 @@
-# Membrane Template Plugin
+# Membrane YOLO Plugin
 
-[![Hex.pm](https://img.shields.io/hexpm/v/membrane_template_plugin.svg)](https://hex.pm/packages/membrane_template_plugin)
-[![API Docs](https://img.shields.io/badge/api-docs-yellow.svg?style=flat)](https://hexdocs.pm/membrane_template_plugin)
-[![CircleCI](https://circleci.com/gh/membraneframework/membrane_template_plugin.svg?style=svg)](https://circleci.com/gh/membraneframework/membrane_template_plugin)
+[![Hex.pm](https://img.shields.io/hexpm/v/membrane_yolo_plugin.svg)](https://hex.pm/packages/membrane_yolo_plugin)
+[![API Docs](https://img.shields.io/badge/api-docs-yellow.svg?style=flat)](https://hexdocs.pm/membrane_yolo_plugin)
+[![CircleCI](https://circleci.com/gh/membraneframework/membrane_yolo_plugin.svg?style=svg)](https://circleci.com/gh/membraneframework/membrane_yolo§  §_plugin)
 
-This repository contains a template for new plugins.
+Contains 2 Membrane Filters:
+ - `Membrane.YOLO.Detector` - for running object detection on a video stream.
+ - `Membrane.YOLO.Drawer` - for drawing object detection results generated by `Membrane.YOLO.Detector`.
 
-Check out different branches for other flavors of this template.
+Uses under the hood [yolo_elixir](https://github.com/poeticoding/yolo_elixir).
 
 It's a part of the [Membrane Framework](https://membrane.stream).
 
 ## Installation
 
-The package can be installed by adding `membrane_template_plugin` to your list of dependencies in `mix.exs`:
+The package can be installed by adding `membrane_yolo_plugin` to your list of dependencies in `mix.exs`:
 
 ```elixir
 def deps do
   [
-    {:membrane_template_plugin, "~> 0.1.0"}
+    {:membrane_yolo_plugin, "~> 0.1.0"}
   ]
 end
 ```
 
-## Usage
+## Examples
 
-TODO
+Open a Livebook `examples/yolo.livemd` or run `$ elixir examples/live_camera_capture.exs`, `$ elixir examples/live_mp4_processing.exs` and `$ elixir examples/offline_mp4_processing.exs`
 
 ## Copyright and License
 
-Copyright 2020, [Software Mansion](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_template_plugin)
+Copyright 2025, [Software Mansion](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_yolo_plugin)
 
-[![Software Mansion](https://logo.swmansion.com/logo?color=white&variant=desktop&width=200&tag=membrane-github)](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_template_plugin)
+[![Software Mansion](https://logo.swmansion.com/logo?color=white&variant=desktop&width=200&tag=membrane-github)](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_yolo_plugin)
 
 Licensed under the [Apache License, Version 2.0](LICENSE)

examples/live_camera_capture.exs

@@ -0,0 +1,73 @@
+hardware_acceleration =
+  case :os.type() do
+    {:unix, :darwin} -> :coreml
+    {:unix, :linux} -> :cuda
+  end
+
+Mix.install(
+  [
+    {:membrane_yolo_plugin, path: Path.join(__DIR__, "..")},
+    {:membrane_core, "~> 1.0"},
+    {:membrane_camera_capture_plugin, "~> 0.7.4"},
+    {:membrane_ffmpeg_swscale_plugin, "~> 0.16.3"},
+    {:boombox, "~> 0.2.8"},
+    {:exla, "~> 0.10"}
+  ],
+  config: [
+    ortex: [
+      {Ortex.Native, [features: [hardware_acceleration]]}
+    ],
+    nx: [
+      default_backend: EXLA.Backend
+    ]
+  ]
+)
+
+Logger.configure(level: :info)
+
+model_name = "yolox_l.onnx"
+model_path = Path.join("examples/models", model_name)
+
+if not File.exists?(model_path) do
+  model_url =
+    "https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/#{model_name}"
+
+  %{body: data} = Req.get!(model_url)
+  File.write!(model_path, data)
+end
+
+defmodule YOLO.CameraCapture.Pipeline do
+  use Membrane.Pipeline
+
+  @impl true
+  def handle_init(_ctx, _opts) do
+    spec =
+      child(:camera_capture, Membrane.CameraCapture)
+      |> child(:swscale_converter, %Membrane.FFmpeg.SWScale.Converter{
+        format: :RGB,
+        output_width: 640
+      })
+      |> child(:yolo_detector, %Membrane.YOLO.Detector{
+        mode: :live_low_latency,
+        yolo_model:
+          YOLO.load(
+            model_impl: YOLO.Models.YOLOX,
+            model_path: "examples/models/yolox_l.onnx",
+            classes_path: "examples/models/coco_classes.json",
+            eps: [unquote(hardware_acceleration)]
+          )
+      })
+      |> child(:yolo_drawer, Membrane.YOLO.Drawer)
+      |> via_in(:input, options: [kind: :video])
+      |> child(:boombox_sink, %Boombox.Bin{output: :player})
+
+    {[spec: spec], %{}}
+  end
+end
+
+{:ok, _supervisor, pipeline} = Membrane.Pipeline.start_link(YOLO.CameraCapture.Pipeline, [])
+Process.monitor(pipeline)
+
+receive do
+  {:DOWN, _ref, :process, _pid, _reason} -> :ok
+end

examples/live_mp4_processing.exs

@@ -0,0 +1,82 @@
+hardware_acceleration =
+  case :os.type() do
+    {:unix, :darwin} -> :coreml
+    {:unix, :linux} -> :cuda
+  end
+
+Mix.install(
+  [
+    {:membrane_yolo_plugin, path: Path.join(__DIR__, "..")},
+    {:membrane_core, "~> 1.0"},
+    {:membrane_camera_capture_plugin, "~> 0.7.4"},
+    {:membrane_ffmpeg_swscale_plugin, "~> 0.16.3"},
+    {:boombox, "~> 0.2.8"},
+    {:exla, "~> 0.10"}
+  ],
+  config: [
+    ortex: [
+      {Ortex.Native, [features: [hardware_acceleration]]}
+    ],
+    nx: [
+      default_backend: EXLA.Backend
+    ]
+  ]
+)
+
+Logger.configure(level: :info)
+
+model_name = "yolox_l.onnx"
+model_path = Path.join("examples/models", model_name)
+
+if not File.exists?(model_path) do
+  model_url =
+    "https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/#{model_name}"
+
+  %{body: data} = Req.get!(model_url)
+  File.write!(model_path, data)
+end
+
+defmodule YOLO.MP4.LivePipeline do
+  use Membrane.Pipeline
+
+  @impl true
+  def handle_init(_ctx, _opts) do
+    spec =
+      child(:mp4_source, %Boombox.Bin{input: "examples/fixtures/street.mp4"})
+      |> via_out(:output, options: [kind: :video])
+      |> child(:transcoder, %Membrane.Transcoder{output_stream_format: Membrane.RawVideo})
+      |> child(:realtimer, Membrane.Realtimer)
+      |> child(:swscale_converter, %Membrane.FFmpeg.SWScale.Converter{
+        format: :RGB,
+        output_width: 640
+      })
+      |> child(:yolo_detector, %Membrane.YOLO.Detector{
+        mode: :live,
+        yolo_model:
+          YOLO.load(
+            model_impl: YOLO.Models.YOLOX,
+            model_path: "examples/models/yolox_l.onnx",
+            classes_path: "examples/models/coco_classes.json",
+            eps: [unquote(hardware_acceleration)]
+          ),
+        additional_latency: Membrane.Time.milliseconds(500)
+      })
+      |> child(:yolo_drawer, Membrane.YOLO.Drawer)
+      |> via_in(:input, options: [kind: :video])
+      |> child(:boombox_sink, %Boombox.Bin{output: :player})
+
+    {[spec: spec], %{}}
+  end
+
+  @impl true
+  def handle_child_notification(:processing_finished, :boombox_sink, _ctx, state) do
+    {[terminate: :normal], state}
+  end
+end
+
+{:ok, supervisor, _pipeline} = Membrane.Pipeline.start_link(YOLO.MP4.LivePipeline, [])
+Process.monitor(supervisor)
+
+receive do
+  {:DOWN, _ref, :process, _pid, _reason} -> :ok
+end

examples/offline_mp4_processing.exs

@@ -0,0 +1,97 @@
+hardware_acceleration =
+  case :os.type() do
+    {:unix, :darwin} -> :coreml
+    {:unix, :linux} -> :cuda
+  end
+
+Mix.install(
+  [
+    {:membrane_yolo_plugin, path: Path.join(__DIR__, "..")},
+    {:membrane_core, "~> 1.0"},
+    {:membrane_camera_capture_plugin, "~> 0.7.4"},
+    {:membrane_ffmpeg_swscale_plugin, "~> 0.16.3"},
+    {:boombox, "~> 0.2.8"},
+    {:exla, "~> 0.10"}
+  ],
+  config: [
+    ortex: [
+      {Ortex.Native, [features: [hardware_acceleration]]}
+    ],
+    nx: [
+      default_backend: EXLA.Backend
+    ]
+  ]
+)
+
+Logger.configure(level: :info)
+
+model_name = "yolox_l.onnx"
+model_path = Path.join("examples/models", model_name)
+
+if not File.exists?(model_path) do
+  model_url =
+    "https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/#{model_name}"
+
+  %{body: data} = Req.get!(model_url)
+  File.write!(model_path, data)
+end
+
+result_file_path = "examples/outputs/street_bounding_boxes.mp4"
+
+defmodule YOLO.MP4.OfflinePipeline do
+  use Membrane.Pipeline
+  require Membrane.Logger
+
+  @impl true
+  def handle_init(_ctx, _opts) do
+    spec =
+      child(:mp4_source, %Boombox.Bin{input: "examples/fixtures/street_short.mp4"})
+      |> via_out(:output, options: [kind: :video])
+      |> child(:transcoder, %Membrane.Transcoder{output_stream_format: Membrane.RawVideo})
+      |> child(:rgb_converter, %Membrane.FFmpeg.SWScale.Converter{
+        format: :RGB,
+        output_width: 640
+      })
+      |> child(:yolo_detector, %Membrane.YOLO.Detector{
+        mode: :offline,
+        yolo_model:
+          YOLO.load(
+            model_impl: YOLO.Models.YOLOX,
+            model_path: "examples/models/yolox_l.onnx",
+            classes_path: "examples/models/coco_classes.json",
+            eps: [unquote(hardware_acceleration)]
+          )
+      })
+      |> child(:yolo_drawer, Membrane.YOLO.Drawer)
+      |> child(:debug_logger, %Membrane.Debug.Filter{
+        handle_buffer: fn buffer ->
+          pts_ms = Membrane.Time.as_milliseconds(buffer.pts, :round)
+
+          Membrane.Logger.info("""
+          Processed #{inspect(pts_ms)} ms of 10_000 ms of fixture video
+          """)
+        end
+      })
+      |> child(:i420_converter, %Membrane.FFmpeg.SWScale.Converter{
+        format: :I420
+      })
+      |> via_in(:input, options: [kind: :video])
+      |> child(:boombox_sink, %Boombox.Bin{output: unquote(result_file_path)})
+
+    {[spec: spec], %{}}
+  end
+
+  @impl true
+  def handle_child_notification(:processing_finished, :boombox_sink, _ctx, state) do
+    {[terminate: :normal], state}
+  end
+end
+
+{:ok, supervisor, _pipeline} = Membrane.Pipeline.start_link(YOLO.MP4.OfflinePipeline, [])
+Process.monitor(supervisor)
+
+receive do
+  {:DOWN, _ref, :process, _pid, :normal} -> :ok
+end
+
+Boombox.play(result_file_path)