Merge pull request #692 from aisingapore/dev

Merge `dev` into `main` for v1.3 release

.github/workflows/pre_merge_ci.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-2019, ubuntu-18.04]
+        os: [ubuntu-18.04, windows-2019]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

README.md

@@ -57,7 +57,7 @@ PeekingDuck can also be [installed in a virtual environment](https://peekingduck
 
 
 ```
-> peekingduck --verify_install
+> peekingduck verify-install
 ```
 
 You should see a video of a person waving his hand with

__main__.py

@@ -45,9 +45,19 @@
     type=int,
     help="Stop pipeline after running this number of iterations",
 )
+@click.option(
+    "--viewer",
+    default=False,
+    is_flag=True,
+    help="Launch PeekingDuck viewer",
+)
 @click.pass_context
 def main(
-    context: click.Context, config_path: str, log_level: str, num_iter: int
+    context: click.Context,
+    config_path: str,
+    log_level: str,
+    num_iter: int,
+    viewer: bool,
 ) -> None:
     """Invokes the run() CLI command with some different defaults for
     ``node_config`` and ``nodes_parent_dir``.
@@ -74,6 +84,7 @@ def main(
         log_level=log_level,
         num_iter=num_iter,
         nodes_parent_dir=nodes_parent_dir,
+        viewer=viewer,
     )
 
 

cicd_requirements.txt

@@ -4,6 +4,8 @@ coverage == 5.5
 mypy == 0.910
 pylint == 2.7.4
 pytest == 6.2.3
+pytest-lazy-fixture == 0.6.3    # can remove if/when https://docs.pytest.org/en/7.1.x/proposals/parametrize_with_fixtures.html gets implemented
+scikit-image == 0.17.2
 types-PyYAML == 5.4.3
 types-requests == 2.25.0
 types-setuptools == 57.4.2

docs/source/conf.py

@@ -88,9 +88,11 @@
     "model.fairmotv1",
     "model.hrnetv1",
     "model.jdev1",
+    "model.mask_rcnnv1",
     "model.mtcnnv1",
     "model.posenetv1",
     "model.movenetv1",
+    "model.yolact_edgev1",
     "model.yolov4",
     "model.yolov4_face",
     "model.yolov4_license_plate",

docs/source/ecosystem/index.rst

@@ -0,0 +1,17 @@
+:orphan:
+
+.. include:: /include/substitution.rst
+
+.. _applications:
+
+*********************
+PeekingDuck Ecosystem
+*********************
+
+This section covers the extensions to the PeekingDuck ecosystem.
+
+.. toctree::
+   :maxdepth: 2
+
+   /ecosystem/viewer
+

docs/source/ecosystem/viewer.rst

@@ -0,0 +1,109 @@
+******************
+PeekingDuck Viewer
+******************
+
+.. include:: /include/substitution.rst
+
+.. _pkd_viewer:
+
+The PeekingDuck Viewer offers you an interactive GUI application to manage and run
+PeekingDuck pipelines, and to view and analyze the output video.
+
+
+Running the Viewer
+------------------
+
+The PeekingDuck Viewer can be activated using the CLI ``--viewer`` option:
+
+   .. admonition:: Terminal Session
+
+      | \ :blue:`[~user]` \ > \ :green:`peekingduck run \-\-viewer` \
+
+A screenshot of the Viewer and its GUI components is shown below:
+
+   .. figure:: /assets/tutorials/viewer_main.png
+      :alt: PeekingDuck Viewer Screenshot
+
+      The PeekingDuck Viewer screen, with explanations for the main controls.
+
+Once the Viewer screen appears, PeekingDuck will begin executing the current pipeline.  
+The pipeline output is displayed as a video in the center of the screen, with a progress
+bar below it.
+
+If pipeline input length is deterministic (e.g. using a video file as the source), the
+progress bar functions like a normal progress bar moving from start to end.  
+Upon completion, the progress bar will be replaced with a slider that you can use 
+to navigate the output video.
+
+If the length is non-deterministic (e.g. capturing a webcam video), then the progress bar
+will function in a non-deterministic manner: animating itself to indicate progress
+but without an end point (as PeekingDuck has no idea how long the webcam video will be). 
+In this case, click the ``Play/Stop`` button to end the webcam video capture, and the
+progress bar will become a slider.
+
+
+Navigating the Output Video
+---------------------------
+
+You can examine the output video of the executed pipeline by using the
+``Play/Stop`` button to replay the entire video. 
+
+You may also scrub through the video using the slider to go directly to the frames 
+of interest.
+The current video frame number is shown to the right of the slider, serving as a
+position indicator.
+To "jump" to a particular position on the slider, click the *right* mouse button on that
+position.
+To move frame-by-frame forward/backward, click the *left* mouse button anywhere to the
+right/left of the current slider position.
+
+The ``+`` (zoom in) and ``-`` (zoom out) buttons allow you to adjust the video size. 
+You may also use keyboard shortcuts to adjust the zoom: ``CTRL`` - ``-`` zoom out,
+``CTRL`` - ``+`` zoom in, ``CTRL`` - ``=`` reset zoom
+
+
+Using the Pipeline Playlist
+---------------------------
+
+Clicking the ``Playlist`` button will show/hide the playlist.
+
+   .. figure:: /assets/tutorials/viewer_playlist.png
+      :alt: PeekingDuck Viewer with Playlist Screenshot
+
+      PeekingDuck Viewer with playlist shown.
+
+The above screenshot shows the playlist on the right. 
+The playlist is a collection of pipeline files that can be run with PeekingDuck. 
+The current pipeline is automatically added to the playlist. 
+This playlist is specific to you and is saved across different PeekingDuck Viewer runs.
+
+Click to select a pipeline in the playlist. The pipeline's information will be displayed in 
+the ``Pipeline Information`` panel below. It shows the pipeline's name, last modified date/time, 
+and full file path.
+
+To run the currently selected pipeline, click the ``Run`` button.
+
+The ``Add`` button lets you manually add a pipeline file to the playlist. It will display 
+a File Explorer dialog. Use it to select a PeekingDuck pipeline YAML file and it will be 
+added to your playlist.
+
+The ``Delete`` button will remove the currently selected pipeline from the playlist, after 
+you have confirmed the deletion. 
+
+If the pipeline in the playlist is red, it means the pipeline YAML file is missing. 
+This could mean the pipeline had been added earlier, but its YAML file had since been 
+deleted or moved to another folder. 
+Delete the missing pipeline entry to remove it from the playlist.
+
+The list of pipelines can be sorted in reverse order by clicking the playlist header.
+
+.. note::
+
+   The playlist is saved in `~/.peekingduck/playlist.yml`, where `~` is the user's home folder.
+
+
+Exiting the Viewer
+------------------
+
+To exit the Viewer, close the Viewer window.
+

docs/source/edge_ai/index.rst

@@ -0,0 +1,143 @@
+:orphan:
+
+.. include:: /include/substitution.rst
+
+.. _edge_ai:
+
+*******
+Edge AI
+*******
+
+PeekingDuck supports running optimized TensorRT [1]_ models on devices with NVIDIA GPUs.
+Using the TensorRT model on these devices provides a speed boost over the regular
+TensorFlow/PyTorch version.
+A potential use case is running PeekingDuck on an NVIDIA Jetson device for Edge AI
+inference.
+
+Currently, PeekingDuck includes TensorRT versions of the following models:
+
+#. MoveNet model for pose estimation,
+#. YOLOX model for object detection.
+
+
+Installing TensorRT
+===================
+
+The following packages are required to run PeekingDuck's TensorRT models:
+
+#. TensorFlow
+#. PyTorch
+#. PyCUDA
+
+As the actual installation steps vary greatly depending on the user's device, operating
+system, software environment, and pre-installed libraries/packages,
+we are unable to provide step-by-step installation instructions.
+
+The user may refer to `NVIDIA's TensorRT Documentation 
+<https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html>`_ for detailed
+TensorRT installation information.
+
+
+
+
+
+Using TensorRT Models
+=====================
+
+To use the TensorRT version of a model, change the ``model_format`` of the model
+configuration to ``tensorrt``.
+
+The following ``pipeline_config.yml`` shows how to use the MoveNet TensorRT model 
+for pose estimation:
+
+.. code-block:: yaml
+    :linenos:
+
+    nodes:
+    - input.visual:
+        source: https://storage.googleapis.com/peekingduck/videos/wave.mp4
+    - model.movenet:
+        model_format: tensorrt
+        model_type: singlepose_lightning
+    - draw.poses
+    - dabble.fps
+    - draw.legend:
+        show: ["fps"]
+    - output.screen
+
+
+The following ``pipeline_config.yml`` shows how to use the YOLOX TensorRT model 
+for object detection:
+
+.. code-block:: yaml
+    :linenos:
+
+    nodes:
+    - input.visual:
+        source: https://storage.googleapis.com/peekingduck/videos/cat_and_computer.mp4
+    - model.yolox:
+        detect: ["cup", "cat", "laptop", "keyboard", "mouse"]
+        model_format: tensorrt
+        model_type: yolox-tiny
+    - draw.bbox:
+        show_labels: True    # configure draw.bbox to display object labels
+    - dabble.fps
+    - draw.legend:
+        show: ["fps"]
+    - output.screen
+
+
+Performance Speedup
+===================
+
+The following charts show the speed up obtainable with the TensorRT models.
+The numbers were obtained from our in-house testing with the actual devices.
+
+
+NVIDIA Jetson Xavier NX with 8GB RAM
+------------------------------------
+
+.. figure:: /assets/charts/tensorrt_nx_movenet_fps.png
+.. figure:: /assets/charts/tensorrt_nx_yolox_fps.png
+
+    Jetson Xavier NX specs used for testing: [2]_
+
+    ``CPU``: 6 cores (6MB L2 + 4MB L3) |br|
+    ``GPU``: 384-core Volta, 48 Tensor cores |br|
+    ``RAM``: 8 GB
+
+
+NVIDIA Jetson Xavier AGX with 16GB RAM
+--------------------------------------
+
+.. figure:: /assets/charts/tensorrt_agx_movenet_fps.png
+.. figure:: /assets/charts/tensorrt_agx_yolox_fps.png
+
+    Jetson Xavier AGX specs used for testing: [3]_
+
+    ``CPU``: 8 cores (8MB L2 + 4MB L3) |br|
+    ``GPU``: 512-core Volta, 64 Tensor cores |br|
+    ``RAM``: 16 GB
+
+
+Test Conditions
+^^^^^^^^^^^^^^^
+
+The following test conditions were followed:
+ | - :mod:`input.visual`, the model of interest, and :mod:`dabble.fps` nodes were used to perform
+     inference on videos
+ | - 2 videos were used to benchmark each model, one with only 1 human (``single``), and the other
+     with multiple humans (``multiple``)
+ | - Both videos are about 1 minute each, recorded at ~30 FPS, which translates to about 1,800
+     frames to process per video
+ | - 1280×720 (HD ready) resolution was used, as a bridge between 640×480 (VGA) of poorer quality
+     webcams, and 1920×1080 (Full HD) of CCTVs
+ | - FPS numbers are averaged over 5 separate runs
+
+
+References
+==========
+
+.. [1] `NVIDIA TensorRT Reference <https://developer.nvidia.com/tensorrt>`_
+.. [2] `NVIDIA Jetson Xavier NX Tech Specs <https://developer.nvidia.com/embedded/jetson-xavier-nx-devkit>`_
+.. [3] `NVIDIA Jetson Xavier AGX Tech Specs <https://developer.nvidia.com/embedded/jetson-agx-xavier-developer-kit>`_

docs/source/getting_started/02_standard_install.rst

@@ -98,7 +98,11 @@ To check that PeekingDuck is installed successfully, run the following command:
 
 .. admonition:: Terminal Session
 
-    | \ :blue:`[~user]` \ > \ :green:`peekingduck -\-verify_install` \
+    | \ :blue:`[~user]` \ > \ :green:`peekingduck verify-install` \
+
+.. versionchanged:: 1.3.0
+    The verify installation command has been changed from ``--verify_install`` to
+    ``verify-install``.
 
 You should see a video of a person waving his hand (`taken from here <https://www.youtube.com/watch?v=IKj_z2hgYUM>`_)
 with bounding boxes overlaid as shown below:

docs/source/getting_started/03_custom_install.rst

@@ -26,7 +26,10 @@ Verify the installation using:
 
 .. admonition:: Terminal Session
 
-   | \ :blue:`[~user]` \ > \ :green:`peekingduck -\-verify_install` \
+   | \ :blue:`[~user]` \ > \ :green:`peekingduck verify-install` \
+
+*See* :ref:`here <verify_installation>` *for changes to the verify installation command in
+version 1.3.0.*
 
 You should see a video of :ref:`a person waving his hand with bounding boxes overlaid
 <custom_install_verify_gif>`.
@@ -88,7 +91,11 @@ macOS Monterey.
    .. admonition:: Terminal Session
 
       | \ :blue:`[~user]` \ > \ :green:`pip install peekingduck -\-no-dependencies` \
-      | \ :blue:`[~user]` \ > \ :green:`peekingduck -\-verify_install` \
+      | \ :blue:`[~user]` \ > \ :green:`peekingduck verify-install` \
+
+
+   *See* :ref:`here <verify_installation>` *for changes to the verify installation command in
+   version 1.3.0.*
 
 .. _custom_install_verify_gif:
 

docs/source/glossary.rst

@@ -60,6 +60,9 @@ types when working with custom nodes.
 
    |large_groups|
       |large_groups_def|
+
+   |masks|
+      |masks_def|
 
    (input) |none|
       |none_input_def|