ITEP-68945 ITEP-68944 Using WebRTC for frames streaming

dlstreamer-pipeline-server/queuing-config-webrtc.json

@@ -0,0 +1,77 @@
+{
+  "config": {
+    "logging": {
+      "C_LOG_LEVEL": "INFO",
+      "PY_LOG_LEVEL": "INFO"
+    },
+    "pipelines": [
+      {
+        "name": "qcam2",
+        "source": "gstreamer",
+        "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/qcam2.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvametapublish name=destination ! appsink sync=true",
+        "auto_start": true,
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "ntp_config": {
+              "element": {
+                "name": "timesync",
+                "property": "kwarg",
+                "format": "json"
+              },
+              "type": "object",
+              "properties": {
+                "ntpServer": {
+                  "type": "string"
+                }
+              }
+            },
+            "camera_config": {
+              "element": {
+                "name": "datapublisher",
+                "property": "kwarg",
+                "format": "json"
+              },
+              "type": "object",
+              "properties": {
+                "cameraid": {
+                  "type": "string"
+                },
+                "metadatagenpolicy": {
+                  "type": "string",
+                  "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy"
+                },
+                "publish_frame": {
+                  "type": "boolean",
+                  "description": "Publish frame to mqtt"
+                }
+              }
+            }
+          }
+        },
+        "payload": {
+          "destination": {
+            "metadata": {
+              "type": "file",
+              "path": "/tmp/results.jsonl",
+              "format": "json-lines"
+            },
+            "frame": {
+              "type": "webrtc",
+              "peer-id": "cam"
+            }
+          },
+          "parameters": {
+            "ntp_config": {
+              "ntpServer": "ntpserv"
+            },
+            "camera_config": {
+              "cameraid": "atag-qcam2",
+              "metadatagenpolicy": "detectionPolicy"
+            }
+          }
+        }
+      }
+    ]
+  }
+}

dlstreamer-pipeline-server/queuing-config.json

@@ -119,4 +119,4 @@
       }
     ]
   }
-}
+}

manager/Dockerfile

@@ -14,11 +14,11 @@ SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
 RUN : \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
-        libboost-python-dev \
-        libegl1 \
-        libgl1 \
-        libglib2.0 \
-        python3-dev \
+    libboost-python-dev \
+    libegl1 \
+    libgl1 \
+    libglib2.0 \
+    python3-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # ---------- Manager Runtime Stage ------------------
@@ -42,38 +42,38 @@ ENV SCENESCAPE_HOME=/home/$WSUSER/SceneScape
 ARG FORCE_VAAPI
 
 RUN : \
-&& apt-get update \
+    && apt-get update \
     && apt-get install -y --no-install-recommends \
-        curl \
-        ca-certificates \
+    curl \
+    ca-certificates \
     && install -d /usr/share/postgresql-common/pgdg \
     && curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc \
     && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
-        apache2 \
-        bindfs \
-        bzip2 \
-        libegl1 \
-        libgl1 \
-        libapache2-mod-wsgi-py3 \
-        libapache2-mod-qos \
-        libglib2.0-0 \
-        libgomp1 \
-        mosquitto-clients \
-        netcat \
-        postgresql-15 \
-        python-is-python3 \
-        python3-pip \
-        sudo \
+    apache2 \
+    bindfs \
+    bzip2 \
+    libegl1 \
+    libgl1 \
+    libapache2-mod-wsgi-py3 \
+    libapache2-mod-qos \
+    libglib2.0-0 \
+    libgomp1 \
+    mosquitto-clients \
+    netcat \
+    postgresql-15 \
+    python-is-python3 \
+    python3-pip \
+    sudo \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 # Runtime dependencies
 COPY ./manager/requirements-runtime.txt /tmp
 RUN pip3 install --upgrade --no-cache-dir \
-        --extra-index-url https://download.pytorch.org/whl/cpu \
-        -r /tmp/requirements-runtime.txt
+    --extra-index-url https://download.pytorch.org/whl/cpu \
+    -r /tmp/requirements-runtime.txt
 
 RUN : \
     ; PSQLVER=$(ls /etc/postgresql) \
@@ -83,19 +83,19 @@ RUN : \
 
 RUN : \
     ; a2dismod -f \
-          auth_basic \
-          authn_core \
-          authn_file \
-          authz_host \
-          authz_user \
-          autoindex \
-          status \
+    auth_basic \
+    authn_core \
+    authn_file \
+    authz_host \
+    authz_user \
+    autoindex \
+    status \
     ; a2enmod \
-        allowmethods \
-        headers \
-        proxy \
-        proxy_wstunnel \
-        ssl \
+    allowmethods \
+    headers \
+    proxy \
+    proxy_wstunnel \
+    ssl \
     ; a2ensite default-ssl \
     ;
 
@@ -155,12 +155,15 @@ RUN chmod -R u=rwX,go=rX $SCENESCAPE_HOME/manager/static/bootstrap/
 RUN OPENCV_SHA=$(openssl dgst -sha256 -binary "$SCENESCAPE_HOME/manager/static/assets/opencv.js" | openssl base64) && \
     sed -i "s|sha256-opencv|sha256-${OPENCV_SHA}|g" /etc/apache2/sites-available/000-default.conf
 
+RUN READER_SHA=$(openssl dgst -sha256 -binary "$SCENESCAPE_HOME/manager/static/assets/reader.js" | openssl base64) && \
+    sed -i "s|sha256-reader|sha256-${READER_SHA}|g" /etc/apache2/sites-available/000-default.conf
+
 COPY ./manager/config/user_access_config.json $SCENESCAPE_HOME
 
 COPY ./manager/config/webserver-init \
-     ./manager/config/database-init \
-     ./manager/config/scenescape-init \
-     /usr/local/bin/
+    ./manager/config/database-init \
+    ./manager/config/scenescape-init \
+    /usr/local/bin/
 
 HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
     CMD pgrep postgres || exit 1
@@ -176,20 +179,20 @@ RUN : \
     ; apt-get update \
     ; install -d -m 0755 /etc/apt/keyrings \
     ; apt-get install -y --no-install-recommends \
-        wget \
-        libpci-dev \
-        libnss3-tools \
-        software-properties-common \
-        unzip \
-        xserver-xephyr \
-        xvfb \
+    wget \
+    libpci-dev \
+    libnss3-tools \
+    software-properties-common \
+    unzip \
+    xserver-xephyr \
+    xvfb \
     ; wget -q https://packages.mozilla.org/apt/repo-signing-key.gpg -O- | tee /etc/apt/keyrings/packages.mozilla.org.asc > /dev/null \
     ; echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" | tee -a /etc/apt/sources.list.d/mozilla.list > /dev/null \
     ; echo -e ' \n\
-Package: * \n\
-Pin: origin packages.mozilla.org \n\
-Pin-Priority: 1000 \n\
-' | tee /etc/apt/preferences.d/mozilla \
+    Package: * \n\
+    Pin: origin packages.mozilla.org \n\
+    Pin-Priority: 1000 \n\
+    ' | tee /etc/apt/preferences.d/mozilla \
     ; apt-get update && apt-get install -y --no-install-recommends firefox \
     ; rm -rf /var/lib/apt/lists/*
 

manager/Makefile

@@ -92,6 +92,7 @@ jslibraries: \
   $(JSLIBDIR)/assets/axios.min.js.map                        \
   $(JSLIBDIR)/assets/doxygen-awesome.css                     \
   $(JSLIBDIR)/assets/jszip.min.js                            \
+	$(JSLIBDIR)/assets/reader.js                               \
 
 $(JSLIBDIR)/assets/jquery-3.6.0.min.js:
 	URL=https://code.jquery.com && PKG=$$(basename $@) \
@@ -135,6 +136,11 @@ $(JSLIBDIR)/assets/glfx.min.js:
 	  && echo "$${URL}/$${PKG}" >> $(BUILD_DIR)/$(IMAGE)-downloaded-deps.txt \
 	  && curl -f $(CURL_FLAGS) --location --create-dirs --output $@ $${URL}/$${PKG}
 
+$(JSLIBDIR)/assets/reader.js:
+	URL=https://raw.githubusercontent.com/bluenviron/mediamtx/refs/tags/v1.14.0/internal/servers/webrtc/reader.js  \
+	  && echo https://raw.githubusercontent.com/bluenviron/mediamtx/refs/tags/v1.14.0/internal/servers/webrtc/reader.js >> $(BUILD_DIR)/$(IMAGE)-downloaded-deps.txt \
+	  && curl -f $(CURL_FLAGS) --location --create-dirs --output $@ https://raw.githubusercontent.com/bluenviron/mediamtx/refs/tags/v1.14.0/internal/servers/webrtc/reader.js
+
 $(JSLIBDIR)/bootstrap-4.6.1-dist.zip:
 	VERSION=$$(echo $@ | sed -E -e 's/.*bootstrap-([0-9]+[.][0-9]+[.][0-9]+)-.*/\1/') \
 	  URL=https://github.com/twbs/bootstrap/releases/download/v$${VERSION} \

manager/config/000-default.conf

@@ -56,6 +56,6 @@ ServerTokens Prod
 ServerSignature Off
 TraceEnable Off
 
-Header set Content-Security-Policy "frame-ancestors 'none'; form-action 'self'; script-src 'self' 'sha256-opencv' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; frame-src 'none';"
+Header set Content-Security-Policy "frame-ancestors 'none'; form-action 'self'; script-src 'self' 'sha256-opencv' 'sha256-reader' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; frame-src 'self' https://10.123.233.203:8889; upgrade-insecure-requests;"
 Header set Strict-Transport-Security "max-age=1024000; includeSubDomains"
 

manager/src/django/settings.py

@@ -17,7 +17,7 @@
 # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = os.getenv('DJANGO_DEBUG', 'False').lower() in ['true', '1', 't']
+DEBUG = True
 
 ALLOWED_HOSTS = ['*']
 DEFAULT_CHARSET = "utf-8"

manager/src/management/commands/waitfordb.py

@@ -58,8 +58,8 @@ def handle(self, *args, **options):
         log.info("Database ready")
         break
 
-      except psycopg2.OperationalError:
-        log.error("Server not online")
+      except psycopg2.OperationalError as er:
+        log.error("Server not online", er)
         time.sleep(2)
 
       except Exception as ex:

manager/src/static/js/sscape.js

@@ -101,6 +101,7 @@ async function checkBrokerConnections() {
   $("#connect").on("click", function () {
     console.log("Attempting to connect to " + broker.value);
     var client = mqtt.connect(broker.value);
+    var webRTCClient = null
     sessionStorage.setItem("connectToMqtt", true);
 
     client.on("connect", function () {
@@ -259,7 +260,7 @@ async function checkBrokerConnections() {
             .stop()
             .show()
             .css("opacity", 1)
-            .animate({ opacity: 0.6 }, 5000, function () {})
+            .animate({ opacity: 0.6 }, 5000, function () { })
             .prevAll(".cam-offline")
             .hide();
         }

manager/src/static/js/video.js

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: (C) 2024 - 2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * @file camcanvas.js
+ * @description This file defines the CamCanvas class for displaying video frames.
+ */
+
+"use strict";
+
+
+function playVideo() {
+  const video = document.getElementById('whepVideo');
+  let defaultControls = false;
+  let reader = null;
+
+  const loadAttributesFromQuery = () => {
+    video.controls = false;
+    video.muted = true;
+    video.autoplay = true;
+    video.playsInline = true;
+    video.disablepictureinpicture = true;
+  };
+
+  window.addEventListener('load', () => {
+    loadAttributesFromQuery();
+
+    reader = new MediaMTXWebRTCReader({
+      url: new URL('whep', 'http://10.123.233.203:8889/cam'),
+      onError: (err) => {
+        console.log(err)
+      },
+      onTrack: (evt) => {
+        video.srcObject = evt.streams[0];
+      },
+    });
+  });
+
+  window.addEventListener('beforeunload', () => {
+    if (reader !== null) {
+      reader.close();
+    }
+  });
+}