diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 000000000..2530c53ff --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 aler9 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/dlstreamer-pipeline-server/convert_video_to_ts.sh b/dlstreamer-pipeline-server/convert_video_to_ts.sh index 9390a73af..0de327d23 100755 --- a/dlstreamer-pipeline-server/convert_video_to_ts.sh +++ b/dlstreamer-pipeline-server/convert_video_to_ts.sh @@ -7,12 +7,12 @@ # to ts files so that gstreamer pipeline can keep running the files # in infinite loop without having to deallocate buffers -docker pull intel/intel-optimized-ffmpeg:latest +docker pull linuxserver/ffmpeg:version-8.0-cli DIRNAME=${PWD} SAMPLE_DATA_DIRECTORY=${DIRNAME}/sample_data FFMPEG_DIR="/app/data" -FFMPEG_IMAGE="intel/intel-optimized-ffmpeg:latest" +FFMPEG_IMAGE="linuxserver/ffmpeg:version-8.0-cli" EXTENSION=${1:-mp4} PATTERN="*.${EXTENSION}" @@ -26,7 +26,7 @@ for mfile in "$SAMPLE_DATA_DIRECTORY"/$PATTERN; do if [ -f $tsfile ]; then echo "skipping $basefile as $tsfile is available already" else - ffmpegcmd="/opt/build/bin/ffmpeg -i ${FFMPEG_DIR}/${basefile}.${EXTENSION} -c copy ${FFMPEG_DIR}/${basefile}.ts" + ffmpegcmd="ffmpeg -i ${FFMPEG_DIR}/${basefile}.${EXTENSION} -c:v libx264 -preset ultrafast -x264-params keyint=10:min-keyint=5 -tune zerolatency ${FFMPEG_DIR}/${basefile}.ts" cmd="$DOCKER_RUN_CMD_PREFIX -c '$ffmpegcmd'" eval $cmd fi diff --git a/dlstreamer-pipeline-server/perf-config.json b/dlstreamer-pipeline-server/perf-config.json index b379663e0..026a6cff9 100644 --- a/dlstreamer-pipeline-server/perf-config.json +++ b/dlstreamer-pipeline-server/perf-config.json @@ -8,7 +8,7 @@ { "name": "apriltag-cam1", "source": "gstreamer", - "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam1.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera1-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -70,7 +70,7 @@ { "name": "apriltag-cam2", "source": "gstreamer", - "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam2.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera2-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", diff --git a/dlstreamer-pipeline-server/queuing-config-no-ntp.json b/dlstreamer-pipeline-server/queuing-config-no-ntp.json index 660654b80..71ebd0116 100644 --- a/dlstreamer-pipeline-server/queuing-config-no-ntp.json +++ b/dlstreamer-pipeline-server/queuing-config-no-ntp.json @@ -8,7 +8,7 @@ { "name": "qcam1", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -27,16 +27,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "atag-qcam1", + "overlay": false + } + }, "parameters": { "camera_config": { "cameraid": "atag-qcam1", @@ -48,7 +51,7 @@ { "name": "qcam2", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -67,16 +70,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "atag-qcam2", + "overlay": false + } + }, "parameters": { "camera_config": { "cameraid": "atag-qcam2", diff --git a/dlstreamer-pipeline-server/queuing-config-no-rtsp.json b/dlstreamer-pipeline-server/queuing-config-no-rtsp.json deleted file mode 100644 index 10b4250c8..000000000 --- a/dlstreamer-pipeline-server/queuing-config-no-rtsp.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "config": { - "logging": { - "C_LOG_LEVEL": "INFO", - "PY_LOG_LEVEL": "INFO" - }, - "pipelines": [ - { - "name": "qcam1", - "source": "gstreamer", - "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/qcam1.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", - "auto_start": true, - "parameters": { - "type": "object", - "properties": { - "ntp_config": { - "element": { - "name": "timesync", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "ntpServer": { - "type": "string" - } - } - }, - "camera_config": { - "element": { - "name": "datapublisher", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "cameraid": { - "type": "string" - }, - "metadatagenpolicy": { - "type": "string", - "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" - } - } - } - } - }, - "payload": { - "parameters": { - "ntp_config": { - "ntpServer": "ntpserv" - }, - "camera_config": { - "cameraid": "atag-qcam1", - "metadatagenpolicy": "detectionPolicy" - } - } - } - }, - { - "name": "qcam2", - "source": "gstreamer", - "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/qcam2.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", - "auto_start": true, - "parameters": { - "type": "object", - "properties": { - "ntp_config": { - "element": { - "name": "timesync", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "ntpServer": { - "type": "string" - } - } - }, - "camera_config": { - "element": { - "name": "datapublisher", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "cameraid": { - "type": "string" - }, - "metadatagenpolicy": { - "type": "string", - "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" - } - } - } - } - }, - "payload": { - "parameters": { - "ntp_config": { - "ntpServer": "ntpserv" - }, - "camera_config": { - "cameraid": "atag-qcam2", - "metadatagenpolicy": "detectionPolicy" - } - } - } - } - ] - } -} diff --git a/dlstreamer-pipeline-server/queuing-config-reid.json b/dlstreamer-pipeline-server/queuing-config-reid.json index 31fa76133..dcfa54086 100644 --- a/dlstreamer-pipeline-server/queuing-config-reid.json +++ b/dlstreamer-pipeline-server/queuing-config-reid.json @@ -8,7 +8,7 @@ { "name": "reid-qcam1", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/atag-qcam1-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -40,16 +40,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "atag-qcam1", + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" @@ -64,7 +67,7 @@ { "name": "reid-qcam2", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/atag-qcam2-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -96,16 +99,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "atag-qcam2", + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" diff --git a/dlstreamer-pipeline-server/queuing-config.json b/dlstreamer-pipeline-server/queuing-config.json index 2f72d3d61..e3625ab97 100644 --- a/dlstreamer-pipeline-server/queuing-config.json +++ b/dlstreamer-pipeline-server/queuing-config.json @@ -8,7 +8,7 @@ { "name": "qcam1", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/atag-qcam1-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -40,16 +40,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "atag-qcam1", + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" @@ -64,7 +67,7 @@ { "name": "qcam2", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/queuing-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/atag-qcam2-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -96,16 +99,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "atag-qcam2", + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" diff --git a/dlstreamer-pipeline-server/retail-config-no-ntp.json b/dlstreamer-pipeline-server/retail-config-no-ntp.json index 44ed2f39e..6f52f2056 100644 --- a/dlstreamer-pipeline-server/retail-config-no-ntp.json +++ b/dlstreamer-pipeline-server/retail-config-no-ntp.json @@ -8,7 +8,7 @@ { "name": "apriltag-cam1", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/retail-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera1-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -27,16 +27,20 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "camera1", + "bitrate": 5000, + "overlay": false + } + }, "parameters": { "camera_config": { "cameraid": "camera1", @@ -48,7 +52,7 @@ { "name": "apriltag-cam2", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/retail-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera2-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -67,15 +71,19 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "camera2", + "bitrate": 5000, + "overlay": false + } + }, "payload": { "parameters": { "camera_config": { diff --git a/dlstreamer-pipeline-server/retail-config-no-rtsp.json b/dlstreamer-pipeline-server/retail-config-no-rtsp.json deleted file mode 100644 index 961716b4d..000000000 --- a/dlstreamer-pipeline-server/retail-config-no-rtsp.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "config": { - "logging": { - "C_LOG_LEVEL": "INFO", - "PY_LOG_LEVEL": "INFO" - }, - "pipelines": [ - { - "name": "apriltag-cam1", - "source": "gstreamer", - "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam1.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", - "auto_start": true, - "parameters": { - "type": "object", - "properties": { - "ntp_config": { - "element": { - "name": "timesync", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "ntpServer": { - "type": "string" - } - } - }, - "camera_config": { - "element": { - "name": "datapublisher", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "cameraid": { - "type": "string" - }, - "metadatagenpolicy": { - "type": "string", - "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" - } - } - } - } - }, - "payload": { - "parameters": { - "ntp_config": { - "ntpServer": "ntpserv" - }, - "camera_config": { - "cameraid": "camera1", - "metadatagenpolicy": "detectionPolicy" - } - } - } - }, - { - "name": "apriltag-cam2", - "source": "gstreamer", - "pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam2.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", - "auto_start": true, - "parameters": { - "type": "object", - "properties": { - "ntp_config": { - "element": { - "name": "timesync", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "ntpServer": { - "type": "string" - } - } - }, - "camera_config": { - "element": { - "name": "datapublisher", - "property": "kwarg", - "format": "json" - }, - "type": "object", - "properties": { - "cameraid": { - "type": "string" - }, - "metadatagenpolicy": { - "type": "string", - "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" - } - } - } - } - }, - "payload": { - "parameters": { - "ntp_config": { - "ntpServer": "ntpserv" - }, - "camera_config": { - "cameraid": "camera2", - "metadatagenpolicy": "detectionPolicy" - } - } - } - } - ] - } -} diff --git a/dlstreamer-pipeline-server/retail-config-reid.json b/dlstreamer-pipeline-server/retail-config-reid.json index eddf98408..0a8637d02 100644 --- a/dlstreamer-pipeline-server/retail-config-reid.json +++ b/dlstreamer-pipeline-server/retail-config-reid.json @@ -8,7 +8,7 @@ { "name": "reid_apriltag-cam1", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/retail-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera1-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -40,16 +40,20 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "camera1", + "bitrate": 5000, + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" @@ -64,7 +68,7 @@ { "name": "reid_apriltag-cam2", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/retail-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera2-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json name=detection ! gvainference model=/home/pipeline-server/models/intel/person-reidentification-retail-0277/FP32/person-reidentification-retail-0277.xml inference-region=roi-list ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -96,16 +100,20 @@ "metadatagenpolicy": { "type": "string", "description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy" - }, - "publish_frame": { - "type": "boolean", - "description": "Publish frame to mqtt" } } } } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "camera2", + "bitrate": 5000, + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" diff --git a/dlstreamer-pipeline-server/retail-config.json b/dlstreamer-pipeline-server/retail-config.json index f1c0d9a53..cec2f6e1f 100644 --- a/dlstreamer-pipeline-server/retail-config.json +++ b/dlstreamer-pipeline-server/retail-config.json @@ -8,7 +8,7 @@ { "name": "apriltag-cam1", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/retail-cam1 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera1-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -50,6 +50,14 @@ } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "camera1", + "bitrate": 5000, + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" @@ -64,7 +72,7 @@ { "name": "apriltag-cam2", "source": "gstreamer", - "pipeline": "rtspsrc location=rtsp://mediaserver:8554/retail-cam2 latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true", + "pipeline": "rtspsrc location=rtsp://mediaserver:8554/camera2-raw latency=200 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvawatermark ! gvametapublish name=destination ! appsink sync=true", "auto_start": true, "parameters": { "type": "object", @@ -106,6 +114,14 @@ } }, "payload": { + "destination": { + "frame": { + "type": "webrtc", + "peer-id": "camera2", + "bitrate": 5000, + "overlay": false + } + }, "parameters": { "ntp_config": { "ntpServer": "ntpserv" diff --git a/dlstreamer-pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py b/dlstreamer-pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py index 2dd6a88f1..50e44688a 100644 --- a/dlstreamer-pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py +++ b/dlstreamer-pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py @@ -88,10 +88,9 @@ def processFrame(self, frame): return True class PostInferenceDataPublish: - def __init__(self, cameraid, metadatagenpolicy='detectionPolicy', publish_image=False): + def __init__(self, cameraid, metadatagenpolicy='detectionPolicy'): self.cameraid = cameraid - self.is_publish_image = publish_image self.is_publish_calibration_image = False self.cam_auto_calibrate = False self.cam_auto_calibrate_intrinsics = None @@ -122,9 +121,7 @@ def setupMQTT(self): def handleCameraMessage(self, client, userdata, message): msg = message.payload.decode("utf-8") - if msg == "getimage": - self.is_publish_image = True - elif msg == "getcalibrationimage": + if msg == "getcalibrationimage": self.is_publish_calibration_image = True else: try: @@ -168,7 +165,7 @@ def annotateFPS(self, img, fpsval): 1 * scale, (255,255,255), 2 * scale) return - def buildImgData(self, imgdatadict, gvaframe, annotate, original_image_base64=None): + def buildImgData(self, imgdatadict, gvaframe, original_image_base64=None): imgdatadict.update({ 'timestamp': self.frame_level_data['timestamp'], 'id': self.cameraid @@ -186,9 +183,6 @@ def buildImgData(self, imgdatadict, gvaframe, annotate, original_image_base64=No image = original_image except (ValueError, Exception) as e: print(f"Error using original image: {e}. Falling back to current frame.") - if annotate: - self.annotateObjects(image) - self.annotateFPS(image, self.frame_level_data['rate']) _, jpeg = cv2.imencode(".jpg", image) jpeg = base64.b64encode(jpeg).decode('utf-8') imgdatadict['image'] = jpeg @@ -240,21 +234,10 @@ def processFrame(self, frame): original_image_base64 = gvametadata['original_image_base64'] self.buildObjData(gvametadata) - if self.is_publish_image: - self.buildImgData(annotated_img, frame, True, original_image_base64) - self.client.publish(f"scenescape/image/camera/{self.cameraid}", json.dumps(annotated_img)) - self.is_publish_image = False - - if self.is_publish_calibration_image: - if not unannotated_img: - self.buildImgData(unannotated_img, frame, False, original_image_base64) - self.client.publish(f"scenescape/image/calibration/camera/{self.cameraid}", json.dumps(unannotated_img)) - self.is_publish_calibration_image = False - if self.cam_auto_calibrate: self.cam_auto_calibrate = False if not unannotated_img: - self.buildImgData(unannotated_img, frame, False) + self.buildImgData(unannotated_img, frame) unannotated_img['calibrate'] = True if self.cam_auto_calibrate_intrinsics: unannotated_img['intrinsics'] = self.cam_auto_calibrate_intrinsics @@ -263,3 +246,4 @@ def processFrame(self, frame): self.client.publish(f"scenescape/data/camera/{self.cameraid}", json.dumps(self.frame_level_data)) frame.add_message(json.dumps(self.frame_level_data)) return True + diff --git a/docs/user-guide/Getting-Started-Guide.md b/docs/user-guide/Getting-Started-Guide.md index d5d8a3820..94e59f74e 100644 --- a/docs/user-guide/Getting-Started-Guide.md +++ b/docs/user-guide/Getting-Started-Guide.md @@ -119,9 +119,30 @@ make demo ### Step 6: Verify a successful deployment -If you are running remotely, connect using `"https://"` or `"https://"`, using the correct IP address or hostname of the remote Intel® SceneScape system. If accessing on a local system use `"https://localhost"`. If you see a certificate warning, click the prompts to continue to the site. For example, in Chrome click "Advanced" and then "Proceed to <ip_address> (unsafe)". +#### Update the hosts file -> **Note:** These certificate warnings are expected due to the use of a self-signed certificate for initial deployment purposes. This certificate is generated at deploy time and is unique to the instance. +To ensure proper name resolution for Intel® SceneScape services, update your system's `/etc/hosts` file with the IP address of your machine for the following addresses: + +``` + coturn.scenescape.intel.com + web.scenescape.intel.com +``` + +Replace `` with the actual IP address of your Intel® SceneScape system. For example, if your machine's IP is `192.168.1.100`, add: + +``` +192.168.1.100 coturn.scenescape.intel.com +192.168.1.100 web.scenescape.intel.com +``` + +You may need administrative privileges to edit the `/etc/hosts` file. + +#### Download and install the SSL certificate + +From your your Intel® SceneScape system, download the SSL certificate to your local machine. It's under `scenescape/manager/secrets/certs/scenescape-ca.crt`. +Install the certificate in your browser or operating system to avoid certificate warnings when accessing Intel® SceneScape web interface. + +If you are running remotely, connect using `"https://web.scenescape.intel.com"`. ### Logging In diff --git a/manager/src/static/js/calibration.js b/manager/src/static/js/calibration.js index 3e6135bd8..bc43853ce 100644 --- a/manager/src/static/js/calibration.js +++ b/manager/src/static/js/calibration.js @@ -76,12 +76,13 @@ function initializeCalibrationSettings() { if ($(".cameraCal").length) { camera_calibration.initializeCamCanvas( $("#camera_img_canvas")[0], - $("#camera_img").attr("src"), + $("#video")[0], ); camera_calibration.initializeViewport( $("#map_canvas_3D")[0], $("#scale").val(), $("#scene").val(), + $("#video")[0], `Token ${$("#auth-token").val()}`, ); @@ -112,24 +113,6 @@ function initializeCalibrationSettings() { } } -function updateCalibrationView(msg) { - const image = "data:image/jpeg;base64," + msg.image; - const cameraMatrix = [ - [$("#id_intrinsics_fx").val(), 0, $("#id_intrinsics_cx").val()], - [0, $("#id_intrinsics_fy").val(), $("#id_intrinsics_cy").val()], - [0, 0, 1], - ]; - const distCoeffs = [ - $("#id_distortion_k1").val(), - $("#id_distortion_k2").val(), - $("#id_distortion_p1").val(), - $("#id_distortion_p2").val(), - $("#id_distortion_k3").val(), - ]; - camera_calibration.updateCalibrationViews(image, cameraMatrix, distCoeffs); - $("#snapshot").trigger("click"); -} - function handleAutoCalibrationPose(msg) { if (msg.error === "False") { camera_calibration.clearCalibrationPoints(); @@ -158,7 +141,6 @@ export { registerAutoCameraCalibration, manageCalibrationState, initializeCalibrationSettings, - updateCalibrationView, handleAutoCalibrationPose, setMqttForCalibration, }; diff --git a/manager/src/static/js/camcanvas.js b/manager/src/static/js/camcanvas.js index 09acfae25..826863f53 100644 --- a/manager/src/static/js/camcanvas.js +++ b/manager/src/static/js/camcanvas.js @@ -17,10 +17,10 @@ import { } from "/static/js/constants.js"; class CamCanvas { - constructor(canvas, initialImageSrc) { + constructor(canvas, video) { this.canvas = canvas; this.ctx = canvas.getContext("2d"); - this.image = new Image(); + this.video = video; this.calibrationPoints = []; this.calibrationPointNames = []; @@ -40,16 +40,16 @@ class CamCanvas { this.draggingPoint = null; this.calibrationUpdated = false; - this.image.onload = () => { + if (video) { this.handleImageLoad(); - }; + this.startVideoRendering(); + } this.initializeEventListeners(); - this.updateImageSrc(initialImageSrc); } - getImageSize() { - return [this.image.width, this.image.height]; + getVideoSize() { + return [this.video.videoWidth, this.video.videoHeight]; } initializeEventListeners() { @@ -93,7 +93,12 @@ class CamCanvas { } #isPointInBounds(x, y) { - return x >= 0 && x < this.image.width && y >= 0 && y < this.image.height; + return ( + x >= 0 && + x < this.video.videoWidth && + y >= 0 && + y < this.video.videoHeight + ); } // Interaction Handlers @@ -203,7 +208,8 @@ class CamCanvas { this.ctx.save(); this.ctx.translate(this.panX, this.panY); this.ctx.scale(this.scale, this.scale); - this.ctx.drawImage(this.image, 0, 0, width, height); + this.ctx.drawImage(this.video, 0, 0, width, height); + for (const point of this.calibrationPoints) { this.drawPoint( point.x * this.camScaleFactor, @@ -215,10 +221,37 @@ class CamCanvas { this.ctx.restore(); } + startVideoRendering() { + const renderFrame = () => { + if (this.video.readyState >= this.video.HAVE_CURRENT_DATA) { + // Update canvas size based on video dimensions + const aspectRatio = this.video.videoWidth / this.video.videoHeight; + this.camScaleFactor = this.canvas.clientWidth / this.video.videoWidth; + this.calibrationPointSize = + (this.canvas.clientWidth * CALIBRATION_POINT_SCALE) / this.scale; + + let newWidth = this.canvas.clientWidth; + let newHeight = this.canvas.clientWidth / aspectRatio; + + this.drawImage(newWidth, newHeight); + requestAnimationFrame(renderFrame); + } + }; + + this.video.addEventListener("loadedmetadata", () => { + renderFrame(); + }); + + // Start immediately if video is already loaded + if (this.video.readyState >= this.video.HAVE_METADATA) { + renderFrame(); + } + } + handleImageLoad() { // Do resizing and find the new width and height - const aspectRatio = this.image.width / this.image.height; - this.camScaleFactor = this.canvas.clientWidth / this.image.width; + const aspectRatio = this.video.videoWidth / this.video.videoHeight; + this.camScaleFactor = this.canvas.clientWidth / this.video.videoWidth; this.calibrationPointSize = (this.canvas.clientWidth * CALIBRATION_POINT_SCALE) / this.scale; let newWidth = this.canvas.clientWidth; @@ -226,10 +259,6 @@ class CamCanvas { this.drawImage(newWidth, newHeight); } - updateImageSrc(base64Image) { - this.image.src = base64Image; - } - resetCameraView() { this.scale = 1; this.panX = 0; diff --git a/manager/src/static/js/cameracalibrate.js b/manager/src/static/js/cameracalibrate.js index d514f1df4..71695ef2d 100644 --- a/manager/src/static/js/cameracalibrate.js +++ b/manager/src/static/js/cameracalibrate.js @@ -81,8 +81,8 @@ export class ConvergedCameraCalibration { }); } - initializeCamCanvas(canvasElement, imageSrc) { - this.camCanvas = new CamCanvas(canvasElement, imageSrc); + initializeCamCanvas(canvasElement, video) { + this.camCanvas = new CamCanvas(canvasElement, video); // FIXME: Find a better way to do these event listeners which require interacting with both // the camCanvas and viewport this.camCanvas.canvas.addEventListener("mouseup", (event) => { @@ -98,7 +98,7 @@ export class ConvergedCameraCalibration { }); } - initializeViewport(canvas, scale, sceneID, authToken) { + initializeViewport(canvas, scale, sceneID, video, authToken) { const gltfLoader = new GLTFLoader(); const renderer = new THREE.WebGLRenderer({ canvas: canvas, @@ -113,6 +113,7 @@ export class ConvergedCameraCalibration { gltfLoader, renderer, ); + this.video = video; this.viewport = viewport; viewport @@ -138,6 +139,7 @@ export class ConvergedCameraCalibration { }) .then(() => { viewport.initializeEventListeners(); + this.startVideoProjection(); viewport.renderer.domElement.addEventListener("mouseup", (event) => { this.calculateCalibrationIntrinsics(); @@ -267,7 +269,7 @@ export class ConvergedCameraCalibration { fixIntrinsics: fixIntrinsics, intrinsics: intrinsicData, distortion: distortionData, - imageSize: this.camCanvas.getImageSize(), + imageSize: this.camCanvas.getVideoSize(), }; $.ajax({ @@ -549,75 +551,70 @@ export class ConvergedCameraCalibration { } } - undistortAndProjectImage(image, cameraMatrix, distCoeffs) { - this.projectionImage.src = image; - this.projectionImage.onload = () => { - this.projectionCanvas.width = this.projectionImage.width; - this.projectionCanvas.height = this.projectionImage.height; - this.projectionCtx.drawImage(this.projectionImage, 0, 0); - const distortedImage = cv.imread(this.projectionCanvas); - - const h = distortedImage.rows; - const w = distortedImage.cols; - - const map_x = new cv.Mat(); - const map_y = new cv.Mat(); - const cameraMatrixMat = cv.matFromArray( - 3, - 3, - cv.CV_64F, - cameraMatrix.flat(), - ); - const distCoeffsMat = cv.matFromArray(1, 5, cv.CV_64F, distCoeffs.flat()); - // 3x3 identity matrix - const identityMatrix = cv.matFromArray( - 3, - 3, - cv.CV_64F, - [1, 0, 0, 0, 1, 0, 0, 0, 1], - ); - cv.initUndistortRectifyMap( - cameraMatrixMat, - distCoeffsMat, - identityMatrix, - cameraMatrixMat, - new cv.Size(w, h), - 5, - map_x, - map_y, - ); - const undistortedImage = new cv.Mat(); - cv.remap(distortedImage, undistortedImage, map_x, map_y, cv.INTER_LINEAR); - - // Put undistorted image on canvas to use with projection later - const imageData = new ImageData( - new Uint8ClampedArray(undistortedImage.data), - undistortedImage.cols, - undistortedImage.rows, - ); - this.projectionCtx.putImageData(imageData, 0, 0); + undistortAndProjectImage(cameraMatrix, distCoeffs) { + this.projectionCanvas.width = this.video.videoWidth; + this.projectionCanvas.height = this.video.videoHeight; + this.projectionCtx.drawImage(this.video, 0, 0); + const distortedImage = cv.imread(this.projectionCanvas); + + const h = distortedImage.rows; + const w = distortedImage.cols; + + const map_x = new cv.Mat(); + const map_y = new cv.Mat(); + const cameraMatrixMat = cv.matFromArray( + 3, + 3, + cv.CV_64F, + cameraMatrix.flat(), + ); + const distCoeffsMat = cv.matFromArray(1, 5, cv.CV_64F, distCoeffs.flat()); + // 3x3 identity matrix + const identityMatrix = cv.matFromArray( + 3, + 3, + cv.CV_64F, + [1, 0, 0, 0, 1, 0, 0, 0, 1], + ); + cv.initUndistortRectifyMap( + cameraMatrixMat, + distCoeffsMat, + identityMatrix, + cameraMatrixMat, + new cv.Size(w, h), + 5, + map_x, + map_y, + ); + const undistortedImage = new cv.Mat(); + cv.remap(distortedImage, undistortedImage, map_x, map_y, cv.INTER_LINEAR); + + // Put undistorted image on canvas to use with projection later + const imageData = new ImageData( + new Uint8ClampedArray(undistortedImage.data), + undistortedImage.cols, + undistortedImage.rows, + ); + this.projectionCtx.putImageData(imageData, 0, 0); - this.projectImage( - this.projectionCanvas.toDataURL("image/jpeg"), - cameraMatrix, - ); + this.projectImage(cameraMatrix); - distortedImage.delete(); - undistortedImage.delete(); - map_x.delete(); - map_y.delete(); - cameraMatrixMat.delete(); - distCoeffsMat.delete(); - identityMatrix.delete(); - }; + distortedImage.delete(); + undistortedImage.delete(); + map_x.delete(); + map_y.delete(); + cameraMatrixMat.delete(); + distCoeffsMat.delete(); + identityMatrix.delete(); } - projectImage(image, cameraMatrix) { + projectImage(cameraMatrix) { if (this.projectionEnabled === false) { this.viewport.setProjectionVisibility(false); return; } - this.viewport.projectImage(image, cameraMatrix); + + this.viewport.projectImage(cameraMatrix, this.video); } updateCameraOpticalCenter(resolution, cameraMatrix) { @@ -640,14 +637,41 @@ export class ConvergedCameraCalibration { } } - updateCalibrationViews(image, cameraMatrix, distCoeffs) { - this.camCanvas.updateImageSrc(image); - this.updateCameraOpticalCenter(this.camCanvas.getImageSize(), cameraMatrix); - this.getCameraPositionAndRotation(cameraMatrix, distCoeffs); - if (distCoeffs.some((coeff) => coeff !== 0)) { - this.undistortAndProjectImage(image, cameraMatrix, distCoeffs); - } else { - this.projectImage(image, cameraMatrix); + startVideoProjection() { + const renderFrame = () => { + if (this.video.readyState >= this.video.HAVE_CURRENT_DATA) { + // Project the current video frame + const cameraMatrix = [ + [$("#id_intrinsics_fx").val(), 0, $("#id_intrinsics_cx").val()], + [0, $("#id_intrinsics_fy").val(), $("#id_intrinsics_cy").val()], + [0, 0, 1], + ]; + const distCoeffs = [ + $("#id_distortion_k1").val(), + $("#id_distortion_k2").val(), + $("#id_distortion_p1").val(), + $("#id_distortion_p2").val(), + $("#id_distortion_k3").val(), + ]; + this.updateCameraOpticalCenter( + this.camCanvas.getVideoSize(), + cameraMatrix, + ); + this.getCameraPositionAndRotation(cameraMatrix, distCoeffs); + + if (distCoeffs.some((coeff) => coeff !== 0)) { + this.undistortAndProjectImage(cameraMatrix, distCoeffs); + } else { + this.projectImage(cameraMatrix); + } + requestAnimationFrame(renderFrame); + } + }; + this.video.addEventListener("loadedmetadata", () => { + renderFrame(); + }); + if (this.video.readyState >= this.video.HAVE_METADATA) { + renderFrame(); } } } diff --git a/manager/src/static/js/sscape.js b/manager/src/static/js/sscape.js index 7a220f121..047527746 100644 --- a/manager/src/static/js/sscape.js +++ b/manager/src/static/js/sscape.js @@ -29,9 +29,9 @@ import { registerAutoCameraCalibration, manageCalibrationState, initializeCalibrationSettings, - updateCalibrationView, handleAutoCalibrationPose, } from "/static/js/calibration.js"; +import { MediaMTXWebRTCReader } from "/static/js/webrtc_reader.js"; var svgCanvas = Snap("#svgout"); import RESTClient from "/static/js/restclient.js"; @@ -135,28 +135,7 @@ async function checkBrokerConnections() { $("#mqtt_status").addClass("connected"); - // Capture thumbnail snapshots - if ($(".snapshot-image").length) { - client.subscribe(APP_NAME + IMAGE_CAMERA + "+"); - - $(".snapshot-image").each(function () { - client.publish($(this).attr("topic"), "getimage"); - }); - - $("input#live-view").on("change", function () { - if ($(this).is(":checked")) { - $(".snapshot-image").each(function () { - client.publish($(this).attr("topic"), "getimage"); - }); - $("#cameras-tab").click(); // Select the cameras tab - $(".camera-card").addClass("live-view"); - // $(".hide-live").hide(); - } else { - $(".camera-card").removeClass("live-view"); - // $(".hide-live").show(); - } - }); - } else if ($("#auto-camcalibration").length) { + if ($("#auto-camcalibration").length) { var auto_topic = APP_NAME + DATA_AUTOCALIB_CAM_POSE + $("#sensor_id").val(); client.subscribe(auto_topic); @@ -239,31 +218,6 @@ async function checkBrokerConnections() { } } else if (topic.includes("singleton")) { plotSingleton(msg); - } else if (topic.includes(IMAGE_CAMERA)) { - // Use native JS since jQuery.load() pukes on data URI's - if ($(".snapshot-image").length) { - var id = topic.split("camera/")[1]; - - img = document.getElementById(id); - if (img !== undefined && img !== null) { - img.setAttribute("src", "data:image/jpeg;base64," + msg.image); - } - - if ($("input#live-view").is(":checked")) { - client.publish(APP_NAME + CMD_CAMERA + id, "getimage"); - } - - // If ID contains special characters, selector $("#" + id) fails - $("[id='" + id + "']") - .stop() - .show() - .css("opacity", 1) - .animate({ opacity: 0.6 }, 5000, function () {}) - .prevAll(".cam-offline") - .hide(); - } - } else if (topic.includes(IMAGE_CALIBRATE)) { - updateCalibrationView(msg); } else if (topic.includes(DATA_CAMERA)) { var id = topic.slice(topic.lastIndexOf("/") + 1); $("#rate-" + id).text(msg.rate + " FPS"); @@ -340,6 +294,65 @@ async function checkBrokerConnections() { } } +function openWebRTCStream() { + const videos = document.querySelectorAll("video[topic]"); + const readers = []; + + console.log("Setting WebRTC connections..."); + + const loadAttributes = (video) => { + video.controls = false; + video.muted = true; + video.autoplay = true; + video.playsInline = true; + video.disablepictureinpicture = true; + }; + + window.addEventListener("load", () => { + videos.forEach((video) => { + loadAttributes(video); + const offlineClip = video.getAttribute("src"); + const topic = video.getAttribute("topic"); + const reader = new MediaMTXWebRTCReader({ + url: new URL( + "whep", + "https://mediamtx-proxy.scenescape.intel.com" + + ":8443/" + + topic + + "/", + ), + onTrack: (evt) => { + console.log("WebRTC connection established for topic:", topic); + video.srcObject = evt.streams[0]; + }, + onError: (evt) => { + console.log( + "Video error for topic:", + video.getAttribute("topic"), + evt, + ); + // Fallback to offline clip + video.srcObject = null; + if (offlineClip) { + video.setAttribute("src", offlineClip); + video.load(); + video.play().catch(() => {}); + } + }, + }); + readers.push(reader); + }); + }); + + window.addEventListener("beforeunload", () => { + readers.forEach((reader) => { + if (reader !== null) { + reader.close(); + } + }); + }); +} + function plotSingleton(m) { var $sensor = $("#sensor_" + m.id); @@ -1694,6 +1707,8 @@ $(document).ready(function () { setColorForAllROIs(); }); + openWebRTCStream(); + // Operations to take after images are loaded $(".content").imagesLoaded(function () { // Camera calibration interface diff --git a/manager/src/static/js/viewport.js b/manager/src/static/js/viewport.js index 6843211f3..eff4bfa3d 100644 --- a/manager/src/static/js/viewport.js +++ b/manager/src/static/js/viewport.js @@ -42,6 +42,7 @@ class Viewport extends THREE.Scene { this.gltfLoader = gltfLoader; this.renderer = renderer; this.raycaster = new THREE.Raycaster(); + this.texture = null; this.textureLoader = new THREE.TextureLoader(); this.isDragging = false; @@ -315,30 +316,34 @@ class Viewport extends THREE.Scene { // Re-enable the projection if it has been disabled by clearing calibration points } - projectImage(image, cameraMtx) { + projectImage(cameraMtx, video) { + if (!this.texture) { + this.texture = new THREE.VideoTexture(video); + this.texture.minFilter = THREE.LinearFilter; + this.texture.magFilter = THREE.LinearFilter; + this.texture.format = THREE.RGBFormat; + } + if (this.sceneMesh !== null) { - this.textureLoader.load(image, (texture) => { - this.projectionCamera.aspect = - texture.image.width / texture.image.height; - this.projectionCamera.fov = THREE.MathUtils.radToDeg( - 2 * Math.atan(texture.image.height / (2 * cameraMtx[1][1])), - ); - this.projectionCamera.updateProjectionMatrix(); - if (this.projectedMaterial === null) { - [this.projectedMaterial, this.mesh] = - this.drawObject.createProjectionMaterial( - this.projectionCamera, - this.sceneMesh, - texture, - ); - this.projectedMaterial.opacity = this.initialOpacity; - this.add(this.mesh); - } else { - this.projectedMaterial.texture = texture; - this.projectedMaterial.project(this.mesh); - } - this.setProjectionVisibility(true); - }); + this.projectionCamera.aspect = video.videoWidth / video.videoHeight; + this.projectionCamera.fov = THREE.MathUtils.radToDeg( + 2 * Math.atan(video.videoHeight / (2 * cameraMtx[1][1])), + ); + this.projectionCamera.updateProjectionMatrix(); + if (this.projectedMaterial === null) { + [this.projectedMaterial, this.mesh] = + this.drawObject.createProjectionMaterial( + this.projectionCamera, + this.sceneMesh, + this.texture, + ); + this.projectedMaterial.opacity = this.initialOpacity; + this.add(this.mesh); + } else { + this.projectedMaterial.texture = this.texture; + this.projectedMaterial.project(this.mesh); + } + this.setProjectionVisibility(true); } } diff --git a/manager/src/static/js/webrtc_reader.js b/manager/src/static/js/webrtc_reader.js new file mode 100644 index 000000000..49d74128b --- /dev/null +++ b/manager/src/static/js/webrtc_reader.js @@ -0,0 +1,638 @@ +// SPDX-FileCopyrightText: (C) 2019 +// SPDX-FileContributor: aler9 +// SPDX-License-Identifier: MIT + +"use strict"; + +/** + * @callback OnError + * @param {string} err - error. + */ + +/** + * @callback OnTrack + * @param {RTCTrackEvent} evt - track event. + */ + +/** + * @typedef Conf + * @type {object} + * @property {string} url - absolute URL of the WHEP endpoint. + * @property {OnError} onError - called when there's an error. + * @property {OnTrack} onTrack - called when there's a track available. + */ + +/** WebRTC/WHEP reader. */ +class MediaMTXWebRTCReader { + /** + * Create a MediaMTXWebRTCReader. + * @param {Conf} conf - configuration. + */ + constructor(conf) { + this.retryPause = 2000; + this.conf = conf; + this.state = "getting_codecs"; + this.restartTimeout = null; + this.pc = null; + this.offerData = null; + this.sessionUrl = null; + this.queuedCandidates = []; + this.#getNonAdvertisedCodecs(); + } + + /** + * Close the reader and all its resources. + */ + close() { + this.state = "closed"; + + if (this.pc !== null) { + this.pc.close(); + } + + if (this.restartTimeout !== null) { + clearTimeout(this.restartTimeout); + } + } + + static #supportsNonAdvertisedCodec(codec, fmtp) { + return new Promise((resolve) => { + const pc = new RTCPeerConnection({ iceServers: [] }); + const mediaType = "audio"; + let payloadType = ""; + + pc.addTransceiver(mediaType, { direction: "recvonly" }); + pc.createOffer() + .then((offer) => { + if (offer.sdp === undefined) { + throw new Error("SDP not present"); + } + if (offer.sdp.includes(` ${codec}`)) { + // codec is advertised, there's no need to add it manually + throw new Error("already present"); + } + + const sections = offer.sdp.split(`m=${mediaType}`); + + const payloadTypes = sections + .slice(1) + .map((s) => s.split("\r\n")[0].split(" ").slice(3)) + .reduce((prev, cur) => [...prev, ...cur], []); + payloadType = this.#reservePayloadType(payloadTypes); + + const lines = sections[1].split("\r\n"); + lines[0] += ` ${payloadType}`; + lines.splice(lines.length - 1, 0, `a=rtpmap:${payloadType} ${codec}`); + if (fmtp !== undefined) { + lines.splice(lines.length - 1, 0, `a=fmtp:${payloadType} ${fmtp}`); + } + sections[1] = lines.join("\r\n"); + offer.sdp = sections.join(`m=${mediaType}`); + return pc.setLocalDescription(offer); + }) + .then(() => + pc.setRemoteDescription( + new RTCSessionDescription({ + type: "answer", + sdp: + "v=0\r\n" + + "o=- 6539324223450680508 0 IN IP4 0.0.0.0\r\n" + + "s=-\r\n" + + "t=0 0\r\n" + + "a=fingerprint:sha-256 0D:9F:78:15:42:B5:4B:E6:E2:94:3E:5B:37:78:E1:4B:54:59:A3:36:3A:E5:05:EB:27:EE:8F:D2:2D:41:29:25\r\n" + + `m=${mediaType} 9 UDP/TLS/RTP/SAVPF ${payloadType}\r\n` + + "c=IN IP4 0.0.0.0\r\n" + + "a=ice-pwd:7c3bf4770007e7432ee4ea4d697db675\r\n" + + "a=ice-ufrag:29e036dc\r\n" + + "a=sendonly\r\n" + + "a=rtcp-mux\r\n" + + `a=rtpmap:${payloadType} ${codec}\r\n` + + (fmtp !== undefined ? `a=fmtp:${payloadType} ${fmtp}\r\n` : ""), + }), + ), + ) + .then(() => { + resolve(true); + }) + .catch(() => { + resolve(false); + }) + .finally(() => { + pc.close(); + }); + }); + } + + static #unquoteCredential(v) { + return JSON.parse(`"${v}"`); + } + + static #linkToIceServers(links) { + return links !== null + ? links.split(", ").map((link) => { + const m = link.match( + /^<(.+?)>; rel="ice-server"(; username="(.*?)"; credential="(.*?)"; credential-type="password")?/i, + ); + const ret = { + urls: [m[1]], + }; + + if (m[3] !== undefined) { + ret.username = this.#unquoteCredential(m[3]); + ret.credential = this.#unquoteCredential(m[4]); + ret.credentialType = "password"; + } + + return ret; + }) + : []; + } + + static #parseOffer(sdp) { + const ret = { + iceUfrag: "", + icePwd: "", + medias: [], + }; + + for (const line of sdp.split("\r\n")) { + if (line.startsWith("m=")) { + ret.medias.push(line.slice("m=".length)); + } else if (ret.iceUfrag === "" && line.startsWith("a=ice-ufrag:")) { + ret.iceUfrag = line.slice("a=ice-ufrag:".length); + } else if (ret.icePwd === "" && line.startsWith("a=ice-pwd:")) { + ret.icePwd = line.slice("a=ice-pwd:".length); + } + } + + return ret; + } + + static #reservePayloadType(payloadTypes) { + // everything is valid between 30 and 127, except for interval between 64 and 95 + // https://chromium.googlesource.com/external/webrtc/+/refs/heads/master/call/payload_type.h#29 + for (let i = 30; i <= 127; i++) { + if ((i <= 63 || i >= 96) && !payloadTypes.includes(i.toString())) { + const pl = i.toString(); + payloadTypes.push(pl); + return pl; + } + } + throw Error("unable to find a free payload type"); + } + + static #enableStereoPcmau(payloadTypes, section) { + const lines = section.split("\r\n"); + + let payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice(lines.length - 1, 0, `a=rtpmap:${payloadType} PCMU/8000/2`); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice(lines.length - 1, 0, `a=rtpmap:${payloadType} PCMA/8000/2`); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + return lines.join("\r\n"); + } + + static #enableMultichannelOpus(payloadTypes, section) { + const lines = section.split("\r\n"); + + let payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice( + lines.length - 1, + 0, + `a=rtpmap:${payloadType} multiopus/48000/3`, + ); + lines.splice( + lines.length - 1, + 0, + `a=fmtp:${payloadType} channel_mapping=0,2,1;num_streams=2;coupled_streams=1`, + ); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice( + lines.length - 1, + 0, + `a=rtpmap:${payloadType} multiopus/48000/4`, + ); + lines.splice( + lines.length - 1, + 0, + `a=fmtp:${payloadType} channel_mapping=0,1,2,3;num_streams=2;coupled_streams=2`, + ); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice( + lines.length - 1, + 0, + `a=rtpmap:${payloadType} multiopus/48000/5`, + ); + lines.splice( + lines.length - 1, + 0, + `a=fmtp:${payloadType} channel_mapping=0,4,1,2,3;num_streams=3;coupled_streams=2`, + ); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice( + lines.length - 1, + 0, + `a=rtpmap:${payloadType} multiopus/48000/6`, + ); + lines.splice( + lines.length - 1, + 0, + `a=fmtp:${payloadType} channel_mapping=0,4,1,2,3,5;num_streams=4;coupled_streams=2`, + ); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice( + lines.length - 1, + 0, + `a=rtpmap:${payloadType} multiopus/48000/7`, + ); + lines.splice( + lines.length - 1, + 0, + `a=fmtp:${payloadType} channel_mapping=0,4,1,2,3,5,6;num_streams=4;coupled_streams=4`, + ); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice( + lines.length - 1, + 0, + `a=rtpmap:${payloadType} multiopus/48000/8`, + ); + lines.splice( + lines.length - 1, + 0, + `a=fmtp:${payloadType} channel_mapping=0,6,1,4,5,2,3,7;num_streams=5;coupled_streams=4`, + ); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + return lines.join("\r\n"); + } + + static #enableL16(payloadTypes, section) { + const lines = section.split("\r\n"); + + let payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice(lines.length - 1, 0, `a=rtpmap:${payloadType} L16/8000/2`); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice(lines.length - 1, 0, `a=rtpmap:${payloadType} L16/16000/2`); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + payloadType = this.#reservePayloadType(payloadTypes); + lines[0] += ` ${payloadType}`; + lines.splice(lines.length - 1, 0, `a=rtpmap:${payloadType} L16/48000/2`); + lines.splice(lines.length - 1, 0, `a=rtcp-fb:${payloadType} transport-cc`); + + return lines.join("\r\n"); + } + + static #enableStereoOpus(section) { + let opusPayloadFormat = ""; + const lines = section.split("\r\n"); + + for (let i = 0; i < lines.length; i++) { + if ( + lines[i].startsWith("a=rtpmap:") && + lines[i].toLowerCase().includes("opus/") + ) { + opusPayloadFormat = lines[i].slice("a=rtpmap:".length).split(" ")[0]; + break; + } + } + + if (opusPayloadFormat === "") { + return section; + } + + for (let i = 0; i < lines.length; i++) { + if (lines[i].startsWith(`a=fmtp:${opusPayloadFormat} `)) { + if (!lines[i].includes("stereo")) { + lines[i] += ";stereo=1"; + } + if (!lines[i].includes("sprop-stereo")) { + lines[i] += ";sprop-stereo=1"; + } + } + } + + return lines.join("\r\n"); + } + + static #editOffer(sdp, nonAdvertisedCodecs) { + const sections = sdp.split("m="); + + const payloadTypes = sections + .slice(1) + .map((s) => s.split("\r\n")[0].split(" ").slice(3)) + .reduce((prev, cur) => [...prev, ...cur], []); + + for (let i = 1; i < sections.length; i++) { + if (sections[i].startsWith("audio")) { + sections[i] = this.#enableStereoOpus(sections[i]); + + if (nonAdvertisedCodecs.includes("pcma/8000/2")) { + sections[i] = this.#enableStereoPcmau(payloadTypes, sections[i]); + } + if (nonAdvertisedCodecs.includes("multiopus/48000/6")) { + sections[i] = this.#enableMultichannelOpus(payloadTypes, sections[i]); + } + if (nonAdvertisedCodecs.includes("L16/48000/2")) { + sections[i] = this.#enableL16(payloadTypes, sections[i]); + } + + break; + } + } + + return sections.join("m="); + } + + static #generateSdpFragment(od, candidates) { + const candidatesByMedia = {}; + for (const candidate of candidates) { + const mid = candidate.sdpMLineIndex; + if (candidatesByMedia[mid] === undefined) { + candidatesByMedia[mid] = []; + } + candidatesByMedia[mid].push(candidate); + } + + let frag = `a=ice-ufrag:${od.iceUfrag}\r\n` + `a=ice-pwd:${od.icePwd}\r\n`; + + let mid = 0; + + for (const media of od.medias) { + if (candidatesByMedia[mid] !== undefined) { + frag += `m=${media}\r\n` + `a=mid:${mid}\r\n`; + + for (const candidate of candidatesByMedia[mid]) { + frag += `a=${candidate.candidate}\r\n`; + } + } + mid++; + } + + return frag; + } + + #handleError(err) { + if (this.state === "running") { + if (this.pc !== null) { + this.pc.close(); + this.pc = null; + } + + this.offerData = null; + + if (this.sessionUrl !== null) { + fetch(this.sessionUrl, { + method: "DELETE", + }); + this.sessionUrl = null; + } + + this.queuedCandidates = []; + this.state = "restarting"; + + this.restartTimeout = window.setTimeout(() => { + this.restartTimeout = null; + this.state = "running"; + this.#start(); + }, this.retryPause); + + if (this.conf.onError !== undefined) { + this.conf.onError(`${err}, retrying in some seconds`); + } + } else if (this.state === "getting_codecs") { + this.state = "failed"; + + if (this.conf.onError !== undefined) { + this.conf.onError(err); + } + } + } + + #getNonAdvertisedCodecs() { + Promise.all( + [ + ["pcma/8000/2"], + [ + "multiopus/48000/6", + "channel_mapping=0,4,1,2,3,5;num_streams=4;coupled_streams=2", + ], + ["L16/48000/2"], + ].map((c) => + MediaMTXWebRTCReader.#supportsNonAdvertisedCodec(c[0], c[1]).then( + (r) => (r ? c[0] : false), + ), + ), + ) + .then((c) => c.filter((e) => e !== false)) + .then((codecs) => { + if (this.state !== "getting_codecs") { + throw new Error("closed"); + } + + this.nonAdvertisedCodecs = codecs; + this.state = "running"; + this.#start(); + }) + .catch((err) => { + this.#handleError(err); + }); + } + + #start() { + this.#requestICEServers() + .then((iceServers) => this.#setupPeerConnection(iceServers)) + .then((offer) => this.#sendOffer(offer)) + .then((answer) => this.#setAnswer(answer)) + .catch((err) => { + this.#handleError(err.toString()); + }); + } + + #requestICEServers() { + return fetch(this.conf.url, { + method: "OPTIONS", + }).then((res) => + MediaMTXWebRTCReader.#linkToIceServers(res.headers.get("Link")), + ); + } + + #setupPeerConnection(iceServers) { + if (this.state !== "running") { + throw new Error("closed"); + } + + this.pc = new RTCPeerConnection({ + iceServers, + // https://webrtc.org/getting-started/unified-plan-transition-guide + sdpSemantics: "unified-plan", + }); + + const direction = "recvonly"; + this.pc.addTransceiver("video", { direction }); + //this.pc.addTransceiver('audio', { direction }); + + this.pc.onicecandidate = (evt) => this.#onLocalCandidate(evt); + this.pc.onconnectionstatechange = () => this.#onConnectionState(); + this.pc.ontrack = (evt) => this.#onTrack(evt); + + return this.pc.createOffer().then((offer) => { + offer.sdp = MediaMTXWebRTCReader.#editOffer( + offer.sdp, + this.nonAdvertisedCodecs, + ); + this.offerData = MediaMTXWebRTCReader.#parseOffer(offer.sdp); + + return this.pc.setLocalDescription(offer).then(() => offer.sdp); + }); + } + + #sendOffer(offer) { + if (this.state !== "running") { + throw new Error("closed"); + } + + return fetch(this.conf.url, { + method: "POST", + headers: { "Content-Type": "application/sdp" }, + body: offer, + }).then((res) => { + switch (res.status) { + case 201: + break; + case 404: + throw new Error("stream not found"); + case 400: + return res.json().then((e) => { + throw new Error(e.error); + }); + default: + throw new Error(`bad status code ${res.status}`); + } + + this.sessionUrl = new URL( + res.headers.get("location"), + this.conf.url, + ).toString(); + + return res.text(); + }); + } + + #setAnswer(answer) { + if (this.state !== "running") { + throw new Error("closed"); + } + + return this.pc + .setRemoteDescription( + new RTCSessionDescription({ + type: "answer", + sdp: answer, + }), + ) + .then(() => { + if (this.state !== "running") { + return; + } + + if (this.queuedCandidates.length !== 0) { + this.#sendLocalCandidates(this.queuedCandidates); + this.queuedCandidates = []; + } + }); + } + + #onLocalCandidate(evt) { + if (this.state !== "running") { + return; + } + + if (evt.candidate !== null) { + if (this.sessionUrl === null) { + this.queuedCandidates.push(evt.candidate); + } else { + this.#sendLocalCandidates([evt.candidate]); + } + } + } + + #sendLocalCandidates(candidates) { + fetch(this.sessionUrl, { + method: "PATCH", + headers: { + "Content-Type": "application/trickle-ice-sdpfrag", + "If-Match": "*", + }, + body: MediaMTXWebRTCReader.#generateSdpFragment( + this.offerData, + candidates, + ), + }) + .then((res) => { + switch (res.status) { + case 204: + break; + case 404: + throw new Error("stream not found"); + default: + throw new Error(`bad status code ${res.status}`); + } + }) + .catch((err) => { + this.#handleError(err.toString()); + }); + } + + #onConnectionState() { + if (this.state !== "running") { + return; + } + + // "closed" can arrive before "failed" and without + // the close() method being called at all. + // It happens when the other peer sends a termination + // message like a DTLS CloseNotify. + if ( + this.pc.connectionState === "failed" || + this.pc.connectionState === "closed" + ) { + this.#handleError("peer connection closed"); + } + } + + #onTrack(evt) { + if (this.conf.onTrack !== undefined) { + this.conf.onTrack(evt); + } + } +} + +window.MediaMTXWebRTCReader = MediaMTXWebRTCReader; + +export { MediaMTXWebRTCReader }; diff --git a/manager/src/static/videos/loading.mp4 b/manager/src/static/videos/loading.mp4 new file mode 100644 index 000000000..9a1e8538f Binary files /dev/null and b/manager/src/static/videos/loading.mp4 differ diff --git a/manager/src/templates/cam/cam_calibrate.html b/manager/src/templates/cam/cam_calibrate.html index 606609eb9..5b13410ca 100644 --- a/manager/src/templates/cam/cam_calibrate.html +++ b/manager/src/templates/cam/cam_calibrate.html @@ -99,14 +99,7 @@
{% endif %}
- Camera Offline - {% if caminst.scene.thumbnail %} {% else %} {% endif %} +
diff --git a/manager/src/templates/sscape/sceneDetail.html b/manager/src/templates/sscape/sceneDetail.html index 40643d4b4..13ca0a1d3 100644 --- a/manager/src/templates/sscape/sceneDetail.html +++ b/manager/src/templates/sscape/sceneDetail.html @@ -130,21 +130,6 @@

{{ scene.name }}

-
- - -
{% if user.is_superuser %} -
Camera Offline
- + {% else %} - -
Camera Offline
- + + {% endif %}
diff --git a/sample_data/docker-compose-dl-streamer-example.yml b/sample_data/docker-compose-dl-streamer-example.yml index b2d81138f..6e132b9fd 100644 --- a/sample_data/docker-compose-dl-streamer-example.yml +++ b/sample_data/docker-compose-dl-streamer-example.yml @@ -7,6 +7,7 @@ name: scenescape networks: scenescape: + secrets: root-cert: file: ${SECRETSDIR}/certs/scenescape-ca.pem @@ -22,6 +23,10 @@ secrets: file: ${SECRETSDIR}/certs/scenescape-vdms-s.crt vdms-server-key: file: ${SECRETSDIR}/certs/scenescape-vdms-s.key + mediamtx-key: + file: ${SECRETSDIR}/certs/scenescape-mediamtx.key + mediamtx-cert: + file: ${SECRETSDIR}/certs/scenescape-mediamtx.crt django: file: ${SECRETSDIR}/django controller.auth: @@ -36,8 +41,8 @@ services: image: dockurr/chrony networks: scenescape: - # ports: - # - 123:123/udp + # ports: + # - 123:123/udp restart: on-failure pids_limit: 1000 @@ -57,6 +62,75 @@ services: user: "${UID:-1000}:${GID:-1000}" pids_limit: 1000 + mediaserver: + image: bluenviron/mediamtx:1.14.0 + container_name: mediamtx-server + ports: + - "8889:8889" + - "8889:8889/udp" + - "8189:8189" + - "9997:9997" + - "8554:8554" + configs: + - source: mediamtx-config + target: /mediamtx.yml + secrets: + - source: root-cert + target: /usr/local/share/ca-certificates/scenescape-ca.pem + - source: mediamtx-cert + target: certs/scenescape-mediamtx.crt + - source: mediamtx-key + target: certs/scenescape-mediamtx.key + networks: + scenescape: + aliases: + - mediamtx.scenescape.intel.com + - mediaserver + pids_limit: 1000 + + nginx: + image: nginx:alpine + container_name: nginx-mediamtx + ports: + - "8443:443" + configs: + - source: nginx-config + target: /etc/nginx/nginx.conf + depends_on: + - mediaserver + networks: + scenescape: + aliases: + - mediamtx-proxy.scenescape.intel.com + secrets: + - source: root-cert + target: /usr/local/share/ca-certificates/scenescape-ca.pem + - source: mediamtx-cert + target: /etc/nginx/certs/cert.crt + - source: mediamtx-key + target: /etc/nginx/certs/cert.key + pids_limit: 1000 + + coturn: + image: coturn/coturn:latest + container_name: coturn + ports: + - "3478:3478" + - "3478:3478/udp" + command: [ "-v" ] # Verbose mode for logging + environment: + - TURN_REALM=coturn.scenescape.intel.com + - TURN_USER=user + - TURN_PASSWORD=password + networks: + scenescape: + aliases: + - coturn.scenescape.intel.com + secrets: + - source: root-cert + target: certs/scenescape-ca.pem + pids_limit: 1000 + pgserver: image: postgres:17.6 environment: @@ -82,13 +156,7 @@ services: ports: - "443:443" command: > - webserver - --dbhost pgserver - --dbtype postgres - --dbport 5432 - --broker broker.scenescape.intel.com - --brokerauth /run/secrets/browser.auth - --brokerrootcert /run/secrets/certs/scenescape-ca.pem + webserver --dbhost pgserver --dbtype postgres --dbport 5432 --broker broker.scenescape.intel.com --brokerauth /run/secrets/browser.auth --brokerrootcert /run/secrets/certs/scenescape-ca.pem healthcheck: test: "curl --insecure -X GET https://web.scenescape.intel.com:443/api/v1/database-ready | grep 'true'" interval: 10s @@ -138,10 +206,7 @@ services: # vdms: # condition: service_started command: > - --restauth /run/secrets/controller.auth - --brokerauth /run/secrets/controller.auth - --broker broker.scenescape.intel.com - --ntp ntpserv + --restauth /run/secrets/controller.auth --brokerauth /run/secrets/controller.auth --broker broker.scenescape.intel.com --ntp ntpserv # mount the trackerconfig file to the container configs: - source: tracker-config @@ -181,21 +246,9 @@ services: # target: certs/scenescape-vdms-s.key # restart: always # pids_limit: 1000 - - mediaserver: - image: bluenviron/mediamtx:1.14.0 - networks: - scenescape: - aliases: - - mediaserver - # ports: - # - "8554:8554" - restart: always - pids_limit: 1000 - queuing-cams: image: linuxserver/ffmpeg:version-8.0-cli - command: -nostdin -re -stream_loop -1 -i /workspace/media/qcam1.ts -re -stream_loop -1 -i /workspace/media/qcam2.ts -map 0:v -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/queuing-cam1 -map 1:v -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/queuing-cam2 + command: -nostdin -re -stream_loop -1 -i /workspace/media/qcam1.ts -re -stream_loop -1 -i /workspace/media/qcam2.ts -map 0:v -an -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/atag-qcam1-raw -map 1:v -an -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/atag-qcam2-raw volumes: - vol-sample-data:/workspace/media networks: @@ -204,10 +257,9 @@ services: - mediaserver restart: always pids_limit: 1000 - retail-cams: image: linuxserver/ffmpeg:version-8.0-cli - command: -nostdin -re -stream_loop -1 -i /workspace/media/apriltag-cam1.ts -re -stream_loop -1 -i /workspace/media/apriltag-cam2.ts -map 0:v -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/retail-cam1 -map 1:v -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/retail-cam2 + command: -nostdin -re -stream_loop -1 -i /workspace/media/apriltag-cam1.ts -re -stream_loop -1 -i /workspace/media/apriltag-cam2.ts -map 0:v -an -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/camera1-raw -map 1:v -an -c copy -f rtsp -rtsp_transport tcp rtsp://mediaserver:8554/camera2-raw volumes: - vol-sample-data:/workspace/media networks: @@ -222,7 +274,7 @@ services: networks: scenescape: tty: true - entrypoint: ["./run.sh"] + entrypoint: [ "./run.sh" ] ports: - "8080:8080" devices: @@ -243,7 +295,7 @@ services: retail-cams: condition: service_started healthcheck: - test: ["CMD", "curl", "-I", "-s", "http://localhost:8080/pipelines"] + test: [ "CMD", "curl", "-I", "-s", "http://localhost:8080/pipelines" ] interval: 10s timeout: 5s retries: 5 @@ -259,6 +311,8 @@ services: - MQTT_HOST=broker.scenescape.intel.com - MQTT_PORT=1883 - REST_SERVER_PORT=8080 + - ENABLE_WEBRTC=true + - WEBRTC_SIGNALING_SERVER=http://mediamtx.scenescape.intel.com:8889 configs: - source: retail-config target: /home/pipeline-server/config.json @@ -277,7 +331,7 @@ services: networks: scenescape: tty: true - entrypoint: ["./run.sh"] + entrypoint: [ "./run.sh" ] ports: - "8081:8080" devices: @@ -298,7 +352,7 @@ services: queuing-cams: condition: service_started healthcheck: - test: ["CMD", "curl", "-I", "-s", "http://localhost:8080/pipelines"] + test: [ "CMD", "curl", "-I", "-s", "http://localhost:8080/pipelines" ] interval: 10s timeout: 5s retries: 5 @@ -314,6 +368,8 @@ services: - MQTT_HOST=broker.scenescape.intel.com - MQTT_PORT=1883 - REST_SERVER_PORT=8080 + - ENABLE_WEBRTC=true + - WEBRTC_SIGNALING_SERVER=http://mediamtx.scenescape.intel.com:8889 configs: - source: queuing-config target: /home/pipeline-server/config.json @@ -369,6 +425,12 @@ configs: file: ./dlstreamer-pipeline-server/retail-config.json queuing-config: file: ./dlstreamer-pipeline-server/queuing-config.json + person-detection-queing-config: + file: ./dlstreamer-pipeline-server/model-proc-files/person-detection-retail-0013.json + mediamtx-config: + file: ./sample_data/mediamtx.yml + nginx-config: + file: ./sample_data/nginx.conf volumes: vol-db: diff --git a/sample_data/docker-compose-dls-perf.yml b/sample_data/docker-compose-dls-perf.yml index 776eb307c..f56eff88b 100644 --- a/sample_data/docker-compose-dls-perf.yml +++ b/sample_data/docker-compose-dls-perf.yml @@ -22,6 +22,10 @@ secrets: file: ${SECRETSDIR}/certs/scenescape-vdms-s.crt vdms-server-key: file: ${SECRETSDIR}/certs/scenescape-vdms-s.key + mediamtx-key: + file: ${SECRETSDIR}/certs/scenescape-mediamtx.key + mediamtx-cert: + file: ${SECRETSDIR}/certs/scenescape-mediamtx.crt django: file: ${SECRETSDIR}/django controller.auth: @@ -36,8 +40,8 @@ services: image: dockurr/chrony networks: scenescape: - # ports: - # - 123:123/udp + # ports: + # - 123:123/udp restart: on-failure broker: @@ -53,6 +57,76 @@ services: - broker.scenescape.intel.com user: "${UID:-1000}:${GID:-1000}" + mediaserver: + image: bluenviron/mediamtx:1.14.0 + container_name: mediamtx-server + ports: + - "8889:8889" + - "8889:8889/udp" + - "8189:8189" + - "9997:9997" + - "8554:8554" + configs: + - source: mediamtx-config + target: /mediamtx.yml + volumes: + - ./sample_data/mediamtx.yml:/mediamtx.yml + secrets: + - source: root-cert + target: /usr/local/share/ca-certificates/scenescape-ca.pem + - source: mediamtx-cert + target: certs/scenescape-mediamtx.crt + - source: mediamtx-key + target: certs/scenescape-mediamtx.key + networks: + scenescape: + aliases: + - mediamtx.scenescape.intel.com + - mediaserver + pids_limit: "${UID:-1000}:${GID:-1000}" + + nginx: + image: nginx:alpine + container_name: nginx-mediamtx + ports: + - "8443:443" + volumes: + - ./sample_data/nginx.conf:/etc/nginx/nginx.conf + depends_on: + - mediaserver + networks: + scenescape: + aliases: + - mediamtx-proxy.scenescape.intel.com + secrets: + - source: root-cert + target: /usr/local/share/ca-certificates/scenescape-ca.pem + - source: mediamtx-cert + target: /etc/nginx/certs/cert.crt + - source: mediamtx-key + target: /etc/nginx/certs/cert.key + pids_limit: "${UID:-1000}:${GID:-1000}" + + coturn: + image: coturn/coturn:latest + container_name: coturn + ports: + - "3478:3478" + - "3478:3478/udp" + command: ["-v"] # Verbose mode for logging + environment: + - TURN_REALM=coturn.scenescape.intel.com + - TURN_USER=user + - TURN_PASSWORD=password + networks: + scenescape: + aliases: + - coturn.scenescape.intel.com + secrets: + - source: root-cert + target: certs/scenescape-ca.pem + pids_limit: "${UID:-1000}:${GID:-1000}" + pgserver: image: scenescape-manager:${VERSION:-latest} init: true @@ -133,7 +207,7 @@ services: condition: service_started # vdms: # condition: service_started - # mount the trackerconfig file to the container + # mount the trackerconfig file to the container command: --restauth /run/secrets/controller.auth --brokerauth /run/secrets/controller.auth --broker broker.scenescape.intel.com --ntp ntpserv volumes: - vol-media:/home/scenescape/SceneScape/media @@ -205,6 +279,8 @@ services: - APPEND_PIPELINE_NAME_TO_PUBLISHER_TOPIC=false - MQTT_HOST=broker.scenescape.intel.com - MQTT_PORT=1883 + - ENABLE_WEBRTC=true + - WEBRTC_SIGNALING_SERVER=http://mediamtx.scenescape.intel.com:8889 volumes: - ./dlstreamer-pipeline-server/perf-config.json:/home/pipeline-server/config.json - ./dlstreamer-pipeline-server/user_scripts:/home/pipeline-server/user_scripts diff --git a/sample_data/mediamtx.yml b/sample_data/mediamtx.yml new file mode 100644 index 000000000..a236163da --- /dev/null +++ b/sample_data/mediamtx.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +logLevel: info +api: yes + +# RTSP +rtsp: yes + +# WebRTC +webrtc: yes +webrtcAddress: :8889 +webrtcTrackGatherTimeout: 2s +webrtcLocalTCPAddress: :8189 +webrtcICEServers2: + - url: turn:coturn.scenescape.intel.com:3478 + username: user + password: password + clientOnly: true +webrtcAdditionalHosts: + - mediamtx.scenescape.intel.com + - mediamtx-proxy.scenescape.intel.com + +paths: + all_others: + +# Turn off other protocols +rtmp: no +srt: no +hls: no diff --git a/sample_data/nginx.conf b/sample_data/nginx.conf new file mode 100644 index 000000000..1337d818d --- /dev/null +++ b/sample_data/nginx.conf @@ -0,0 +1,25 @@ +events {} + +http { + server { + listen 443 ssl; + server_name mediamtx-proxy.scenescape.intel.com; + + ssl_certificate /etc/nginx/certs/cert.crt; + ssl_certificate_key /etc/nginx/certs/cert.key; + + # WebRTC and HTTP proxy + location / { + proxy_pass http://mediamtx.scenescape.intel.com:8889; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # For WebRTC/WHIP, upgrade headers if needed + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + } +} \ No newline at end of file diff --git a/tools/certificates/Makefile b/tools/certificates/Makefile index 746799285..7b218b9f7 100644 --- a/tools/certificates/Makefile +++ b/tools/certificates/Makefile @@ -12,15 +12,15 @@ default: deploy-certificates #.PHONY targets won't be skipped if files with these names happen to exist .PHONY: default ca deploy-certificates deploy-csr -.PHONY: broker-cert web-cert vdms-c-cert vdms-s-cert -.PHONY: broker-csr web-csr vdms-c-csr vdms-s-csr +.PHONY: broker-cert web-cert vdms-c-cert vdms-s-cert mediamtx-cert +.PHONY: broker-csr web-csr vdms-c-csr vdms-s-csr mediamtx-csr #.SECONDARY prevents make from deleting files created by intermediate targets .SECONDARY: -deploy-certificates: ca broker-cert web-cert vdms-c-cert vdms-s-cert +deploy-certificates: ca broker-cert web-cert vdms-c-cert vdms-s-cert mediamtx-cert -deploy-csr: broker-csr web-csr vdms-c-csr vdms-s-csr +deploy-csr: broker-csr web-csr vdms-c-csr vdms-s-csr mediamtx-csr broker-cert: HOST=broker broker-cert: KEY_USAGE=serverAuth @@ -46,6 +46,12 @@ vdms-s-cert: \ $(SECRETSDIR)/certs/scenescape-vdms-s.key \ $(SECRETSDIR)/certs/scenescape-vdms-s.crt \ +mediamtx-cert: HOST=mediamtx-proxy +mediamtx-cert: KEY_USAGE=serverAuth +mediamtx-cert: \ + $(SECRETSDIR)/certs/scenescape-mediamtx.key \ + $(SECRETSDIR)/certs/scenescape-mediamtx.crt \ + broker-csr: HOST=broker broker-csr: KEY_USAGE=serverAuth broker-csr: \ @@ -70,6 +76,12 @@ vdms-s-csr: \ $(SECRETSDIR)/certs/scenescape-vdms-s.key \ $(SECRETSDIR)/certs/scenescape-vdms-s.csr \ +mediamtx-csr: HOST=mediamtx-proxy +mediamtx-csr: KEY_USAGE=serverAuth +mediamtx-csr: \ + $(SECRETSDIR)/certs/scenescape-mediamtx.key \ + $(SECRETSDIR)/certs/scenescape-mediamtx.csr \ + ca: $(CASECRETSDIR)/certs/scenescape-ca.pem $(CASECRETSDIR)/ca/scenescape-ca.key: @@ -85,7 +97,8 @@ $(CASECRETSDIR)/certs/scenescape-ca.pem: $(CASECRETSDIR)/ca/scenescape-ca.key @if [ -n "$(CERTPASS)" ] ; then PASSFLAGS="-passin pass:$(CERTPASS)" ; fi ; \ openssl req $${PASSFLAGS} -x509 -new -key $< -days 1825 -out $@ \ -subj "/CN=ca.$(CERTDOMAIN)" - @chmod 0644 $@ + openssl x509 -outform der -in $@ -out $(CASECRETSDIR)/certs/scenescape-ca.crt + @chmod 0644 $@ $(CASECRETSDIR)/certs/scenescape-ca.crt $(SECRETSDIR)/certs/%.key: @echo Generating $*.key