user-story/1079-run-and-develop-otvision-in-docker-container

.github/workflows/cicd.yml

@@ -8,6 +8,12 @@ on:
   push:
     branches:
       - main
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  OTVISION_VERSION: 0.1.3
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -46,3 +52,25 @@ jobs:
           mypy .
       - name: Test with pytest
         run: pytest --cov=OTVision tests/
+  deploy:
+    runs-on: ubuntu-latest
+    needs: test
+    strategy:
+      matrix:
+        package: ["detect", "track"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile.${{ matrix.package }}
+          push: true
+          tags: "${{ env.REGISTRY }}/opentrafficcam/otvision:\
+            ${{ env.OTVISION_VERSION }}-${{ matrix.package }}"

.gitignore

@@ -5,6 +5,9 @@ obj_train_data/*
 *ffmpeg.exe
 *ffmpeg.zip
 tests/resources/
+data
+models
+config
 *.log
 !test.log
 

.vscode/launch.json

@@ -27,17 +27,13 @@
             "args": [
                 "--path",
                 ".tmp/debug/Testvideo_Cars-Cyclist_FR20_2020-01-01_00-00-00.mp4",
-                "--filetypes",
-                "mp4",
                 "--weights",
                 "yolov5s"
             ],
             "windows": {
                 "args": [
                     "--path",
                     ".tmp\\debug\\Testvideo_Cars-Cyclist_FR20_2020-01-01_00-00-00.mp4",
-                    "--filetypes",
-                    "mp4",
                     "--weights",
                     "yolov5s"
                 ]

Dockerfile.detect

@@ -0,0 +1,10 @@
+# Use ubuntu as base image instead of a python image, because the rest of the layers
+# will be the same in the cuda container
+FROM pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime
+WORKDIR /OpenTrafficCam/OTVision
+COPY requirements_detect.txt requirements_detect.txt
+RUN pip install --no-cache-dir -r requirements_detect.txt
+ENV PYTHONPATH /OpenTrafficCam/OTVision
+COPY OTVision OTVision
+COPY detect.py .
+ENTRYPOINT [ "python3.10", "detect.py", "--config", "config/user_config.otvision.yaml" ]

Dockerfile.track

@@ -0,0 +1,11 @@
+# Use ubuntu as base image instead of a python image, because the rest of the layers
+# will be the same in the cuda container
+FROM python:3.10-bullseye
+WORKDIR /OpenTrafficCam/OTVision
+COPY requirements_track.txt requirements_track.txt
+RUN pip3.10 install --no-cache-dir -r requirements_track.txt
+
+ENV PYTHONPATH /OpenTrafficCam/OTVision
+COPY OTVision OTVision
+COPY track.py .
+ENTRYPOINT [ "python3.10", "track.py", "--config", "config/user_config.otvision.yaml" ]

OTVision/__init__.py

@@ -17,14 +17,3 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-
-# TODO: Might need to change this
-from .convert.convert import main as convert
-from .detect.detect import main as detect
-from .track.track import main as track
-from .transform.transform import main as transform
-
-# from .view.view import main as view
-
-__all__: list = ["detect", "track", "convert", "transform"]

OTVision/view/view.py

@@ -22,7 +22,7 @@
 import tkinter as tk
 
 from OTVision.config import CONFIG, PAD
-from OTVision.helpers.machine import ON_LINUX, ON_WINDOWS
+from OTVision.helpers.machine import ON_WINDOWS
 from OTVision.view.view_convert import FrameConvert
 from OTVision.view.view_detect import FrameDetect
 from OTVision.view.view_helpers import FrameFileTree, FrameRunChained
@@ -40,10 +40,10 @@ def __init__(self, **kwargs):
             self.iconbitmap(CONFIG["GUI"]["OTC ICON"])
         self.set_layout()
         self.minsize(900, 620)
-        if ON_LINUX:
-            self.state("normal")
-        else:
+        try:
             self.state("zoomed")
+        except tk.TclError:
+            self.attributes("-zoomed", True)
 
     def set_layout(self):
         for col in range(3):

README.md

@@ -51,6 +51,62 @@ make PY_VERSION=3.10 run
 make PY_VERSION=3.10 install
 ```
 
+## Docker and container
+
+In the long run it is planned to ship modules of OTVision as dedicated containers. Allowing users to configure the container input (mount data storage and config file) and run OTVision in a container environment. The container contains all necessary dependencies. There is no need to install anything more than the container engine. Therefore, two docker files are prepared.
+
+The container were tested on a slurm cluster. Unfortunately, the container for detection grow extremly in size (about 5GB). Therefore, the approach was set on hold. For the next steps, the base images of the containers should be checked. nvidia, pytorch and yolo provide base containers to use for ml tasks. One of them could be a good start.
+
+### Dockerfile.detect
+
+`Dockerfile.detect` runs OTVisions detection. The following bash-Script shows the usage of the container. The name of the container and its version are parameters of the script together with the config for OTVision to run detection.
+
+```bash
+#!/bin/sh
+
+if [ 3 -gt $# ]; then
+    echo "Please provide a container name and a version and a config name!"
+    exit 1
+fi
+
+host_input_dir="/scratch/ws/1/labr704e-p_trafficcam_t_30"
+base_container="/OpenTrafficCam/OTVision"
+home_dir=$(realpath ~)
+container_folder=/projects/p_trafficcam/container
+container_name=$1
+container_version=$2
+config_name=$3
+execution_path=${container_folder}/"${container_name}"-"${container_version}".sif
+
+singularity run --nv --contain \
+-B "${home_dir}" \
+--bind "${host_input_dir}"/config/"${config_name}":"${base_container}"/config/user_config.otvision.yaml:ro \
+--bind "${host_input_dir}"/data:"${base_container}"/data \
+--bind "${host_input_dir}"/models:"${base_container}"/models \
+--pwd ${base_container}/ \
+--env PYTHONPATH=${base_container} \
+"${execution_path}"
+```
+
+### Dockerfile.track
+
+`Dockerfile.track` runs OTVisions tracking. The following bash-Script shows the usage of the container.
+
+```bash
+#!/bin/sh
+
+otvision_dir=../OTVision
+host_input_dir=$(realpath ${otvision_dir})
+base_container="/OpenTrafficCam/OTVision"
+config_name="user_config.otvision.miovision.yaml"
+
+docker run -it \
+--mount type=bind,source="${host_input_dir}"/models,target=${base_container}/models \
+--mount type=bind,source="${host_input_dir}"/config/${config_name},target=${base_container}/config/user_config.otvision.yaml,readonly \
+--mount type=bind,source="${host_input_dir}"/data,target=${base_container}/data \
+ otvision:0.1.3-track
+ ```
+
 ## Development
 
 For development please install also the ```requirements_dev.txt``` (and use flake8 for linting and black for autoformatting with line length 88).

convert.py

@@ -23,8 +23,8 @@
 import logging
 from pathlib import Path
 
-import OTVision
 import OTVision.config as config
+from OTVision.convert.convert import main as convert
 from OTVision.helpers.files import check_if_all_paths_exist
 from OTVision.helpers.log import LOGGER_NAME, VALID_LOG_LEVELS, log
 
@@ -193,7 +193,7 @@ def main(argv: list[str] | None = None) -> None:  # sourcery skip: assign-if-exp
     log.info(f"Arguments: {vars(args)}")
 
     try:
-        OTVision.convert(
+        convert(
             paths=paths,
             input_fps=input_fps,
             fps_from_filename=fps_from_filename,

detect.py

@@ -23,8 +23,8 @@
 import logging
 from pathlib import Path
 
-import OTVision
 import OTVision.config as config
+from OTVision.detect.detect import main as detect
 from OTVision.helpers.files import check_if_all_paths_exist
 from OTVision.helpers.log import LOGGER_NAME, VALID_LOG_LEVELS, log
 
@@ -257,7 +257,7 @@ def main(argv: list[str] | None = None) -> None:  # sourcery skip: assign-if-exp
     log.info(f"Arguments: {vars(args)}")
 
     try:
-        OTVision.detect(
+        detect(
             paths=paths,
             weights=weights,
             conf=conf,

hadolint.yaml

@@ -0,0 +1,2 @@
+ignored:
+  - DL3008

requirements.txt

@@ -1,3 +1,5 @@
+-r requirements_detect.txt
+-r requirements_track.txt
 geopandas==0.12.1
 IPython==8.10.0
 matplotlib==3.6.2

requirements_detect.txt

@@ -0,0 +1,17 @@
+# geopandas==0.12.1
+IPython==8.10.0
+matplotlib==3.6.2
+moviepy==1.0.3
+# numpy==1.23.5
+opencv-python==4.6.0.66
+pandas==1.5.2
+# progressbar==2.5
+psutil==5.9.4
+# pyogrio==0.5.0
+PyYAML==5.4.1
+seaborn==0.12.1
+# Follow instructions at https://pytorch.org to install torch with CUDA
+torch==1.13.1 #  NOTE: Omit if using torch with CUDA and install manually
+torchvision==0.14.1 # NOTE: Omit if using torch with CUDA and install manually
+tqdm==4.64.1
+ujson==5.7.0

requirements_track.txt

@@ -0,0 +1,3 @@
+numpy==1.23.5
+PyYAML==5.4.1
+ujson==5.7.0

track.py

@@ -23,10 +23,10 @@
 import logging
 from pathlib import Path
 
-import OTVision
 import OTVision.config as config
 from OTVision.helpers.files import check_if_all_paths_exist
 from OTVision.helpers.log import LOGGER_NAME, VALID_LOG_LEVELS, log
+from OTVision.track.track import main as track
 
 
 def parse(argv: list[str] | None) -> argparse.Namespace:
@@ -220,7 +220,7 @@ def main(argv: list[str] | None = None) -> None:  # sourcery skip: assign-if-exp
     log.info(f"Arguments: {vars(args)}")
 
     try:
-        OTVision.track(
+        track(
             paths=paths,
             sigma_l=sigma_l,
             sigma_h=sigma_h,

transform.py

@@ -23,10 +23,10 @@
 import logging
 from pathlib import Path
 
-import OTVision
 import OTVision.config as config
 from OTVision.helpers.files import check_if_all_paths_exist
 from OTVision.helpers.log import LOGGER_NAME, VALID_LOG_LEVELS, log
+from OTVision.transform.transform import main as transform
 
 
 def parse() -> argparse.Namespace:
@@ -172,7 +172,7 @@ def main() -> None:
     log.info(f"Arguments: {vars(args)}")
 
     try:
-        OTVision.transform(paths=paths, refpts_file=refpts_file, overwrite=overwrite)
+        transform(paths=paths, refpts_file=refpts_file, overwrite=overwrite)
     except FileNotFoundError:
         log.exception(f"One of the following files cannot be found: {paths}")
         raise