Add yolo v4 config (#16)

* Add copy ci action to Pytorch repo

* Add yolo v4 config

* Remove test and CI references

* Remove readme

* [GH-Actions] Add new files from specs (#17)

.github/deployment-configs/deploy-yolo-notebook.yaml

Co-authored-by: Alexandre <18074599+payoto@users.noreply.github.com>

* Add test entry

* [GH-Actions] Add modified files from specs (#18)

.github/deployment-configs/deploy-yolo-notebook.yaml

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Downgrade torch requirement

* loosen old requirements

* Modified files for add-yolo-notebook (#19)

* [GH-Actions] Add modified files from specs

.github/deployment-configs/deploy-yolo-notebook.yaml

* Update object-detection-with-yolo/requirements.txt

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Alexandre Payot <18074599+payoto@users.noreply.github.com>

* Modified files for add-yolo-notebook (#20)

* [GH-Actions] Add modified files from specs

.github/deployment-configs/deploy-yolo-notebook.yaml

* [GH-Actions] Add new files from specs

.github/deployment-configs/deploy-yolo-notebook.yaml

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Remove unnecessary files

* Loosen requirements

* Downgrade torch requirements

* Add run on gradient button with shortened link

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: evawGraphcore <110815532+evawGraphcore@users.noreply.github.com>

Author: 3 people

.github/deployment-configs/deploy-yolo-notebook.yaml

@@ -0,0 +1,28 @@
+_examples_internal_repository: &_examples_internal_repository
+  origin: examples-internal/
+  ref: null
+
+_current_repo_in_github_actions: &_current_repo_in_github_actions
+    origin: notebooks/
+    ref: null
+
+yolov4-object-detection:
+    source:
+      paths:
+      - expression: '*'
+        path: vision/yolo_v4/pytorch
+        recursive: true
+      repository:
+        origin: examples-internal/
+        ref: yolo-notebook
+        prefix: vision/yolo_v4/pytorch
+      excludes:
+        - path: vision/yolo_v4/pytorch/.ci/
+        - path: vision/yolo_v4/pytorch/conftest.py
+        - path: vision/yolo_v4/pytorch/tests/
+        - path: vision/yolo_v4/pytorch/README.md
+    target:
+      renames: {}
+      repository:
+        <<: *_current_repo_in_github_actions
+        prefix: object-detection-with-yolo

.gradient/notebook-tests.yaml

@@ -98,6 +98,14 @@ vit-model-training:
     timeout: 10000
   requirements_file: requirements.txt
 
+object-detection-with-yolo:
+  location: ../object-detection-with-yolo
+  generated: true
+  notebook:
+    file: notebook_yolo.ipynb
+    timeout: 3600
+
+
 # Useful references
 useful-managing-ipu-resources:
   location: ../useful-tips

object-detection-with-yolo/Makefile

@@ -0,0 +1,29 @@
+
+CXX ?= g++
+CXXFLAGS =  -std=c++14 -g -fPIC -shared -I..
+LDLIBS =  -lpopart -lpoplar -lpopops
+ONNX_NAMESPACE = -DONNX_NAMESPACE=onnx
+
+BUILD_DIR = utils/custom_ops/build
+COPY_SOURCE = utils/custom_ops/copy_tensor/copy_custom_op.cpp
+COPY_TARGET = $(BUILD_DIR)/copy_tensor_custom_op.so
+
+NMS_DIR = utils/custom_ops/nms
+NMS_SOURCE = $(NMS_DIR)/nms_custom_op.cpp
+NMS_TARGET = $(BUILD_DIR)/nms_custom_op.so
+
+all: create_build_dir copy_custom_op nms_custom_op
+
+.PHONY: create_build_dir
+create_build_dir:
+	mkdir -p $(BUILD_DIR)
+
+copy_custom_op: $(COPY_SOURCE)
+	$(CXX) $? $(LDLIBS) $(CXXFLAGS) $(ONNX_NAMESPACE) -o $(COPY_TARGET)
+
+nms_custom_op: $(NMS_SOURCE) $(NMS_DIR)/nms.cpp $(NMS_DIR)/ipu_utils.cpp
+	$(CXX) $? $(LDLIBS) $(CXXFLAGS) $(ONNX_NAMESPACE) -o $(NMS_TARGET)
+
+.PHONY: clean
+clean:
+	rm -rf  $(BUILD_DIR)

object-detection-with-yolo/api.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2023 Graphcore Ltd. All rights reserved.
+import yacs
+import torch
+import poptorch
+import numpy as np
+from PIL import Image
+from ruamel import yaml
+from run import ipu_options
+from typing import NamedTuple
+from typing import List, Tuple
+from poptorch import inferenceModel
+from models.yolov4_p5 import Yolov4P5
+from utils.config import get_cfg_defaults
+from torchvision.transforms import Compose
+from utils.visualization import plotting_tool
+from utils.postprocessing import post_processing
+from utils.tools import load_and_fuse_pretrained_weights
+from utils.preprocessing import ResizeImage, Pad, ToNumpy, ToTensor
+
+
+class YoloOutput(NamedTuple):
+    x_top_left: float
+    y_top_left: float
+    width: float
+    height: float
+    detection_confidence: float
+    detected_class_id: float
+    detected_class_name: str
+
+
+class YOLOv4InferencePipeline:
+    def __init__(self, checkpoint_path: str):
+        self.cfg = get_cfg_defaults()
+        self.config = "configs/inference-yolov4p5.yaml"
+        self.cfg.merge_from_file(self.config)
+        self.cfg.freeze()
+        self.checkpoint = checkpoint_path
+        self.model = self.prepare_inference_yolo_v4_model_for_ipu()
+        self.class_names = yaml.safe_load(open("configs/class_name.yaml"))["class_names"]
+
+    def __call__(self, image: Image) -> Tuple[List[YoloOutput], List[np.array]]:
+        transformed_image, size = self.preprocess_image(image)
+        y = self.model(transformed_image)
+        dummy_label = []
+        processed_batch = post_processing(self.cfg, y, [size], dummy_label)
+        batch_as_list = processed_batch[0][0].tolist()
+        named_output = [
+            YoloOutput._make(obj[0:5] + [obj[-1]] + [self.class_names[int(obj[-1])]]) for obj in batch_as_list
+        ]
+
+        return named_output
+
+    def preprocess_image(self, img: Image) -> Tuple[torch.Tensor, torch.Tensor]:
+        height, width = img.size
+        print("original image dimensions:\nh:", height, "w:", width)
+
+        img_conv = img.convert("RGB")
+
+        # Change the data type of the dataloader depending of the options
+        if self.cfg.model.uint_io:
+            image_type = "uint"
+        elif not self.cfg.model.ipu or not self.cfg.model.half:
+            image_type = "float"
+        else:
+            image_type = "half"
+
+        size = torch.as_tensor(img_conv.size)
+        resize_img_mthd = ResizeImage(self.cfg.model.image_size)
+        print("img_size: ", self.cfg.model.image_size)
+        pad_mthd = Pad(self.cfg.model.image_size)
+        image_to_tensor_mthd = Compose([ToNumpy(), ToTensor(int(self.cfg.dataset.max_bbox_per_scale), image_type)])
+        dummy_label = np.array([[1.0, 1.0, 1.0, 1.0, 1.0]])
+        transformed_image, transformed_labels = resize_img_mthd((img_conv, dummy_label))
+        transformed_image, transformed_labels = pad_mthd((transformed_image, transformed_labels))
+        transformed_image, transformed_labels = image_to_tensor_mthd((transformed_image, transformed_labels))
+        transformed_image, _ = torch.unsqueeze(transformed_image, dim=0), torch.unsqueeze(transformed_labels, dim=0)
+
+        return transformed_image, size
+
+    def prepare_inference_yolo_v4_model_for_ipu(self) -> poptorch.PoplarExecutor:
+        mode = "inference"
+        original_model = Yolov4P5
+        model = original_model(self.cfg)
+
+        # Insert the pipeline splits if using pipeline
+        if self.cfg.model.pipeline_splits:
+            named_layers = {name: layer for name, layer in model.named_modules()}
+            for ipu_idx, split in enumerate(self.cfg.model.pipeline_splits):
+                named_layers[split] = poptorch.BeginBlock(ipu_id=ipu_idx + 1, layer_to_call=named_layers[split])
+
+        model = load_and_fuse_pretrained_weights(model, self.checkpoint)
+        model.optimize_for_inference()
+        model.eval()
+
+        # Create the specific ipu options if self.cfg.model.ipu
+        ipu_opts = ipu_options(self.cfg, model, mode) if self.cfg.model.ipu else None
+
+        model = inferenceModel(model, ipu_opts)
+
+        return model
+
+    def plot_img(self, processed_batch: List[YoloOutput], original_img: Image) -> List[str]:
+        list_of_predictions = [torch.Tensor([list(p)[0:6] for p in processed_batch])]
+        img_path = plotting_tool(self.cfg, list_of_predictions, [original_img])
+        return img_path
+
+    def detach(self):
+        if self.model.isAttachedToDevice():
+            self.model.detachFromDevice()

object-detection-with-yolo/benchmarks.yml

@@ -0,0 +1,42 @@
+---
+config_options: &config_options
+  requirements_path: requirements.txt
+  pre_run_commands: [make]
+
+pytorch_yolov4_infer_real_pod4:
+  <<: *config_options
+  description:
+    PyTorch Yolov4 inference benchmark on 4 IPUs using coco val2017 dataset
+  parameters:
+    - [img_size, batch_size, device_iterations, max_detections, topk_nums]
+    - [896, 1, 5, 200, 1800]
+    - [896, 1, 10, 300, 1468]
+    - [640, 2, 5, 200, 1200]
+    - [640, 2, 10, 300, 1300]
+    - [512, 2, 5, 200, 2600]
+    - [512, 2, 10, 300, 1400]
+    - [416, 2, 5, 200, 3000]
+    - [416, 4, 10, 100, 900]
+  cmd: >-
+    mpirun --tag-output --np 4 --allow-run-as-root
+      python3 run.py
+        --device-iterations {device_iterations}
+        --image-size {img_size}
+        --micro-batch-size {batch_size}
+        --benchmark
+        --mode test_inference
+        --nms-max-detections {max_detections}
+        --pre-nms-topk-k {topk_nums}
+  data:
+    throughput:
+      regexp: 'throughput: *(.*?) samples\/sec'
+    latency:
+      regexp: 'latency avg: *(.*?) ms'
+  output:
+    - [Batchsize, "batch_size"]
+    - [Device iterations, "device_iterations"]
+    - [Image size, "img_size"]
+    - [Pre-nms Topk, "topk_nums"]
+    - [NMS max detections, "max_detections"]
+    - [samples/sec, "throughput"]
+    - [latency(ms), "latency"]

object-detection-with-yolo/configs/class_name.yaml

@@ -0,0 +1,12 @@
+class_names: ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light',
+            'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',
+            'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
+            'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',
+            'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
+            'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
+            'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',
+            'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',
+            'hair drier', 'toothbrush']
+coco_91_class:  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34,
+         35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
+         64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]

object-detection-with-yolo/configs/inference-yolov4p5.yaml

@@ -0,0 +1,49 @@
+system:
+  num_ipus: 1
+  num_workers: 20
+model:
+  input_channels: 3
+  activation: mish
+  normalization: batch
+  anchors:
+    p3width: [13, 31, 24, 61]
+    p3height: [17, 25, 51, 45]
+    p4width: [48, 119, 97, 217]
+    p4height: [102, 96, 189, 184]
+    p5width: [171, 324, 616, 800]
+    p5height: [384, 451, 618, 800]
+  n_classes: 80
+  class_name_path: ./configs/class_name.yaml
+  strides: [8, 16, 32]
+  precision: half
+  image_size: 896
+  micro_batch_size: 1
+  mode: test
+  ipu: true
+ipuopts:
+  device_iterations: 1
+inference:
+  class_conf_threshold: 0.4
+  obj_threshold: 0.4
+  iou_threshold: 0.65
+  plot_output: false
+  plot_step: 250
+  plot_dir: plots
+dataset:
+  name: coco
+  max_bbox_per_scale: 90
+  train:
+    cache_data: false
+    file: train2017.txt
+    cache_path: ./utils/data/train
+    data_aug: false
+  test:
+    cache_data: false
+    file: val2017.txt
+    cache_path: ./utils/data/test
+    data_aug: false
+  mosaic: false
+  color: false
+eval:
+  metrics: true
+  verbose: false

object-detection-with-yolo/configs/training-yolov4p5.yaml

@@ -0,0 +1,58 @@
+system:
+  num_ipus: 16
+  num_workers: 20
+model:
+  input_channels: 3
+  activation: mish
+  normalization: group
+  pipeline_splits: [backbone.cspdark2.csp.res_modules.1, backbone.cspdark4.csp.bottleneck_conv1, neck.cspUp2.bneck_csp.res_modules.0.conv1]
+  anchors:
+    p3width: [6, 13, 11, 27]
+    p3height: [8, 12, 26, 19]
+    p4width: [24, 54, 39, 88]
+    p4height: [40, 38, 74, 73]
+    p5width: [66, 123, 226, 293]
+    p5height: [135, 195, 119, 258]
+  n_classes: 80
+  class_name_path: ./configs/class_name.yaml
+  strides: [8, 16, 32]
+  precision: mixed
+  image_size: 416
+  micro_batch_size: 1
+  mode: train
+  ipu: true
+  uint_io: true
+  max_nlabels_p3: 1000
+  max_nlabels_p4: 500
+  max_nlabels_p5: 200
+ipuopts:
+  device_iterations: 4
+  gradient_accumulation: 16
+training:
+  initial_lr: 0.01
+  stochastic_rounding: True
+inference:
+  class_conf_threshold: 0.001
+  obj_threshold: 0.001
+  iou_threshold: 0.6
+  plot_output: false
+  plot_step: 250
+  plot_dir: plots
+dataset:
+  name: coco
+  max_bbox_per_scale: 150
+  train:
+    cache_data: false
+    file: train2017.txt
+    cache_path: ./utils/data/train
+    data_aug: true
+  test:
+    cache_data: false
+    file: val2017.txt
+    cache_path: ./utils/data/test
+    data_aug: false
+  mosaic: true
+  color: true
+eval:
+  metrics: false
+  verbose: false

object-detection-with-yolo/models/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2021 Graphcore Ltd. All rights reserved.