(WIP) Tfdsv4 carla mot

armory/baseline_models/pytorch/carla_mot_frcnn_byte.py

@@ -85,6 +85,51 @@ def predict_object_detector(self, x, **kwargs):
         """
         return super().predict(x, **kwargs)
 
+    def mot_array_to_coco(batch):
+        """
+        Map from 3D array (batch_size x detections x 9) to extended coco format
+            of dimension (batch_size x frames x detections_per_frame)
+
+        NOTE: 'image_id' is given as the frame of a video, so is not unique
+        """
+        if batch.ndim == 2:
+            not_batch = True
+            batch = [batch]
+        elif batch.ndim == 3:
+            not_batch = False
+        else:
+            raise ValueError(f"batch.ndim {batch.ndim} is not in (2, 3)")
+
+        output = np.empty(len(batch), dtype=object)
+        for i, array in enumerate(batch):
+            if not len(array):
+                # no object detections
+                output.append([])
+                continue
+
+            frames = []
+            for detection in array:
+                frames.append(
+                    {
+                        # TODO: should image_id include video number as well?
+                        "image_id": int(np.round(detection[0])),
+                        "category_id": int(np.round(detection[7])),
+                        "bbox": [float(x) for x in detection[2:6]],
+                        "score": float(detection[6]),
+                        # The following are extended fields
+                        "object_id": int(
+                            np.round(detection[1])
+                        ),  # for a specific object across frames
+                        "visibility": float(detection[8]),
+                    }
+                )
+            output[i] = frames
+
+        if not_batch:
+            output = output[0]
+
+        return output
+
     def predict(self, x):
         """
         Perform tracking prediction for a batch of inputs by performing object detection, updating Kalman filters, and outputing filter predictions.

armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py

@@ -0,0 +1,3 @@
+"""carla_multi_object_tracking_dev dataset."""
+
+from .carla_multi_object_tracking_dev import CarlaMultiObjectTrackingDev

armory/data/adversarial/carla_mot_dev.py → armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py

@@ -1,11 +1,11 @@
-"""carla_mot_dev dataset."""
+"""carla_multi_object_tracking_dev dataset."""
 
 import glob
 import os
 import re
 
 import numpy as np
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
 import tensorflow_datasets as tfds
 
 _DESCRIPTION = """
@@ -22,7 +22,7 @@
 }
 """
 
-_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz"
+_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz"
 
 
 class CarlaMOTDev(tfds.core.GeneratorBasedBuilder):
@@ -70,14 +70,9 @@ def _info(self) -> tfds.core.DatasetInfo:
 
     def _split_generators(self, dl_manager: tfds.download.DownloadManager):
         """Returns SplitGenerators."""
-        path = dl_manager.download_and_extract(_URLS)
+        path = dl_manager.download_and_extract(_URL)
 
-        return [
-            tfds.core.SplitGenerator(
-                name="dev",
-                gen_kwargs={"path": os.path.join(path, "dev")},
-            )
-        ]
+        return {"dev": self._generate_examples(path / "dev")}
 
     def _generate_examples(self, path):
         """Yields examples."""

armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv

@@ -0,0 +1 @@
+https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz	704303119	cdd4be9cd3bcb5c2f94a6628350f106deec6fdc7b6a9c05711309b2bcc814f3d	carla_mot_dev_1.0.0.tar.gz

armory/datasets/cached_datasets.json

@@ -27,6 +27,13 @@
         "url": null,
         "version": "1.0.0"
     },
+    "carla_over_obj_det_train": {
+        "sha256": "1fa2626726df4de8c0878f97ededdde90d4c55a4a845c8cee8db26a9797a2c6f",
+        "size": 14396331191,
+        "subdir": "carla_over_obj_det_train/1.0.0",
+        "url": null,
+        "version": "1.0.0"
+    },
     "carla_video_tracking_dev": {
         "sha256": "958d470dcd394928050f4123a7af05b0e389ceeec6fa0a3261df55a65e553b69",
         "size": 1281628036,
@@ -40,12 +47,6 @@
         "subdir": "carla_video_tracking_test/2.0.0",
         "url": null,
         "version": "2.0.0"
-    "carla_over_obj_det_train": {
-        "sha256": "1fa2626726df4de8c0878f97ededdde90d4c55a4a845c8cee8db26a9797a2c6f",
-        "size": 14396331191,
-        "subdir": "carla_over_obj_det_train/1.0.0",
-        "url": null,
-        "version": "1.0.0"
     },
     "cifar10": {
         "sha256": "05bf2d90665560e08ca6cf6e896b503ed4b83cd0e332d3a2e5970c9329acbaee",

armory/datasets/preprocessing.py

@@ -146,6 +146,12 @@ def carla_over_obj_det_train(element, modality="rgb"):
     ), convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"])
 
 
+@register
+def carla_multi_object_tracking_dev(element, coco_format=True):
+    breakpoint()
+    pass
+
+
 @register
 def xview(element):
     return image_to_canon(element["image"]), convert_tf_obj_det_label_to_pytorch(

armory/utils/config_loading.py

@@ -31,6 +31,7 @@
 from armory.art_experimental.attacks import patch
 from armory.art_experimental.attacks.sweep import SweepAttack
 from armory.datasets.generator import ArmoryDataGenerator
+from armory.data.datasets import EvalGenerator  # TODO: Remove before PR merge
 from armory.data.utils import maybe_download_weights_from_s3
 from armory.utils import labels
 

scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json

@@ -16,8 +16,8 @@
     "dataset": {
         "test": {
             "batch_size": 1,
-            "split": "dev",
-            "name": "carla_video_tracking_dev"
+            "name": "carla_video_tracking_dev",
+            "split": "dev"
         }
     },
     "defense": {

scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json

@@ -16,8 +16,8 @@
     "dataset": {
         "test": {
             "batch_size": 1,
-            "split": "dev",
-            "name": "carla_video_tracking_dev"
+            "name": "carla_video_tracking_dev",
+            "split": "dev"
         }
     },
     "defense": null,

scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json

@@ -17,12 +17,14 @@
         "use_label": true
     },
     "dataset": {
-        "batch_size": 1,
-        "coco_format": true,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_multi_object_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_multi_object_tracking_dev",
+            "preprocessor_kwargs": {
+                "coco_format": true
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {

scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json

@@ -14,12 +14,14 @@
         "use_label": false
     },
     "dataset": {
-        "batch_size": 1,
-        "coco_format": true,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_multi_object_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_multi_object_tracking_dev",
+            "preprocessor_kwargs": {
+                "coco_format": true
+            },
+            "split": "dev"
+        }
     },
     "defense": {
         "kwargs": {

scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json

@@ -14,12 +14,14 @@
         "use_label": false
     },
     "dataset": {
-        "batch_size": 1,
-        "coco_format": true,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_multi_object_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_multi_object_tracking_dev",
+            "preprocessor_kwargs": {
+                "coco_format": true
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {