From 956405a8b440a29a54eae0eead3fabbe3da08ffc Mon Sep 17 00:00:00 2001 From: Jonathan Prokos Date: Mon, 26 Dec 2022 16:12:52 +0000 Subject: [PATCH 1/5] built dataset --- .../adversarial/carla_mot_dev/__init__.py | 3 +++ .../carla_mot_dev}/carla_mot_dev.py | 13 +++------- .../carla_mot_dev/carla_mot_dev_test.py | 25 +++++++++++++++++++ .../adversarial/carla_mot_dev/checksums.tsv | 1 + .../TODO-add_fake_data_in_this_directory.txt | 0 5 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 armory/datasets/adversarial/carla_mot_dev/__init__.py rename armory/{data/adversarial => datasets/adversarial/carla_mot_dev}/carla_mot_dev.py (92%) create mode 100644 armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py create mode 100644 armory/datasets/adversarial/carla_mot_dev/checksums.tsv create mode 100644 armory/datasets/adversarial/carla_mot_dev/dummy_data/TODO-add_fake_data_in_this_directory.txt diff --git a/armory/datasets/adversarial/carla_mot_dev/__init__.py b/armory/datasets/adversarial/carla_mot_dev/__init__.py new file mode 100644 index 000000000..c3ab15ebe --- /dev/null +++ b/armory/datasets/adversarial/carla_mot_dev/__init__.py @@ -0,0 +1,3 @@ +"""carla_mot_dev dataset.""" + +from .carla_mot_dev import CarlaMotDev diff --git a/armory/data/adversarial/carla_mot_dev.py b/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev.py similarity index 92% rename from armory/data/adversarial/carla_mot_dev.py rename to armory/datasets/adversarial/carla_mot_dev/carla_mot_dev.py index d54617d5d..4e3e80214 100644 --- a/armory/data/adversarial/carla_mot_dev.py +++ b/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev.py @@ -4,7 +4,7 @@ import glob import re import numpy as np -import tensorflow.compat.v1 as tf +import tensorflow as tf import tensorflow_datasets as tfds @@ -22,7 +22,7 @@ } """ -_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" +_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" class CarlaMOTDev(tfds.core.GeneratorBasedBuilder): @@ -70,14 +70,9 @@ def _info(self) -> tfds.core.DatasetInfo: def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" - path = dl_manager.download_and_extract(_URLS) + path = dl_manager.download_and_extract(_URL) - return [ - tfds.core.SplitGenerator( - name="dev", - gen_kwargs={"path": os.path.join(path, "dev")}, - ) - ] + return {"dev": self._generate_examples(path / "dev")} def _generate_examples(self, path): """Yields examples.""" diff --git a/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py b/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py new file mode 100644 index 000000000..0ffaa05e2 --- /dev/null +++ b/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py @@ -0,0 +1,25 @@ +"""carla_mot_dev dataset.""" + +import tensorflow_datasets as tfds +from . import carla_mot_dev + + +class CarlaMotDevTest(tfds.testing.DatasetBuilderTestCase): + """Tests for carla_mot_dev dataset.""" + + # TODO(carla_mot_dev): + DATASET_CLASS = carla_mot_dev.CarlaMotDev + SPLITS = { + "train": 3, # Number of fake train example + "test": 1, # Number of fake test example + } + + # If you are calling `download/download_and_extract` with a dict, like: + # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) + # then the tests needs to provide the fake output paths relative to the + # fake data directory + # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} + + +if __name__ == "__main__": + tfds.testing.test_main() diff --git a/armory/datasets/adversarial/carla_mot_dev/checksums.tsv b/armory/datasets/adversarial/carla_mot_dev/checksums.tsv new file mode 100644 index 000000000..b4574a33d --- /dev/null +++ b/armory/datasets/adversarial/carla_mot_dev/checksums.tsv @@ -0,0 +1 @@ +https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz 704303119 cdd4be9cd3bcb5c2f94a6628350f106deec6fdc7b6a9c05711309b2bcc814f3d carla_mot_dev_1.0.0.tar.gz diff --git a/armory/datasets/adversarial/carla_mot_dev/dummy_data/TODO-add_fake_data_in_this_directory.txt b/armory/datasets/adversarial/carla_mot_dev/dummy_data/TODO-add_fake_data_in_this_directory.txt new file mode 100644 index 000000000..e69de29bb From 49410f1fc434fc64fe703d97e29b210f4bc9fac6 Mon Sep 17 00:00:00 2001 From: Jonathan Prokos Date: Mon, 26 Dec 2022 19:48:49 +0000 Subject: [PATCH 2/5] Revert "built dataset" This reverts commit 956405a8b440a29a54eae0eead3fabbe3da08ffc. --- .../adversarial}/carla_mot_dev.py | 13 +++++++--- .../adversarial/carla_mot_dev/__init__.py | 3 --- .../carla_mot_dev/carla_mot_dev_test.py | 25 ------------------- .../adversarial/carla_mot_dev/checksums.tsv | 1 - .../TODO-add_fake_data_in_this_directory.txt | 0 5 files changed, 9 insertions(+), 33 deletions(-) rename armory/{datasets/adversarial/carla_mot_dev => data/adversarial}/carla_mot_dev.py (92%) delete mode 100644 armory/datasets/adversarial/carla_mot_dev/__init__.py delete mode 100644 armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py delete mode 100644 armory/datasets/adversarial/carla_mot_dev/checksums.tsv delete mode 100644 armory/datasets/adversarial/carla_mot_dev/dummy_data/TODO-add_fake_data_in_this_directory.txt diff --git a/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev.py b/armory/data/adversarial/carla_mot_dev.py similarity index 92% rename from armory/datasets/adversarial/carla_mot_dev/carla_mot_dev.py rename to armory/data/adversarial/carla_mot_dev.py index 4e3e80214..d54617d5d 100644 --- a/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev.py +++ b/armory/data/adversarial/carla_mot_dev.py @@ -4,7 +4,7 @@ import glob import re import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf import tensorflow_datasets as tfds @@ -22,7 +22,7 @@ } """ -_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" +_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" class CarlaMOTDev(tfds.core.GeneratorBasedBuilder): @@ -70,9 +70,14 @@ def _info(self) -> tfds.core.DatasetInfo: def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" - path = dl_manager.download_and_extract(_URL) + path = dl_manager.download_and_extract(_URLS) - return {"dev": self._generate_examples(path / "dev")} + return [ + tfds.core.SplitGenerator( + name="dev", + gen_kwargs={"path": os.path.join(path, "dev")}, + ) + ] def _generate_examples(self, path): """Yields examples.""" diff --git a/armory/datasets/adversarial/carla_mot_dev/__init__.py b/armory/datasets/adversarial/carla_mot_dev/__init__.py deleted file mode 100644 index c3ab15ebe..000000000 --- a/armory/datasets/adversarial/carla_mot_dev/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""carla_mot_dev dataset.""" - -from .carla_mot_dev import CarlaMotDev diff --git a/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py b/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py deleted file mode 100644 index 0ffaa05e2..000000000 --- a/armory/datasets/adversarial/carla_mot_dev/carla_mot_dev_test.py +++ /dev/null @@ -1,25 +0,0 @@ -"""carla_mot_dev dataset.""" - -import tensorflow_datasets as tfds -from . import carla_mot_dev - - -class CarlaMotDevTest(tfds.testing.DatasetBuilderTestCase): - """Tests for carla_mot_dev dataset.""" - - # TODO(carla_mot_dev): - DATASET_CLASS = carla_mot_dev.CarlaMotDev - SPLITS = { - "train": 3, # Number of fake train example - "test": 1, # Number of fake test example - } - - # If you are calling `download/download_and_extract` with a dict, like: - # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) - # then the tests needs to provide the fake output paths relative to the - # fake data directory - # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} - - -if __name__ == "__main__": - tfds.testing.test_main() diff --git a/armory/datasets/adversarial/carla_mot_dev/checksums.tsv b/armory/datasets/adversarial/carla_mot_dev/checksums.tsv deleted file mode 100644 index b4574a33d..000000000 --- a/armory/datasets/adversarial/carla_mot_dev/checksums.tsv +++ /dev/null @@ -1 +0,0 @@ -https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz 704303119 cdd4be9cd3bcb5c2f94a6628350f106deec6fdc7b6a9c05711309b2bcc814f3d carla_mot_dev_1.0.0.tar.gz diff --git a/armory/datasets/adversarial/carla_mot_dev/dummy_data/TODO-add_fake_data_in_this_directory.txt b/armory/datasets/adversarial/carla_mot_dev/dummy_data/TODO-add_fake_data_in_this_directory.txt deleted file mode 100644 index e69de29bb..000000000 From 5edd3c19e736130007cd75465207a711c2c2e399 Mon Sep 17 00:00:00 2001 From: Jonathan Prokos Date: Thu, 29 Dec 2022 18:04:03 +0000 Subject: [PATCH 3/5] preprocessing not implemented --- .../pytorch/carla_mot_frcnn_byte.py | 47 ++++++++++++++++++- .../__init__.py | 3 ++ .../carla_multi_object_tracking_dev.py} | 15 ++---- .../checksums.tsv | 1 + armory/datasets/preprocessing.py | 6 +++ ...carla_mot_adversarialpatch_undefended.json | 14 +++--- .../carla_mot/carla_mot_dpatch_defended.json | 14 +++--- .../carla_mot_dpatch_undefended.json | 14 +++--- 8 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py rename armory/{data/adversarial/carla_mot_dev.py => datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py} (91%) create mode 100644 armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv diff --git a/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py b/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py index dc682774a..e4d58be7b 100644 --- a/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py +++ b/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py @@ -10,8 +10,6 @@ from torchvision import models import dataclasses -from armory.data.adversarial_datasets import mot_array_to_coco - from yolox.tracker.byte_tracker import ( BYTETracker, ) # clone from https://github.com/ifzhang/ByteTrack @@ -85,6 +83,51 @@ def predict_object_detector(self, x, **kwargs): - scores [N]: the scores or each prediction. """ return super().predict(x, **kwargs) + + def mot_array_to_coco(batch): + """ + Map from 3D array (batch_size x detections x 9) to extended coco format + of dimension (batch_size x frames x detections_per_frame) + + NOTE: 'image_id' is given as the frame of a video, so is not unique + """ + if batch.ndim == 2: + not_batch = True + batch = [batch] + elif batch.ndim == 3: + not_batch = False + else: + raise ValueError(f"batch.ndim {batch.ndim} is not in (2, 3)") + + output = np.empty(len(batch), dtype=object) + for i, array in enumerate(batch): + if not len(array): + # no object detections + output.append([]) + continue + + frames = [] + for detection in array: + frames.append( + { + # TODO: should image_id include video number as well? + "image_id": int(np.round(detection[0])), + "category_id": int(np.round(detection[7])), + "bbox": [float(x) for x in detection[2:6]], + "score": float(detection[6]), + # The following are extended fields + "object_id": int( + np.round(detection[1]) + ), # for a specific object across frames + "visibility": float(detection[8]), + } + ) + output[i] = frames + + if not_batch: + output = output[0] + + return output def predict(self, x): """ diff --git a/armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py b/armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py new file mode 100644 index 000000000..bff467263 --- /dev/null +++ b/armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py @@ -0,0 +1,3 @@ +"""carla_multi_object_tracking_dev dataset.""" + +from .carla_multi_object_tracking_dev import CarlaMultiObjectTrackingDev diff --git a/armory/data/adversarial/carla_mot_dev.py b/armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py similarity index 91% rename from armory/data/adversarial/carla_mot_dev.py rename to armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py index d54617d5d..e4cb442e9 100644 --- a/armory/data/adversarial/carla_mot_dev.py +++ b/armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py @@ -1,10 +1,10 @@ -"""carla_mot_dev dataset.""" +"""carla_multi_object_tracking_dev dataset.""" import os import glob import re import numpy as np -import tensorflow.compat.v1 as tf +import tensorflow as tf import tensorflow_datasets as tfds @@ -22,7 +22,7 @@ } """ -_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" +_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" class CarlaMOTDev(tfds.core.GeneratorBasedBuilder): @@ -70,14 +70,9 @@ def _info(self) -> tfds.core.DatasetInfo: def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" - path = dl_manager.download_and_extract(_URLS) + path = dl_manager.download_and_extract(_URL) - return [ - tfds.core.SplitGenerator( - name="dev", - gen_kwargs={"path": os.path.join(path, "dev")}, - ) - ] + return {"dev": self._generate_examples(path / "dev")} def _generate_examples(self, path): """Yields examples.""" diff --git a/armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv b/armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv new file mode 100644 index 000000000..b4574a33d --- /dev/null +++ b/armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv @@ -0,0 +1 @@ +https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz 704303119 cdd4be9cd3bcb5c2f94a6628350f106deec6fdc7b6a9c05711309b2bcc814f3d carla_mot_dev_1.0.0.tar.gz diff --git a/armory/datasets/preprocessing.py b/armory/datasets/preprocessing.py index 4b8ce1a79..014624e52 100644 --- a/armory/datasets/preprocessing.py +++ b/armory/datasets/preprocessing.py @@ -66,6 +66,12 @@ def carla_over_obj_det_dev(element, modality="rgb"): ) +@register +def carla_multi_object_tracking_dev(element, coco_format=True): + breakpoint() + return carla_multi_object_tracking(element["video"], ) + + @register def xview(element): return image_to_canon(element["image"]), convert_tf_obj_det_label_to_pytorch( diff --git a/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json b/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json index 6472bd921..73f310159 100644 --- a/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json +++ b/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json @@ -17,12 +17,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "coco_format": true, - "eval_split": "dev", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "carla_multi_object_tracking_dev" + "test": { + "batch_size": 1, + "name": "carla_multi_object_tracking_dev", + "preprocessor_kwargs": { + "coco_format": true + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json index 140f1ddbb..1e610e975 100644 --- a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json +++ b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "coco_format": true, - "eval_split": "dev", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "carla_multi_object_tracking_dev" + "test": { + "batch_size": 1, + "name": "carla_multi_object_tracking_dev", + "preprocessor_kwargs": { + "coco_format": true + }, + "split": "dev" + } }, "defense": { "kwargs": { diff --git a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json index 27a4b47a5..5890e45f6 100644 --- a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json +++ b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "coco_format": true, - "eval_split": "dev", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "carla_multi_object_tracking_dev" + "test": { + "batch_size": 1, + "name": "carla_multi_object_tracking_dev", + "preprocessor_kwargs": { + "coco_format": true + }, + "split": "dev" + } }, "defense": null, "metric": { From 0d15a1bb466fbe16dbd75d6e14e9a4511eaa1a9e Mon Sep 17 00:00:00 2001 From: Jonathan Prokos Date: Wed, 29 Mar 2023 17:43:08 +0000 Subject: [PATCH 4/5] reverting to https://github.com/twosixlabs/armory/blob/tfdsv4/armory/utils/config_loading.py --- armory/utils/config_loading.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/armory/utils/config_loading.py b/armory/utils/config_loading.py index 9ba4524d3..7cf849394 100644 --- a/armory/utils/config_loading.py +++ b/armory/utils/config_loading.py @@ -3,6 +3,7 @@ """ from importlib import import_module + from armory.logs import log # import torch before tensorflow to ensure torch.utils.data.DataLoader can utilize @@ -20,16 +21,19 @@ "ART 1.2 support is deprecated and will be removed in ARMORY 0.11. Use ART 1.3" ) from art.classifiers import Classifier + +import copy + from art.defences.postprocessor import Postprocessor from art.defences.preprocessor import Preprocessor from art.defences.trainer import Trainer from armory.art_experimental.attacks import patch from armory.art_experimental.attacks.sweep import SweepAttack -from armory.data.datasets import ArmoryDataGenerator, EvalGenerator +from armory.datasets.generator import ArmoryDataGenerator +from armory.data.datasets import EvalGenerator # TODO: Remove before PR merge from armory.data.utils import maybe_download_weights_from_s3 from armory.utils import labels -import copy def load(sub_config): @@ -211,7 +215,15 @@ def load_defense_wrapper(defense_config, classifier): defense_module = import_module(defense_config["module"]) defense_fn = getattr(defense_module, defense_config["name"]) - defense = defense_fn(classifier, **defense_config["kwargs"]) + kwargs = copy.deepcopy(defense_config["kwargs"]) + if "augmentations" in kwargs and kwargs["augmentations"] is not None: + # create Preprocess object and add it to kwargs + aug_config = kwargs.pop("augmentations") + aug_module = import_module("art.defences.preprocessor") + aug_fn = getattr(aug_module, aug_config["name"]) + augmentation = aug_fn(**aug_config["kwargs"]) + kwargs["augmentations"] = augmentation + defense = defense_fn(classifier, **kwargs) _check_defense_api(defense, Trainer) return defense From 4c1bf441ecec8cb20244338f933bee38583282c4 Mon Sep 17 00:00:00 2001 From: Jonathan Prokos Date: Thu, 30 Mar 2023 16:51:13 +0000 Subject: [PATCH 5/5] WIP MOT preprocessing. Cannot store dictionary in tensor... --- .../pytorch/carla_mot_frcnn_byte.py | 45 ------- armory/datasets/preprocessing.py | 118 ++++++++++++++++-- 2 files changed, 106 insertions(+), 57 deletions(-) diff --git a/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py b/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py index aa3ddf9dd..20e88ca7b 100644 --- a/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py +++ b/armory/baseline_models/pytorch/carla_mot_frcnn_byte.py @@ -85,51 +85,6 @@ def predict_object_detector(self, x, **kwargs): """ return super().predict(x, **kwargs) - def mot_array_to_coco(batch): - """ - Map from 3D array (batch_size x detections x 9) to extended coco format - of dimension (batch_size x frames x detections_per_frame) - - NOTE: 'image_id' is given as the frame of a video, so is not unique - """ - if batch.ndim == 2: - not_batch = True - batch = [batch] - elif batch.ndim == 3: - not_batch = False - else: - raise ValueError(f"batch.ndim {batch.ndim} is not in (2, 3)") - - output = np.empty(len(batch), dtype=object) - for i, array in enumerate(batch): - if not len(array): - # no object detections - output.append([]) - continue - - frames = [] - for detection in array: - frames.append( - { - # TODO: should image_id include video number as well? - "image_id": int(np.round(detection[0])), - "category_id": int(np.round(detection[7])), - "bbox": [float(x) for x in detection[2:6]], - "score": float(detection[6]), - # The following are extended fields - "object_id": int( - np.round(detection[1]) - ), # for a specific object across frames - "visibility": float(detection[8]), - } - ) - output[i] = frames - - if not_batch: - output = output[0] - - return output - def predict(self, x): """ Perform tracking prediction for a batch of inputs by performing object detection, updating Kalman filters, and outputing filter predictions. diff --git a/armory/datasets/preprocessing.py b/armory/datasets/preprocessing.py index 07275eab8..be7525b09 100644 --- a/armory/datasets/preprocessing.py +++ b/armory/datasets/preprocessing.py @@ -98,25 +98,35 @@ def carla_over_obj_det_dev(element, modality="rgb"): ) -def carla_video_tracking_preprocess(x, max_frames=None): - # Clip +def clip(batch, max_frames=None): if max_frames: max_frames = int(max_frames) if max_frames <= 0: raise ValueError(f"max_frames {max_frames} must be > 0") - x = x[:max_frames, :] - x = tf.cast(x, tf.float32) / 255.0 - return x + batch = batch[:max_frames, :] + return batch -def carla_video_tracking_preprocess_labels(y, y_patch_metadata, max_frames=None): - # Clip +def clip_labels(boxes, patch_metadata_dict, max_frames=None): if max_frames: max_frames = int(max_frames) if max_frames <= 0: raise ValueError(f"max_frames {max_frames} must be > 0") - y = y[:max_frames, :] - y_patch_metadata = {k: v[:max_frames, :] for (k, v) in y_patch_metadata.items()} + boxes = boxes[:max_frames, :] + patch_metadata_dict = { + k: v[:max_frames, :] for (k, v) in patch_metadata_dict.items() + } + return boxes, patch_metadata_dict + + +def carla_video_tracking_preprocess(x, max_frames=None): + x = clip(x, max_frames=max_frames) + x = tf.cast(x, tf.float32) / 255.0 + return x + + +def carla_video_tracking_preprocess_labels(y, y_patch_metadata, max_frames=None): + y, y_patch_metadata = clip_labels(y, y_patch_metadata, max_frames=max_frames) # Update labels y = {"boxes": y} y_patch_metadata = { @@ -146,10 +156,94 @@ def carla_over_obj_det_train(element, modality="rgb"): ), convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"]) +def mot_zero_index(y, y_patch_metadata): + if tf.rank(y) == 2: + y = tf.tensor_scatter_nd_update(y, [[0, 0]], [y[0, 0] - 1]) + else: + for i in tf.range(tf.shape(y)[0]): + y_i = tf.expand_dims(y[i], axis=0) # Add an extra dimension + y_i = tf.tensor_scatter_nd_update(y_i, [[0, 0]], [y_i[0, 0] - 1]) + y_i = tf.squeeze(y_i, axis=0) # Remove the extra dimension + y = tf.tensor_scatter_nd_update(y, [[i]], [y_i]) + return y, y_patch_metadata + + +def mot_array_to_coco(batch): + """ + Map from 3D array (batch_size x detections x 9) to extended coco format + of dimension (batch_size x frames x detections_per_frame) + NOTE: 'image_id' is given as the frame of a video, so is not unique + """ + if len(batch.shape) == 2: + not_batch = True + batch = tf.expand_dims(batch, axis=0) + elif len(batch.shape) == 3: + not_batch = False + else: + raise ValueError(f"batch.ndim {len(batch.shape)} is not in (2, 3)") + + # output = tf.TensorArray(dtype=tf.float32, size=batch.shape[0], dynamic_size=False) + output = [] + for i in range(batch.shape[0]): + array = batch[i] + # if not tf.math.greater(tf.shape(array)[0], 0): + if array.shape[0] == 0: + # no object detections + # output = output.write(i, []) + output.append([]) + continue + + # frames = tf.TensorArray(dtype=tf.float32, size=tf.shape(array)[0], dynamic_size=False) + frames = [] + for detection in array: + frame = tf.lookup.StaticHashTable( + { + # TODO: should image_id include video number as well? + "image_id": tf.cast(tf.math.round(detection[0]), tf.int32), + "category_id": tf.cast(tf.math.round(detection[7]), tf.int32), + "bbox": tf.cast(detection[2:6], float), + "score": tf.cast(detection[6], float), + # The following are extended fields + "object_id": tf.cast( + tf.math.round(detection[1]), tf.int32 + ), # for a specific object across frames + "visibility": tf.cast(detection[8], float), + } + ) + frames.append(frame) + # frames = frames.write(frames.size(), frame) + # output = output.write(i, frames) + output.append(frames) + + if not_batch: + output = output[0] + + raise NotImplementedError("This does not work yet") + return output + + +def carla_mot_label_preprocessing( + y, y_patch_metadata, coco_format=False, max_frames=None +): + y, y_patch_metadata = mot_zero_index(y, y_patch_metadata) + y, y_patch_metadata = clip_labels(y, y_patch_metadata, max_frames=max_frames) + if coco_format: + y = mot_array_to_coco(y) + y_patch_metadata = {k: tf.squeeze(v, axis=0) for k, v in y_patch_metadata.items()} + return y, y_patch_metadata + + @register -def carla_multi_object_tracking_dev(element, coco_format=True): - breakpoint() - pass +def carla_multi_object_tracking_dev(element, coco_format=False, max_frames=None): + return carla_video_tracking_preprocess( + element["video"], + max_frames=max_frames, + ), carla_mot_label_preprocessing( + element["annotations"], + element["patch_metadata"], + coco_format=coco_format, + max_frames=max_frames, + ) @register