diff --git a/armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py b/armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py new file mode 100644 index 000000000..bff467263 --- /dev/null +++ b/armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py @@ -0,0 +1,3 @@ +"""carla_multi_object_tracking_dev dataset.""" + +from .carla_multi_object_tracking_dev import CarlaMultiObjectTrackingDev diff --git a/armory/data/adversarial/carla_mot_dev.py b/armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py similarity index 91% rename from armory/data/adversarial/carla_mot_dev.py rename to armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py index aef392b2b..31d7aa23a 100644 --- a/armory/data/adversarial/carla_mot_dev.py +++ b/armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py @@ -1,11 +1,11 @@ -"""carla_mot_dev dataset.""" +"""carla_multi_object_tracking_dev dataset.""" import glob import os import re import numpy as np -import tensorflow.compat.v1 as tf +import tensorflow as tf import tensorflow_datasets as tfds _DESCRIPTION = """ @@ -22,7 +22,7 @@ } """ -_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" +_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz" class CarlaMOTDev(tfds.core.GeneratorBasedBuilder): @@ -70,14 +70,9 @@ def _info(self) -> tfds.core.DatasetInfo: def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" - path = dl_manager.download_and_extract(_URLS) + path = dl_manager.download_and_extract(_URL) - return [ - tfds.core.SplitGenerator( - name="dev", - gen_kwargs={"path": os.path.join(path, "dev")}, - ) - ] + return {"dev": self._generate_examples(path / "dev")} def _generate_examples(self, path): """Yields examples.""" diff --git a/armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv b/armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv new file mode 100644 index 000000000..b4574a33d --- /dev/null +++ b/armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv @@ -0,0 +1 @@ +https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz 704303119 cdd4be9cd3bcb5c2f94a6628350f106deec6fdc7b6a9c05711309b2bcc814f3d carla_mot_dev_1.0.0.tar.gz diff --git a/armory/datasets/cached_datasets.json b/armory/datasets/cached_datasets.json index 51eb26fca..062fdc4f6 100644 --- a/armory/datasets/cached_datasets.json +++ b/armory/datasets/cached_datasets.json @@ -27,6 +27,13 @@ "url": null, "version": "1.0.0" }, + "carla_over_obj_det_train": { + "sha256": "1fa2626726df4de8c0878f97ededdde90d4c55a4a845c8cee8db26a9797a2c6f", + "size": 14396331191, + "subdir": "carla_over_obj_det_train/1.0.0", + "url": null, + "version": "1.0.0" + }, "carla_video_tracking_dev": { "sha256": "958d470dcd394928050f4123a7af05b0e389ceeec6fa0a3261df55a65e553b69", "size": 1281628036, @@ -40,12 +47,6 @@ "subdir": "carla_video_tracking_test/2.0.0", "url": null, "version": "2.0.0" - "carla_over_obj_det_train": { - "sha256": "1fa2626726df4de8c0878f97ededdde90d4c55a4a845c8cee8db26a9797a2c6f", - "size": 14396331191, - "subdir": "carla_over_obj_det_train/1.0.0", - "url": null, - "version": "1.0.0" }, "cifar10": { "sha256": "05bf2d90665560e08ca6cf6e896b503ed4b83cd0e332d3a2e5970c9329acbaee", diff --git a/armory/datasets/preprocessing.py b/armory/datasets/preprocessing.py index a47925924..be7525b09 100644 --- a/armory/datasets/preprocessing.py +++ b/armory/datasets/preprocessing.py @@ -98,25 +98,35 @@ def carla_over_obj_det_dev(element, modality="rgb"): ) -def carla_video_tracking_preprocess(x, max_frames=None): - # Clip +def clip(batch, max_frames=None): if max_frames: max_frames = int(max_frames) if max_frames <= 0: raise ValueError(f"max_frames {max_frames} must be > 0") - x = x[:max_frames, :] - x = tf.cast(x, tf.float32) / 255.0 - return x + batch = batch[:max_frames, :] + return batch -def carla_video_tracking_preprocess_labels(y, y_patch_metadata, max_frames=None): - # Clip +def clip_labels(boxes, patch_metadata_dict, max_frames=None): if max_frames: max_frames = int(max_frames) if max_frames <= 0: raise ValueError(f"max_frames {max_frames} must be > 0") - y = y[:max_frames, :] - y_patch_metadata = {k: v[:max_frames, :] for (k, v) in y_patch_metadata.items()} + boxes = boxes[:max_frames, :] + patch_metadata_dict = { + k: v[:max_frames, :] for (k, v) in patch_metadata_dict.items() + } + return boxes, patch_metadata_dict + + +def carla_video_tracking_preprocess(x, max_frames=None): + x = clip(x, max_frames=max_frames) + x = tf.cast(x, tf.float32) / 255.0 + return x + + +def carla_video_tracking_preprocess_labels(y, y_patch_metadata, max_frames=None): + y, y_patch_metadata = clip_labels(y, y_patch_metadata, max_frames=max_frames) # Update labels y = {"boxes": y} y_patch_metadata = { @@ -146,6 +156,96 @@ def carla_over_obj_det_train(element, modality="rgb"): ), convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"]) +def mot_zero_index(y, y_patch_metadata): + if tf.rank(y) == 2: + y = tf.tensor_scatter_nd_update(y, [[0, 0]], [y[0, 0] - 1]) + else: + for i in tf.range(tf.shape(y)[0]): + y_i = tf.expand_dims(y[i], axis=0) # Add an extra dimension + y_i = tf.tensor_scatter_nd_update(y_i, [[0, 0]], [y_i[0, 0] - 1]) + y_i = tf.squeeze(y_i, axis=0) # Remove the extra dimension + y = tf.tensor_scatter_nd_update(y, [[i]], [y_i]) + return y, y_patch_metadata + + +def mot_array_to_coco(batch): + """ + Map from 3D array (batch_size x detections x 9) to extended coco format + of dimension (batch_size x frames x detections_per_frame) + NOTE: 'image_id' is given as the frame of a video, so is not unique + """ + if len(batch.shape) == 2: + not_batch = True + batch = tf.expand_dims(batch, axis=0) + elif len(batch.shape) == 3: + not_batch = False + else: + raise ValueError(f"batch.ndim {len(batch.shape)} is not in (2, 3)") + + # output = tf.TensorArray(dtype=tf.float32, size=batch.shape[0], dynamic_size=False) + output = [] + for i in range(batch.shape[0]): + array = batch[i] + # if not tf.math.greater(tf.shape(array)[0], 0): + if array.shape[0] == 0: + # no object detections + # output = output.write(i, []) + output.append([]) + continue + + # frames = tf.TensorArray(dtype=tf.float32, size=tf.shape(array)[0], dynamic_size=False) + frames = [] + for detection in array: + frame = tf.lookup.StaticHashTable( + { + # TODO: should image_id include video number as well? + "image_id": tf.cast(tf.math.round(detection[0]), tf.int32), + "category_id": tf.cast(tf.math.round(detection[7]), tf.int32), + "bbox": tf.cast(detection[2:6], float), + "score": tf.cast(detection[6], float), + # The following are extended fields + "object_id": tf.cast( + tf.math.round(detection[1]), tf.int32 + ), # for a specific object across frames + "visibility": tf.cast(detection[8], float), + } + ) + frames.append(frame) + # frames = frames.write(frames.size(), frame) + # output = output.write(i, frames) + output.append(frames) + + if not_batch: + output = output[0] + + raise NotImplementedError("This does not work yet") + return output + + +def carla_mot_label_preprocessing( + y, y_patch_metadata, coco_format=False, max_frames=None +): + y, y_patch_metadata = mot_zero_index(y, y_patch_metadata) + y, y_patch_metadata = clip_labels(y, y_patch_metadata, max_frames=max_frames) + if coco_format: + y = mot_array_to_coco(y) + y_patch_metadata = {k: tf.squeeze(v, axis=0) for k, v in y_patch_metadata.items()} + return y, y_patch_metadata + + +@register +def carla_multi_object_tracking_dev(element, coco_format=False, max_frames=None): + return carla_video_tracking_preprocess( + element["video"], + max_frames=max_frames, + ), carla_mot_label_preprocessing( + element["annotations"], + element["patch_metadata"], + coco_format=coco_format, + max_frames=max_frames, + ) + + @register def xview(element): return image_to_canon(element["image"]), convert_tf_obj_det_label_to_pytorch( diff --git a/armory/utils/config_loading.py b/armory/utils/config_loading.py index 2e08017cb..7cf849394 100644 --- a/armory/utils/config_loading.py +++ b/armory/utils/config_loading.py @@ -31,6 +31,7 @@ from armory.art_experimental.attacks import patch from armory.art_experimental.attacks.sweep import SweepAttack from armory.datasets.generator import ArmoryDataGenerator +from armory.data.datasets import EvalGenerator # TODO: Remove before PR merge from armory.data.utils import maybe_download_weights_from_s3 from armory.utils import labels diff --git a/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json b/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json index f3a8b3e0c..c78468403 100755 --- a/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json +++ b/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json @@ -16,8 +16,8 @@ "dataset": { "test": { "batch_size": 1, - "split": "dev", - "name": "carla_video_tracking_dev" + "name": "carla_video_tracking_dev", + "split": "dev" } }, "defense": { diff --git a/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json b/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json index a6906c4eb..ef9178906 100755 --- a/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json +++ b/scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json @@ -16,8 +16,8 @@ "dataset": { "test": { "batch_size": 1, - "split": "dev", - "name": "carla_video_tracking_dev" + "name": "carla_video_tracking_dev", + "split": "dev" } }, "defense": null, diff --git a/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json b/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json index 5f15432f0..14467e9f3 100644 --- a/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json +++ b/scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json @@ -17,12 +17,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "coco_format": true, - "eval_split": "dev", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "carla_multi_object_tracking_dev" + "test": { + "batch_size": 1, + "name": "carla_multi_object_tracking_dev", + "preprocessor_kwargs": { + "coco_format": true + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json index 1490f7d69..17e27d7a2 100644 --- a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json +++ b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "coco_format": true, - "eval_split": "dev", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "carla_multi_object_tracking_dev" + "test": { + "batch_size": 1, + "name": "carla_multi_object_tracking_dev", + "preprocessor_kwargs": { + "coco_format": true + }, + "split": "dev" + } }, "defense": { "kwargs": { diff --git a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json index df80ca8a0..e3c0ba0cf 100644 --- a/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json +++ b/scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "coco_format": true, - "eval_split": "dev", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "carla_multi_object_tracking_dev" + "test": { + "batch_size": 1, + "name": "carla_multi_object_tracking_dev", + "preprocessor_kwargs": { + "coco_format": true + }, + "split": "dev" + } }, "defense": null, "metric": {