From f0b69e8c5bc6578b9f98bce0d7077be5301e3e13 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Thu, 22 Jul 2021 17:32:16 +0200 Subject: [PATCH 01/17] metric for segmentation --- test/__init__.py | 0 test/test_confusion_matrix.py | 45 ++++++ test/test_model.py | 1 + test/test_segmentation_metric.py | 68 +++++++++ .../segmentation/segmentation_metrics.py | 129 ++++++++++++++++++ 5 files changed, 243 insertions(+) create mode 100644 test/__init__.py create mode 100644 test/test_confusion_matrix.py create mode 100644 test/test_segmentation_metric.py create mode 100644 torch_points3d/metrics/segmentation/segmentation_metrics.py diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_confusion_matrix.py b/test/test_confusion_matrix.py new file mode 100644 index 0000000..ad90340 --- /dev/null +++ b/test/test_confusion_matrix.py @@ -0,0 +1,45 @@ +import unittest +import pytest + + +class TestConfusionMatrix(unittest.TestCase): + + def test_compute_intersection_union_per_class(self): + matrix = torch.tensor([[4, 1], [2, 10]]) + iou = compute_intersection_union_per_class(matrix) + miou = compute_average_intersection_union(matrix) + print(iou) + self.assertAlmostEqual(iou[0].item(), 4 / (4.0 + 1.0 + 2.0)) + self.assertAlmostEqual(iou[1].item(), 10 / (10.0 + 1.0 + 2.0)) + self.assertAlmostEqual(iou.mean().item(), miou.item()) + + def test_compute_overall_accuracy(self): + list_matrix = [ + matrix = torch.tensor([[4, 1], [2, 10]]).float(), + matrix = torch.tensor([[4, 1], [2, 10]]).int(), + matrix = torch.tensor([[0, 0], [0, 0]]).float() + ] + list_answer = [ + (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), + (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), + 0.0 + ] + for i in range(len(list_matrix)): + acc = compute_overall_accuracy(list_matrix[i]) + self.assertAlmostEqual(acc.item(), list_answer[i]) + + + def test_compute_mean_class_accuracy(self): + matrix = torch.tensor([[4, 1], [2, 10]]).float() + macc = compute_mean_class_accuracy(matrix) + self.assertAlmostEqual(macc.item(), (4/5 + 10/12)*0.5) + + + @pytest.mark.parametrize(["missing_as_one", "answer"], [ + pytest.param(True, (0.5 + 0.5) / 2,) + pytest.param(False, (0.5 + 1 + 0.5) / 3) + ]) + def test_test_getMeanIoUMissing(self, missing_as_one, answer): + matrix = torch.tensor([[1, 1, 0], [0, 1, 0], [0, 0, 0]]) + self.assertAlmostEqual(compute_average_intersection_union(matrix, missing_as_one=missing_as_one), answer) + diff --git a/test/test_model.py b/test/test_model.py index 896940a..fb704ab 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -9,6 +9,7 @@ DIR = os.path.dirname(os.path.realpath(__file__)) ROOT = os.path.join(DIR, "..") sys.path.insert(0, ROOT) +sys.path.append('.') from torch_points3d.models.segmentation.sparseconv3d import APIModel diff --git a/test/test_segmentation_metric.py b/test/test_segmentation_metric.py new file mode 100644 index 0000000..c0ae704 --- /dev/null +++ b/test/test_segmentation_metric.py @@ -0,0 +1,68 @@ +import torch + +import unittest +import pytest + +from torch_geometric.data import Data + + +class MockDataset: + INV_OBJECT_LABEL = {0: "first", 1: "wall", 2: "not", 3: "here", 4: "hoy"} + pos = torch.tensor([[1, 0, 0], [2, 0, 0], [3, 0, 0], [-1, 0, 0]]).float() + test_label = torch.tensor([1, 1, 0, 0]) + + def __init__(self): + self.num_classes = 2 + + @property + def test_data(self): + return Data(pos=self.pos, y=self.test_label) + + def has_labels(self, stage): + return True + + +class MockModel: + def __init__(self): + self.iter = 0 + self.losses = [ + {"loss_1": 1, "loss_2": 2}, + {"loss_1": 2, "loss_2": 2}, + {"loss_1": 1, "loss_2": 2}, + {"loss_1": 1, "loss_2": 2}, + ] + self.outputs = [ + torch.tensor([[0, 1], [0, 1]]), + torch.tensor([[1, 0], [1, 0]]), + torch.tensor([[1, 0], [1, 0]]), + torch.tensor([[1, 0], [1, 0], [1, 0]]), + ] + self.labels = [torch.tensor([1, 1]), torch.tensor([1, 1]), torch.tensor([1, 1]), torch.tensor([0, 0, -100])] + self.batch_idx = [torch.tensor([0, 1]), torch.tensor([0, 1]), torch.tensor([0, 1]), torch.tensor([0, 0, 1])] + + def get_input(self): + return Data(pos=MockDataset.pos[:2, :], origin_id=torch.tensor([0, 1])) + + def get_output(self): + return self.outputs[self.iter].float() + + def get_labels(self): + return self.labels[self.iter] + + def get_current_losses(self): + return self.losses[self.iter] + + def get_batch(self): + return self.batch_idx[self.iter] + + @property + def device(self): + return "cpu" + + +class TestSegmentationMetrics(unittest.TestCase): + def test_forward(self): + pass + + def test_finalize(self): + pass diff --git a/torch_points3d/metrics/segmentation/segmentation_metrics.py b/torch_points3d/metrics/segmentation/segmentation_metrics.py new file mode 100644 index 0000000..6459fce --- /dev/null +++ b/torch_points3d/metrics/segmentation/segmentation_metrics.py @@ -0,0 +1,129 @@ +import torch +from typing import Dict, Optional, Tuple, Any +from torchmetrics import ConfusionMatrix +from torchmetrics import Metric + + +def compute_average_intersection_union(matrix: torch.Tensor, missing_as_one: bool = False) -> torch.Tensor: + """ + compute intersection over union on average from confusion matrix + Parameters + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + missing_as_one: bool, default: False + """ + + values, existing_classes_mask = compute_intersection_union_per_class(confusion_matrix, return_existing_mask=True) + if torch.sum(existing_classes_mask) == 0: + return torch.sum(existing_classes_mask) + if missing_as_one: + values[~existing_classes_mask] = 1 + existing_classes_mask[:] = True + return torch.sum(values[existing_classes_mask]) / torch.sum(existing_classes_mask) + + +def compute_mean_class_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: + """ + compute intersection over union on average from confusion matrix + + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + """ + total_gts = confusion_matrix.sum(1) + labels_presents = torch.where(total_gts > 0)[0] + if(len(labels_presents) == 0): + return total_gts[0] + ones = torch.ones_like(total_gts) + max_ones_total_gts = torch.cat([total_gts[None, :], ones[None, :]], 0).max(0)[0] + re = (torch.diagonal(confusion_matrix)[labels_presents].float() / max_ones_total_gts[labels_presents]).sum() + return re / float(len(labels_presents)) + +def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: + """ + compute overall accuracy from confusion matrix + + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + """ + all_values = confusion_matrix.sum() + if all_values == 0: + return 0 + matrix_diagonal = torch.trace(confusion_matrix) + return matrix_diagonal.float() / all_values + +def compute_intersection_union_per_class(confusion_matrix: torch.Tensor, return_existing_mask: bool = False, eps: float = 1e-8) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + compute intersection over union per class from confusion matrix + + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + """ + + TP_plus_FN = confusion_matrix.sum(0) + TP_plus_FP = confusion_matrix.sum(1) + TP = torch.diagonal(confusion_matrix) + union = TP_plus_FN + TP_plus_FP - TP + iou = eps + TP / (union + eps) + existing_class_mask = union > 1e-3 + if return_existing_mask: + return iou, existing_class_mask + else: + return iou, None + + + +class SegmentationTracker(BaseTracker): + """ + track different registration metrics + """ + + def __init__( + self, num_classes: int, + stage: str = "train", + ignore_label: int = -1, + eps: float = 1e-8,): + super().__init__() + self._ignore_label = ignore_label + self._num_classes = num_classes + self.confusion_matrix_metric = ConfusionMatrix(num_classes=self._num_classes) + self.eps = eps + + def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: + acc = compute_overall_accuracy(matrix) + macc = compute_mean_class_accuracy(matrix) + miou = compute_average_intersection_union(matrix) + iou_per_class, _ = compute_intersection_union_per_class(matrix, eps=self.eps) + iou_per_class_dict = {i: 100 * v for i, v in enumerate(iou_per_class)} + return { + "{}_acc".format(self.stage): 100 * acc, + "{}_macc".format(self.stage): 100 * macc, + "{}_miou".format(self.stage): 100 * miou, + "{}_iou_per_class".format(self.stage): iou_per_class_dict, + } + + + def track(targets: torch.Tensor, preds: torch.Tensor, **kwargs) -> Dict[str, Any]: + mask = targets != self._ignore_label + matrix = self.confusion_matrix_metric(preds[mask], targets[mask]) + segmentation_metrics = self.compute_metrics_from_cm(matrix) + return segmentation_metrics + + def finalise(self): + matrix = self.confusion_matrix_metric.compute() + segmentation_metrics = self.compute_metrics_from_cm(matrix) + return segmentation_metrics + + def reset(self, stage: str = "train"): + super().reset(stage) + + + + From 1746d65888524e0fcc43d5d89438c8e179304b73 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Thu, 22 Jul 2021 17:34:45 +0200 Subject: [PATCH 02/17] rename files --- .../{test_segmentation_metric.py => test_segmentation_tracker.py} | 0 .../{segmentation_metrics.py => segmentation_tracker.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/{test_segmentation_metric.py => test_segmentation_tracker.py} (100%) rename torch_points3d/metrics/segmentation/{segmentation_metrics.py => segmentation_tracker.py} (100%) diff --git a/test/test_segmentation_metric.py b/test/test_segmentation_tracker.py similarity index 100% rename from test/test_segmentation_metric.py rename to test/test_segmentation_tracker.py diff --git a/torch_points3d/metrics/segmentation/segmentation_metrics.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py similarity index 100% rename from torch_points3d/metrics/segmentation/segmentation_metrics.py rename to torch_points3d/metrics/segmentation/segmentation_tracker.py From 8754cf800505ba06a005b3a914cc2ce94420a9bd Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Thu, 22 Jul 2021 19:16:15 +0200 Subject: [PATCH 03/17] add tests and test confusion metrics --- test/test_confusion_matrix.py | 39 ++++++++---- test/test_model.py | 1 + torch_points3d/metrics/base_tracker.py | 62 +++++++++++++++++++ .../segmentation/segmentation_tracker.py | 10 +-- 4 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 torch_points3d/metrics/base_tracker.py diff --git a/test/test_confusion_matrix.py b/test/test_confusion_matrix.py index ad90340..bc58b03 100644 --- a/test/test_confusion_matrix.py +++ b/test/test_confusion_matrix.py @@ -1,12 +1,26 @@ +import torch +import os +import sys import unittest import pytest +import numpy as np +DIR = os.path.dirname(os.path.realpath(__file__)) +ROOT = os.path.join(DIR, "..") +sys.path.insert(0, ROOT) +sys.path.append('.') + +from torch_points3d.metrics.segmentation.segmentation_tracker import compute_intersection_union_per_class +from torch_points3d.metrics.segmentation.segmentation_tracker import compute_average_intersection_union +from torch_points3d.metrics.segmentation.segmentation_tracker import compute_overall_accuracy +from torch_points3d.metrics.segmentation.segmentation_tracker import compute_mean_class_accuracy + class TestConfusionMatrix(unittest.TestCase): def test_compute_intersection_union_per_class(self): matrix = torch.tensor([[4, 1], [2, 10]]) - iou = compute_intersection_union_per_class(matrix) + iou, _ = compute_intersection_union_per_class(matrix) miou = compute_average_intersection_union(matrix) print(iou) self.assertAlmostEqual(iou[0].item(), 4 / (4.0 + 1.0 + 2.0)) @@ -15,9 +29,9 @@ def test_compute_intersection_union_per_class(self): def test_compute_overall_accuracy(self): list_matrix = [ - matrix = torch.tensor([[4, 1], [2, 10]]).float(), - matrix = torch.tensor([[4, 1], [2, 10]]).int(), - matrix = torch.tensor([[0, 0], [0, 0]]).float() + torch.tensor([[4, 1], [2, 10]]).float(), + torch.tensor([[4, 1], [2, 10]]).int(), + torch.tensor([[0, 0], [0, 0]]).float() ] list_answer = [ (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), @@ -26,7 +40,10 @@ def test_compute_overall_accuracy(self): ] for i in range(len(list_matrix)): acc = compute_overall_accuracy(list_matrix[i]) - self.assertAlmostEqual(acc.item(), list_answer[i]) + if(isinstance(acc, torch.Tensor)): + self.assertAlmostEqual(acc.item(), list_answer[i]) + else: + self.assertAlmostEqual(acc, list_answer[i]) def test_compute_mean_class_accuracy(self): @@ -35,11 +52,9 @@ def test_compute_mean_class_accuracy(self): self.assertAlmostEqual(macc.item(), (4/5 + 10/12)*0.5) - @pytest.mark.parametrize(["missing_as_one", "answer"], [ - pytest.param(True, (0.5 + 0.5) / 2,) - pytest.param(False, (0.5 + 1 + 0.5) / 3) - ]) - def test_test_getMeanIoUMissing(self, missing_as_one, answer): - matrix = torch.tensor([[1, 1, 0], [0, 1, 0], [0, 0, 0]]) - self.assertAlmostEqual(compute_average_intersection_union(matrix, missing_as_one=missing_as_one), answer) + +@pytest.mark.parametrize("missing_as_one, answer", [pytest.param(False, (0.5 + 0.5) / 2), pytest.param(True, (0.5 + 1 + 0.5) / 3)]) +def test_test_getMeanIoUMissing(missing_as_one, answer): + matrix = torch.tensor([[1, 1, 0], [0, 1, 0], [0, 0, 0]]) + np.testing.assert_allclose(compute_average_intersection_union(matrix, missing_as_one=missing_as_one).item(), answer) diff --git a/test/test_model.py b/test/test_model.py index fb704ab..140f62e 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -15,6 +15,7 @@ class TestAPIModel(unittest.TestCase): + def test_forward(self): option_dataset = OmegaConf.create({"feature_dimension": 1, "num_classes": 10}) diff --git a/torch_points3d/metrics/base_tracker.py b/torch_points3d/metrics/base_tracker.py new file mode 100644 index 0000000..3c915ac --- /dev/null +++ b/torch_points3d/metrics/base_tracker.py @@ -0,0 +1,62 @@ +from typing import Dict, Any, Callable, Optional +import torch +from torch import nn +from torchmetrics import AverageMeter + + +class BaseTracker(nn.Module): + """ + pytorch Module to manage the losses and the metrics + """ + def __init__(self, stage: str = "train"): + super().__init__() + self.stage: str = stage + self._finalised: bool = False + self.loss_metrics: Dict[str, Callable] = dict() + + def track(self, output_model, *args, **kwargs) -> Dict[str, Any]: + raise NotImplementedError + + def track_loss(self, losses: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + out_loss = dict() + for key, loss in losses.items(): + loss_key = "%s_%s" % (self.stage, key) + if loss_key not in self.loss_metrics: + self.loss_metrics[loss_key] = AverageMeter() + val = self.loss_metrics[loss_key](loss) + out_loss[loss_key] = val + return out_loss + + def forward(self, output_model: Dict[str, Any], losses: Optional[Dict[str, torch.Tensor]] = None, *args, **kwargs) -> Dict[str, Any]: + if self._finalised: + raise RuntimeError("Cannot track new values with a finalised tracker, you need to reset it first") + tracked_metric = self.track(output_model, *args, **kwargs) + if losses is not None: + tracked_loss = self.track_loss(losses) + tracked_results = dict(**tracked_loss, **tracked_metric) + else: + tracked_results = tracked_metric + return tracked_results + + def _finalise(self) -> Dict[str, Any]: + raise NotImplementedError("method that aggregae metrics") + + def finalise(self) -> Dict[str, Any]: + metrics = self._finalise() + self._finalised = True + loss_metrics = self.get_final_loss_metrics() + final_metrics = {**loss_metrics, **metrics} + return final_metrics + + def get_final_loss_metrics(self): + metrics = dict() + for key, m in self.loss_metrics.items(): + metrics[key] = m.compute() + self.loss_metrics = dict() + return metrics + + def reset(self, stage: str = "train"): + self._finalised = False + self.stage = stage + self.loss_metrics = dict() + diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index 6459fce..eab4df7 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -1,10 +1,12 @@ import torch -from typing import Dict, Optional, Tuple, Any +from typing import Dict, Optional, Tuple, Any, Union from torchmetrics import ConfusionMatrix from torchmetrics import Metric +from torch_points3d.metrics.base_tracker import BaseTracker -def compute_average_intersection_union(matrix: torch.Tensor, missing_as_one: bool = False) -> torch.Tensor: + +def compute_average_intersection_union(confusion_matrix: torch.Tensor, missing_as_one: bool = False) -> torch.Tensor: """ compute intersection over union on average from confusion matrix Parameters @@ -42,7 +44,7 @@ def compute_mean_class_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: re = (torch.diagonal(confusion_matrix)[labels_presents].float() / max_ones_total_gts[labels_presents]).sum() return re / float(len(labels_presents)) -def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: +def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> Union[int, torch.Tensor]: """ compute overall accuracy from confusion matrix @@ -116,7 +118,7 @@ def track(targets: torch.Tensor, preds: torch.Tensor, **kwargs) -> Dict[str, Any segmentation_metrics = self.compute_metrics_from_cm(matrix) return segmentation_metrics - def finalise(self): + def _finalise(self): matrix = self.confusion_matrix_metric.compute() segmentation_metrics = self.compute_metrics_from_cm(matrix) return segmentation_metrics From c763bdfbba71fa4bddc1e4946cf3348b997eb42e Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Thu, 22 Jul 2021 19:20:46 +0200 Subject: [PATCH 04/17] function instead of classes in pytest --- test/test_confusion_matrix.py | 67 +++++++++++++++++------------------ 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/test/test_confusion_matrix.py b/test/test_confusion_matrix.py index bc58b03..43bb549 100644 --- a/test/test_confusion_matrix.py +++ b/test/test_confusion_matrix.py @@ -16,40 +16,39 @@ from torch_points3d.metrics.segmentation.segmentation_tracker import compute_overall_accuracy from torch_points3d.metrics.segmentation.segmentation_tracker import compute_mean_class_accuracy -class TestConfusionMatrix(unittest.TestCase): - - def test_compute_intersection_union_per_class(self): - matrix = torch.tensor([[4, 1], [2, 10]]) - iou, _ = compute_intersection_union_per_class(matrix) - miou = compute_average_intersection_union(matrix) - print(iou) - self.assertAlmostEqual(iou[0].item(), 4 / (4.0 + 1.0 + 2.0)) - self.assertAlmostEqual(iou[1].item(), 10 / (10.0 + 1.0 + 2.0)) - self.assertAlmostEqual(iou.mean().item(), miou.item()) - - def test_compute_overall_accuracy(self): - list_matrix = [ - torch.tensor([[4, 1], [2, 10]]).float(), - torch.tensor([[4, 1], [2, 10]]).int(), - torch.tensor([[0, 0], [0, 0]]).float() - ] - list_answer = [ - (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), - (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), - 0.0 - ] - for i in range(len(list_matrix)): - acc = compute_overall_accuracy(list_matrix[i]) - if(isinstance(acc, torch.Tensor)): - self.assertAlmostEqual(acc.item(), list_answer[i]) - else: - self.assertAlmostEqual(acc, list_answer[i]) - - - def test_compute_mean_class_accuracy(self): - matrix = torch.tensor([[4, 1], [2, 10]]).float() - macc = compute_mean_class_accuracy(matrix) - self.assertAlmostEqual(macc.item(), (4/5 + 10/12)*0.5) + + +def test_compute_intersection_union_per_class(): + matrix = torch.tensor([[4, 1], [2, 10]]) + iou, _ = compute_intersection_union_per_class(matrix) + miou = compute_average_intersection_union(matrix) + np.testing.assert_allclose(iou[0].item(), 4 / (4.0 + 1.0 + 2.0)) + np.testing.assert_allclose(iou[1].item(), 10 / (10.0 + 1.0 + 2.0)) + np.testing.assert_allclose(iou.mean().item(), miou.item()) + +def test_compute_overall_accuracy(): + list_matrix = [ + torch.tensor([[4, 1], [2, 10]]).float(), + torch.tensor([[4, 1], [2, 10]]).int(), + torch.tensor([[0, 0], [0, 0]]).float() + ] + list_answer = [ + (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), + (4.0+10.0)/(4.0 + 10.0 + 1.0 +2.0), + 0.0 + ] + for i in range(len(list_matrix)): + acc = compute_overall_accuracy(list_matrix[i]) + if(isinstance(acc, torch.Tensor)): + np.testing.assert_allclose(acc.item(), list_answer[i]) + else: + np.testing.assert_allclose(acc, list_answer[i]) + + +def test_compute_mean_class_accuracy(): + matrix = torch.tensor([[4, 1], [2, 10]]).float() + macc = compute_mean_class_accuracy(matrix) + np.testing.assert_allclose(macc.item(), (4/5 + 10/12)*0.5) From 8b31a4e7643293572b2db767a3827e3793f8d96f Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Thu, 22 Jul 2021 19:29:38 +0200 Subject: [PATCH 05/17] need to fix a test --- test/.#test_model.py | 1 + test/test_segmentation_tracker.py | 55 +++++++++++++++++-- .../segmentation/segmentation_tracker.py | 8 +-- 3 files changed, 56 insertions(+), 8 deletions(-) create mode 120000 test/.#test_model.py diff --git a/test/.#test_model.py b/test/.#test_model.py new file mode 120000 index 0000000..523f8d4 --- /dev/null +++ b/test/.#test_model.py @@ -0,0 +1 @@ +admincaor@admincaor.27515:1624465837 \ No newline at end of file diff --git a/test/test_segmentation_tracker.py b/test/test_segmentation_tracker.py index c0ae704..6aa312b 100644 --- a/test/test_segmentation_tracker.py +++ b/test/test_segmentation_tracker.py @@ -1,10 +1,18 @@ import torch +import sys +import os import unittest import pytest from torch_geometric.data import Data +DIR = os.path.dirname(os.path.realpath(__file__)) +ROOT = os.path.join(DIR, "..") +sys.path.insert(0, ROOT) +sys.path.append('.') + +from torch_points3d.metrics.segmentation.segmentation_tracker import SegmentationTracker class MockDataset: INV_OBJECT_LABEL = {0: "first", 1: "wall", 2: "not", 3: "here", 4: "hoy"} @@ -62,7 +70,46 @@ def device(self): class TestSegmentationMetrics(unittest.TestCase): def test_forward(self): - pass - - def test_finalize(self): - pass + tracker = SegmentationTracker(num_classes=2, stage="train") + model = MockModel() + metrics = tracker(model) + # metrics = tracker.get_metrics() + + for k in ["train_acc", "train_miou", "train_macc"]: + self.assertAlmostEqual(metrics[k], 100, 5) + + model.iter += 1 + metrics = tracker(model) + # metrics = tracker.get_metrics() + metrics = tracker.finalise() + for k in ["train_acc", "train_macc"]: + self.assertEqual(metrics[k], 50) + self.assertAlmostEqual(metrics["train_miou"], 25, 5) + self.assertEqual(metrics["train_loss_1"], 1.5) + + tracker.reset("test") + model.iter += 1 + metrics = tracker(model) + # metrics = tracker.get_metrics() + for k in ["test_acc", "test_miou", "test_macc"]: + self.assertAlmostEqual(metrics[k].item(), 0, 5) + + def test_ignore_label(self): + tracker = SegmentationTracker(num_classes=2, ignore_label=-100) + tracker.reset("test") + model = MockModel() + model.iter = 3 + metrics = tracker(model) + # metrics = tracker.get_metrics() + for k in ["test_acc", "test_miou", "test_macc"]: + self.assertAlmostEqual(metrics[k], 100, 5) + + def test_finalise(self): + tracker = SegmentationTracker(num_classes=2, ignore_label=-100) + tracker.reset("test") + model = MockModel() + model.iter = 3 + tracker(output) + tracker.finalise() + with self.assertRaises(RuntimeError): + tracker(model) diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index eab4df7..ae4a1b8 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -92,7 +92,7 @@ def __init__( stage: str = "train", ignore_label: int = -1, eps: float = 1e-8,): - super().__init__() + super().__init__(stage) self._ignore_label = ignore_label self._num_classes = num_classes self.confusion_matrix_metric = ConfusionMatrix(num_classes=self._num_classes) @@ -112,9 +112,9 @@ def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: } - def track(targets: torch.Tensor, preds: torch.Tensor, **kwargs) -> Dict[str, Any]: - mask = targets != self._ignore_label - matrix = self.confusion_matrix_metric(preds[mask], targets[mask]) + def track(self, output, **kwargs) -> Dict[str, Any]: + mask = output['targets'] != self._ignore_label + matrix = self.confusion_matrix_metric(output['preds'][mask], output['targets'][mask]) segmentation_metrics = self.compute_metrics_from_cm(matrix) return segmentation_metrics From 22cf859c1c062e1f290b85202706926d2e252777 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 17:31:37 +0000 Subject: [PATCH 06/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_model.py | 3 +- test/test_segmentation_tracker.py | 3 +- torch_points3d/metrics/base_tracker.py | 8 +++-- .../segmentation/segmentation_tracker.py | 34 +++++++++---------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/test/test_model.py b/test/test_model.py index 140f62e..803273b 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -9,13 +9,12 @@ DIR = os.path.dirname(os.path.realpath(__file__)) ROOT = os.path.join(DIR, "..") sys.path.insert(0, ROOT) -sys.path.append('.') +sys.path.append(".") from torch_points3d.models.segmentation.sparseconv3d import APIModel class TestAPIModel(unittest.TestCase): - def test_forward(self): option_dataset = OmegaConf.create({"feature_dimension": 1, "num_classes": 10}) diff --git a/test/test_segmentation_tracker.py b/test/test_segmentation_tracker.py index 6aa312b..37aa8e8 100644 --- a/test/test_segmentation_tracker.py +++ b/test/test_segmentation_tracker.py @@ -10,10 +10,11 @@ DIR = os.path.dirname(os.path.realpath(__file__)) ROOT = os.path.join(DIR, "..") sys.path.insert(0, ROOT) -sys.path.append('.') +sys.path.append(".") from torch_points3d.metrics.segmentation.segmentation_tracker import SegmentationTracker + class MockDataset: INV_OBJECT_LABEL = {0: "first", 1: "wall", 2: "not", 3: "here", 4: "hoy"} pos = torch.tensor([[1, 0, 0], [2, 0, 0], [3, 0, 0], [-1, 0, 0]]).float() diff --git a/torch_points3d/metrics/base_tracker.py b/torch_points3d/metrics/base_tracker.py index 3c915ac..a53154f 100644 --- a/torch_points3d/metrics/base_tracker.py +++ b/torch_points3d/metrics/base_tracker.py @@ -8,13 +8,14 @@ class BaseTracker(nn.Module): """ pytorch Module to manage the losses and the metrics """ + def __init__(self, stage: str = "train"): super().__init__() self.stage: str = stage self._finalised: bool = False self.loss_metrics: Dict[str, Callable] = dict() - def track(self, output_model, *args, **kwargs) -> Dict[str, Any]: + def track(self, output_model, *args, **kwargs) -> Dict[str, Any]: raise NotImplementedError def track_loss(self, losses: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: @@ -27,7 +28,9 @@ def track_loss(self, losses: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor] out_loss[loss_key] = val return out_loss - def forward(self, output_model: Dict[str, Any], losses: Optional[Dict[str, torch.Tensor]] = None, *args, **kwargs) -> Dict[str, Any]: + def forward( + self, output_model: Dict[str, Any], losses: Optional[Dict[str, torch.Tensor]] = None, *args, **kwargs + ) -> Dict[str, Any]: if self._finalised: raise RuntimeError("Cannot track new values with a finalised tracker, you need to reset it first") tracked_metric = self.track(output_model, *args, **kwargs) @@ -59,4 +62,3 @@ def reset(self, stage: str = "train"): self._finalised = False self.stage = stage self.loss_metrics = dict() - diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index ae4a1b8..ab40a34 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -29,7 +29,7 @@ def compute_average_intersection_union(confusion_matrix: torch.Tensor, missing_a def compute_mean_class_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: """ compute intersection over union on average from confusion matrix - + Parameters ---------- confusion_matrix: torch.Tensor @@ -37,17 +37,18 @@ def compute_mean_class_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: """ total_gts = confusion_matrix.sum(1) labels_presents = torch.where(total_gts > 0)[0] - if(len(labels_presents) == 0): + if len(labels_presents) == 0: return total_gts[0] ones = torch.ones_like(total_gts) max_ones_total_gts = torch.cat([total_gts[None, :], ones[None, :]], 0).max(0)[0] re = (torch.diagonal(confusion_matrix)[labels_presents].float() / max_ones_total_gts[labels_presents]).sum() return re / float(len(labels_presents)) + def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> Union[int, torch.Tensor]: """ compute overall accuracy from confusion matrix - + Parameters ---------- confusion_matrix: torch.Tensor @@ -59,10 +60,13 @@ def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> Union[int, torch matrix_diagonal = torch.trace(confusion_matrix) return matrix_diagonal.float() / all_values -def compute_intersection_union_per_class(confusion_matrix: torch.Tensor, return_existing_mask: bool = False, eps: float = 1e-8) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + +def compute_intersection_union_per_class( + confusion_matrix: torch.Tensor, return_existing_mask: bool = False, eps: float = 1e-8 +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: """ compute intersection over union per class from confusion matrix - + Parameters ---------- confusion_matrix: torch.Tensor @@ -81,17 +85,18 @@ def compute_intersection_union_per_class(confusion_matrix: torch.Tensor, return_ return iou, None - class SegmentationTracker(BaseTracker): """ track different registration metrics """ def __init__( - self, num_classes: int, - stage: str = "train", - ignore_label: int = -1, - eps: float = 1e-8,): + self, + num_classes: int, + stage: str = "train", + ignore_label: int = -1, + eps: float = 1e-8, + ): super().__init__(stage) self._ignore_label = ignore_label self._num_classes = num_classes @@ -111,10 +116,9 @@ def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: "{}_iou_per_class".format(self.stage): iou_per_class_dict, } - def track(self, output, **kwargs) -> Dict[str, Any]: - mask = output['targets'] != self._ignore_label - matrix = self.confusion_matrix_metric(output['preds'][mask], output['targets'][mask]) + mask = output["targets"] != self._ignore_label + matrix = self.confusion_matrix_metric(output["preds"][mask], output["targets"][mask]) segmentation_metrics = self.compute_metrics_from_cm(matrix) return segmentation_metrics @@ -125,7 +129,3 @@ def _finalise(self): def reset(self, stage: str = "train"): super().reset(stage) - - - - From b69912d9e449e94f93004965768d0448bdb54640 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Thu, 22 Jul 2021 19:32:15 +0200 Subject: [PATCH 07/17] remove useless --- test/.#test_model.py | 1 - 1 file changed, 1 deletion(-) delete mode 120000 test/.#test_model.py diff --git a/test/.#test_model.py b/test/.#test_model.py deleted file mode 120000 index 523f8d4..0000000 --- a/test/.#test_model.py +++ /dev/null @@ -1 +0,0 @@ -admincaor@admincaor.27515:1624465837 \ No newline at end of file From f487a63df95565e75f5820c1e089a3bcedd03b07 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Fri, 23 Jul 2021 15:30:12 +0200 Subject: [PATCH 08/17] tests work using pytest --- test/test_segmentation_tracker.py | 92 +++++++++++++++++-------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/test/test_segmentation_tracker.py b/test/test_segmentation_tracker.py index 37aa8e8..e959344 100644 --- a/test/test_segmentation_tracker.py +++ b/test/test_segmentation_tracker.py @@ -1,3 +1,4 @@ +import numpy as np import torch import sys import os @@ -5,6 +6,7 @@ import unittest import pytest + from torch_geometric.data import Data DIR = os.path.dirname(os.path.realpath(__file__)) @@ -69,48 +71,54 @@ def device(self): return "cpu" -class TestSegmentationMetrics(unittest.TestCase): - def test_forward(self): - tracker = SegmentationTracker(num_classes=2, stage="train") - model = MockModel() - metrics = tracker(model) - # metrics = tracker.get_metrics() - - for k in ["train_acc", "train_miou", "train_macc"]: - self.assertAlmostEqual(metrics[k], 100, 5) - - model.iter += 1 - metrics = tracker(model) - # metrics = tracker.get_metrics() - metrics = tracker.finalise() - for k in ["train_acc", "train_macc"]: - self.assertEqual(metrics[k], 50) - self.assertAlmostEqual(metrics["train_miou"], 25, 5) - self.assertEqual(metrics["train_loss_1"], 1.5) - - tracker.reset("test") - model.iter += 1 - metrics = tracker(model) +def test_forward(): + tracker = SegmentationTracker(num_classes=2, stage="train") + model = MockModel() + output = {"preds": model.get_output(), "targets": model.get_labels()} + losses = model.get_current_losses() + metrics = tracker(output, losses) + # metrics = tracker.get_metrics() + + for k in ["train_acc", "train_miou", "train_macc"]: + np.testing.assert_allclose(metrics[k], 100, rtol=1e-5) + model.iter += 1 + output = {"preds": model.get_output(), "targets": model.get_labels()} + losses = model.get_current_losses() + metrics = tracker(output, losses) + # metrics = tracker.get_metrics() + metrics = tracker.finalise() + for k in ["train_acc", "train_macc"]: + assert metrics[k] == 50 + np.testing.assert_allclose(metrics["train_miou"], 25, atol=1e-5) + assert metrics["train_loss_1"] == 1.5 + + tracker.reset("test") + model.iter += 1 + output = {"preds": model.get_output(), "targets": model.get_labels()} + losses = model.get_current_losses() + metrics = tracker(output, losses) + # metrics = tracker.get_metrics() + for name in ["test_acc", "test_miou", "test_macc"]: + np.testing.assert_allclose(metrics[name].item(), 0, atol=1e-5) + + +@pytest.mark.parametrize("finalise", [pytest.param(True), pytest.param(False)]) +def test_ignore_label(finalise): + tracker = SegmentationTracker(num_classes=2, ignore_label=-100) + tracker.reset("test") + model = MockModel() + model.iter = 3 + output = {"preds": model.get_output(), "targets": model.get_labels()} + losses = model.get_current_losses() + metrics = tracker(output, losses) + if not finalise: # metrics = tracker.get_metrics() for k in ["test_acc", "test_miou", "test_macc"]: - self.assertAlmostEqual(metrics[k].item(), 0, 5) - - def test_ignore_label(self): - tracker = SegmentationTracker(num_classes=2, ignore_label=-100) - tracker.reset("test") - model = MockModel() - model.iter = 3 - metrics = tracker(model) - # metrics = tracker.get_metrics() - for k in ["test_acc", "test_miou", "test_macc"]: - self.assertAlmostEqual(metrics[k], 100, 5) - - def test_finalise(self): - tracker = SegmentationTracker(num_classes=2, ignore_label=-100) - tracker.reset("test") - model = MockModel() - model.iter = 3 - tracker(output) + np.testing.assert_allclose(metrics[k], 100) + else: tracker.finalise() - with self.assertRaises(RuntimeError): - tracker(model) + with pytest.raises(RuntimeError): + tracker(output) + + + From 32a9f5e5494f18aa9cbc2e0f33679e432d3192fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:30:28 +0000 Subject: [PATCH 09/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_segmentation_tracker.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/test_segmentation_tracker.py b/test/test_segmentation_tracker.py index e959344..09d2385 100644 --- a/test/test_segmentation_tracker.py +++ b/test/test_segmentation_tracker.py @@ -3,7 +3,6 @@ import sys import os -import unittest import pytest @@ -78,7 +77,7 @@ def test_forward(): losses = model.get_current_losses() metrics = tracker(output, losses) # metrics = tracker.get_metrics() - + for k in ["train_acc", "train_miou", "train_macc"]: np.testing.assert_allclose(metrics[k], 100, rtol=1e-5) model.iter += 1 @@ -119,6 +118,3 @@ def test_ignore_label(finalise): tracker.finalise() with pytest.raises(RuntimeError): tracker(output) - - - From fc4fa2ccf2b9ed5f575bf6f4d75484277ea5489b Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Tue, 27 Jul 2021 17:46:34 +0200 Subject: [PATCH 10/17] integrate metrics in the training --- conf/model/segmentation/default.yaml | 3 +- conf/tracker/segmentation/default.yaml | 2 + test/test_model.py | 45 +++++++++---------- test/test_segmentation_tracker.py | 8 ++-- torch_points3d/core/instantiator.py | 3 ++ .../segmentation/segmentation_tracker.py | 6 +-- torch_points3d/models/base_model.py | 22 +++++++-- .../models/segmentation/base_model.py | 35 +++++++++------ torch_points3d/tasks/base_model.py | 21 ++++++--- 9 files changed, 91 insertions(+), 54 deletions(-) create mode 100644 conf/tracker/segmentation/default.yaml diff --git a/conf/model/segmentation/default.yaml b/conf/model/segmentation/default.yaml index 349caa4..ebf3207 100644 --- a/conf/model/segmentation/default.yaml +++ b/conf/model/segmentation/default.yaml @@ -1,6 +1,7 @@ # @package model defaults: - /model/default + - /tracker: segmentation/default model: _recursive_: false @@ -11,4 +12,4 @@ model: backbone: input_nc: ${dataset.cfg.feature_dimension} - architecture: unet \ No newline at end of file + architecture: unet diff --git a/conf/tracker/segmentation/default.yaml b/conf/tracker/segmentation/default.yaml new file mode 100644 index 0000000..81e6f05 --- /dev/null +++ b/conf/tracker/segmentation/default.yaml @@ -0,0 +1,2 @@ +_target_: torch_points3d.metrics.segmentation.segmentation_tracker.SegmentationTracker +num_classes: ${dataset.cfg.num_classes} diff --git a/test/test_model.py b/test/test_model.py index 803273b..e71696f 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -1,4 +1,4 @@ -import unittest +import pytest import sys import os import torch @@ -11,26 +11,23 @@ sys.path.insert(0, ROOT) sys.path.append(".") -from torch_points3d.models.segmentation.sparseconv3d import APIModel - - -class TestAPIModel(unittest.TestCase): - def test_forward(self): - option_dataset = OmegaConf.create({"feature_dimension": 1, "num_classes": 10}) - - option = OmegaConf.load(os.path.join(ROOT, "conf", "models", "segmentation", "sparseconv3d.yaml")) - name_model = list(option.keys())[0] - model = APIModel(option[name_model], option_dataset) - - pos = torch.randn(1000, 3) - coords = torch.round(pos * 10000) - x = torch.ones(1000, 1) - batch = torch.zeros(1000).long() - y = torch.randint(0, 10, (1000,)) - data = Batch(pos=pos, x=x, batch=batch, y=y, coords=coords) - model.set_input(data) - model.forward() - - -if __name__ == "__main__": - unittest.main() +from torch_points3d.models.segmentation.base_model import SegmentationBaseModel +from torch_points3d.core.instantiator import HydraInstantiator + + +@pytest.mark.skip("For now we skip the tests...") +def test_forward(self): + option_dataset = OmegaConf.create({"feature_dimension": 1, "num_classes": 10}) + option_criterion = OmegaConf.create({"_target_": "torch.nn.NLLLoss"}) + instantiator = HydraInstantiator() + + model = SegmentationBaseModel(instantiator, 10, option_backbone, option_criterion) + + pos = torch.randn(1000, 3) + coords = torch.round(pos * 10000) + x = torch.ones(1000, 6) + batch = torch.zeros(1000).long() + y = torch.randint(0, 10, (1000,)) + data = Batch(pos=pos, x=x, batch=batch, y=y, coords=coords) + model.set_input(data) + model.forward() diff --git a/test/test_segmentation_tracker.py b/test/test_segmentation_tracker.py index 09d2385..cb54eb6 100644 --- a/test/test_segmentation_tracker.py +++ b/test/test_segmentation_tracker.py @@ -73,7 +73,7 @@ def device(self): def test_forward(): tracker = SegmentationTracker(num_classes=2, stage="train") model = MockModel() - output = {"preds": model.get_output(), "targets": model.get_labels()} + output = {"preds": model.get_output(), "labels": model.get_labels()} losses = model.get_current_losses() metrics = tracker(output, losses) # metrics = tracker.get_metrics() @@ -81,7 +81,7 @@ def test_forward(): for k in ["train_acc", "train_miou", "train_macc"]: np.testing.assert_allclose(metrics[k], 100, rtol=1e-5) model.iter += 1 - output = {"preds": model.get_output(), "targets": model.get_labels()} + output = {"preds": model.get_output(), "labels": model.get_labels()} losses = model.get_current_losses() metrics = tracker(output, losses) # metrics = tracker.get_metrics() @@ -93,7 +93,7 @@ def test_forward(): tracker.reset("test") model.iter += 1 - output = {"preds": model.get_output(), "targets": model.get_labels()} + output = {"preds": model.get_output(), "labels": model.get_labels()} losses = model.get_current_losses() metrics = tracker(output, losses) # metrics = tracker.get_metrics() @@ -107,7 +107,7 @@ def test_ignore_label(finalise): tracker.reset("test") model = MockModel() model.iter = 3 - output = {"preds": model.get_output(), "targets": model.get_labels()} + output = {"preds": model.get_output(), "labels": model.get_labels()} losses = model.get_current_losses() metrics = tracker(output, losses) if not finalise: diff --git a/torch_points3d/core/instantiator.py b/torch_points3d/core/instantiator.py index e4b3d04..3be6d90 100644 --- a/torch_points3d/core/instantiator.py +++ b/torch_points3d/core/instantiator.py @@ -44,6 +44,9 @@ def litmodel(self, cfg: DictConfig) -> "PointCloudBaseModule": def model(self, cfg: DictConfig) -> "PointCloudBaseModel": return self.instantiate(cfg, self) + def tracker(self, cfg: DictConfig, stage: str = ""): + return self.instantiate(cfg, stage=stage) + def backbone(self, cfg: DictConfig): return self.instantiate(cfg) diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index ab40a34..c86a53f 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -108,7 +108,7 @@ def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: macc = compute_mean_class_accuracy(matrix) miou = compute_average_intersection_union(matrix) iou_per_class, _ = compute_intersection_union_per_class(matrix, eps=self.eps) - iou_per_class_dict = {i: 100 * v for i, v in enumerate(iou_per_class)} + iou_per_class_dict = {i: (100 * v).item() for i, v in enumerate(iou_per_class)} return { "{}_acc".format(self.stage): 100 * acc, "{}_macc".format(self.stage): 100 * macc, @@ -117,8 +117,8 @@ def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: } def track(self, output, **kwargs) -> Dict[str, Any]: - mask = output["targets"] != self._ignore_label - matrix = self.confusion_matrix_metric(output["preds"][mask], output["targets"][mask]) + mask = output["labels"] != self._ignore_label + matrix = self.confusion_matrix_metric(output["preds"][mask], output["labels"][mask]) segmentation_metrics = self.compute_metrics_from_cm(matrix) return segmentation_metrics diff --git a/torch_points3d/models/base_model.py b/torch_points3d/models/base_model.py index a800fac..7115c15 100644 --- a/torch_points3d/models/base_model.py +++ b/torch_points3d/models/base_model.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Union, Optional, Dict import torch import torch.nn as nn @@ -12,12 +12,28 @@ def __init__(self, instantiator: Instantiator): super().__init__() self.instantiator = instantiator + self._losses: Dict[str, float] = {} def set_input(self, data: Data) -> None: raise (NotImplementedError("set_input needs to be defined!")) - def forward(self) -> Union[torch.Tensor, None]: + def forward(self) -> Optional[torch.Tensor]: raise (NotImplementedError("forward needs to be defined!")) - def get_losses(self) -> Union[torch.Tensor, None]: + def compute_loss(self): raise (NotImplementedError("get_losses needs to be defined!")) + + def get_losses(self, stage: Optional[str] = None) -> Optional[Dict["str", torch.Tensor]]: + if stage is None: + return self._losses + else: + losses = {} + for name, l in self._losses.items(): + losses[f"{stage}_{name}"] = l.item() + return losses + + def get_outputs(self) -> Dict[str, Optional[torch.Tensor]]: + """ + return the outputs to track for the metrics + """ + raise (NotImplementedError("outputs need to be defined")) diff --git a/torch_points3d/models/segmentation/base_model.py b/torch_points3d/models/segmentation/base_model.py index 8e11465..0ffaa7b 100644 --- a/torch_points3d/models/segmentation/base_model.py +++ b/torch_points3d/models/segmentation/base_model.py @@ -1,5 +1,5 @@ from omegaconf import DictConfig -from typing import Union +from typing import Union, Optional, Dict import torch import torch.nn as nn @@ -28,17 +28,26 @@ def set_input(self, data: Data) -> None: else: self.labels = None - def forward(self) -> Union[torch.Tensor, None]: + def forward(self) -> Optional[torch.Tensor]: features = self.backbone(self.input).x logits = self.head(features) - self.output = F.log_softmax(logits, dim=-1) - - return self.get_losses() - - def get_losses(self) -> Union[torch.Tensor, None]: - # only compute loss if loss is defined and the dset has labels - if self.labels is None or self.criterion is None: - return - - self.loss = self.criterion(self.output, self.labels) - return self.loss + self._output = F.log_softmax(logits, dim=-1) + self.compute_losses() + if "loss" in self._losses.keys(): + return self._losses["loss"] + else: + return None + + def compute_losses(self): + """ + compute every loss. store the total loss in an attribute _loss + """ + if self.labels is not None and self.criterion is not None: + self._losses["loss"] = self.criterion(self._output, self.labels) + + def get_outputs(self) -> Dict[str, torch.Tensor]: + """ + return the outputs to track for the metrics + """ + return {"labels": self.labels, "preds": self._output} + diff --git a/torch_points3d/tasks/base_model.py b/torch_points3d/tasks/base_model.py index 339678c..e00eb3b 100644 --- a/torch_points3d/tasks/base_model.py +++ b/torch_points3d/tasks/base_model.py @@ -16,12 +16,14 @@ def __init__( optimizer: OptimizerConfig, instantiator: Instantiator, scheduler: SchedulerConfig = None, # scheduler shouldn't be required + tracker: Optional[DictConfig] = None ): super().__init__() # some optimizers/schedulers need parameters only known dynamically # allow users to override the getter to instantiate them lazily self.optimizer_cfg = optimizer self.scheduler_cfg = scheduler + self.tracker_cfg = tracker self.instantiator = instantiator self._init_model(model) @@ -91,15 +93,22 @@ def configure_metrics(self, stage: str) -> Optional[Any]: This is called on fit start to have access to the data module, and initialize any data specific metrics. """ + self.tracker = self.instantiator.tracker(self.tracker_cfg) - def training_step(self, batch, batch_idx): + def _step(self, batch, batch_idx, stage: str): self.model.set_input(batch) - return self.model.forward() + loss = self.model.forward() + losses = self.model.get_losses(stage=stage) + outputs = self.model.get_outputs() + metric_dict = self.tracker(outputs, losses) + self.log_dict(metric_dict, prog_bar=True, on_step=False, on_epoch=True) + return loss + + def training_step(self, batch, batch_idx): + return self._step(batch, batch_idx, "train") def validation_step(self, batch, batch_idx): - self.model.set_input(batch) - return self.model.forward() + return self._step(batch, batch_idx, "val") def testing_step(self, batch, batch_idx): - self.model.set_input(batch) - return self.model.forward() + return self._step(batch, batch_idx, "test") From 10c40ebd1cf19bfeb070f1d8d643c0b9f83029cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:46:48 +0000 Subject: [PATCH 11/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_model.py | 4 ++-- torch_points3d/models/base_model.py | 4 ++-- torch_points3d/models/segmentation/base_model.py | 3 +-- torch_points3d/tasks/base_model.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/test/test_model.py b/test/test_model.py index e71696f..b97368b 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -18,9 +18,9 @@ @pytest.mark.skip("For now we skip the tests...") def test_forward(self): option_dataset = OmegaConf.create({"feature_dimension": 1, "num_classes": 10}) - option_criterion = OmegaConf.create({"_target_": "torch.nn.NLLLoss"}) + option_criterion = OmegaConf.create({"_target_": "torch.nn.NLLLoss"}) instantiator = HydraInstantiator() - + model = SegmentationBaseModel(instantiator, 10, option_backbone, option_criterion) pos = torch.randn(1000, 3) diff --git a/torch_points3d/models/base_model.py b/torch_points3d/models/base_model.py index 7115c15..c08bc88 100644 --- a/torch_points3d/models/base_model.py +++ b/torch_points3d/models/base_model.py @@ -1,4 +1,4 @@ -from typing import Union, Optional, Dict +from typing import Dict, Optional import torch import torch.nn as nn @@ -12,7 +12,7 @@ def __init__(self, instantiator: Instantiator): super().__init__() self.instantiator = instantiator - self._losses: Dict[str, float] = {} + self._losses: Dict[str, float] = {} def set_input(self, data: Data) -> None: raise (NotImplementedError("set_input needs to be defined!")) diff --git a/torch_points3d/models/segmentation/base_model.py b/torch_points3d/models/segmentation/base_model.py index 0ffaa7b..04d075c 100644 --- a/torch_points3d/models/segmentation/base_model.py +++ b/torch_points3d/models/segmentation/base_model.py @@ -1,5 +1,5 @@ from omegaconf import DictConfig -from typing import Union, Optional, Dict +from typing import Dict, Optional import torch import torch.nn as nn @@ -50,4 +50,3 @@ def get_outputs(self) -> Dict[str, torch.Tensor]: return the outputs to track for the metrics """ return {"labels": self.labels, "preds": self._output} - diff --git a/torch_points3d/tasks/base_model.py b/torch_points3d/tasks/base_model.py index e00eb3b..b2c7e29 100644 --- a/torch_points3d/tasks/base_model.py +++ b/torch_points3d/tasks/base_model.py @@ -16,7 +16,7 @@ def __init__( optimizer: OptimizerConfig, instantiator: Instantiator, scheduler: SchedulerConfig = None, # scheduler shouldn't be required - tracker: Optional[DictConfig] = None + tracker: Optional[DictConfig] = None, ): super().__init__() # some optimizers/schedulers need parameters only known dynamically From e922d8484ccab3e0af73f85c340f45f235263693 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Wed, 28 Jul 2021 11:55:20 +0200 Subject: [PATCH 12/17] solve problem tensor --- .../metrics/segmentation/segmentation_tracker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index c86a53f..aba02fa 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -108,13 +108,14 @@ def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: macc = compute_mean_class_accuracy(matrix) miou = compute_average_intersection_union(matrix) iou_per_class, _ = compute_intersection_union_per_class(matrix, eps=self.eps) - iou_per_class_dict = {i: (100 * v).item() for i, v in enumerate(iou_per_class)} - return { + iou_per_class_dict = {f"{self.stage}_iou_class_{i}": (100 * v) for i, v in enumerate(iou_per_class)} + res = { "{}_acc".format(self.stage): 100 * acc, "{}_macc".format(self.stage): 100 * macc, "{}_miou".format(self.stage): 100 * miou, - "{}_iou_per_class".format(self.stage): iou_per_class_dict, } + res = dict(**res, **iou_per_class_dict) + return res def track(self, output, **kwargs) -> Dict[str, Any]: mask = output["labels"] != self._ignore_label From 043d8d7c91b0c46c5935dbe076174b92067831c0 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Wed, 28 Jul 2021 12:07:23 +0200 Subject: [PATCH 13/17] use ModuleDict instead of dict --- torch_points3d/metrics/base_tracker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torch_points3d/metrics/base_tracker.py b/torch_points3d/metrics/base_tracker.py index a53154f..0aefead 100644 --- a/torch_points3d/metrics/base_tracker.py +++ b/torch_points3d/metrics/base_tracker.py @@ -13,7 +13,7 @@ def __init__(self, stage: str = "train"): super().__init__() self.stage: str = stage self._finalised: bool = False - self.loss_metrics: Dict[str, Callable] = dict() + self.loss_metrics: nn.ModuleDict = nn.ModuleDict() def track(self, output_model, *args, **kwargs) -> Dict[str, Any]: raise NotImplementedError @@ -55,10 +55,10 @@ def get_final_loss_metrics(self): metrics = dict() for key, m in self.loss_metrics.items(): metrics[key] = m.compute() - self.loss_metrics = dict() + self.loss_metrics = nn.ModuleDict() return metrics def reset(self, stage: str = "train"): self._finalised = False self.stage = stage - self.loss_metrics = dict() + self.loss_metrics = nn.ModuleDict() From 545f74dad4edd284eec667cda9c37a1681c251ec Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Wed, 28 Jul 2021 18:29:12 +0200 Subject: [PATCH 14/17] change based model --- torch_points3d/metrics/__init__.py | 0 .../metrics/segmentation/__init__.py | 0 .../metrics/segmentation/metrics.py | 81 +++++++++++++++++ .../segmentation/segmentation_tracker.py | 88 ++----------------- torch_points3d/models/base_model.py | 11 +-- .../models/segmentation/base_model.py | 10 +-- 6 files changed, 94 insertions(+), 96 deletions(-) create mode 100644 torch_points3d/metrics/__init__.py create mode 100644 torch_points3d/metrics/segmentation/__init__.py create mode 100644 torch_points3d/metrics/segmentation/metrics.py diff --git a/torch_points3d/metrics/__init__.py b/torch_points3d/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/torch_points3d/metrics/segmentation/__init__.py b/torch_points3d/metrics/segmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/torch_points3d/metrics/segmentation/metrics.py b/torch_points3d/metrics/segmentation/metrics.py new file mode 100644 index 0000000..dd6cd48 --- /dev/null +++ b/torch_points3d/metrics/segmentation/metrics.py @@ -0,0 +1,81 @@ +import torch +from typing import Dict, Optional, Tuple, Any, Union + + +def compute_average_intersection_union(confusion_matrix: torch.Tensor, missing_as_one: bool = False) -> torch.Tensor: + """ + compute intersection over union on average from confusion matrix + Parameters + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + missing_as_one: bool, default: False + """ + + values, existing_classes_mask = compute_intersection_union_per_class(confusion_matrix, return_existing_mask=True) + if torch.sum(existing_classes_mask) == 0: + return torch.sum(existing_classes_mask) + if missing_as_one: + values[~existing_classes_mask] = 1 + existing_classes_mask[:] = True + return torch.sum(values[existing_classes_mask]) / torch.sum(existing_classes_mask) + + +def compute_mean_class_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: + """ + compute intersection over union on average from confusion matrix + + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + """ + total_gts = confusion_matrix.sum(1) + labels_presents = torch.where(total_gts > 0)[0] + if len(labels_presents) == 0: + return total_gts[0] + ones = torch.ones_like(total_gts) + max_ones_total_gts = torch.cat([total_gts[None, :], ones[None, :]], 0).max(0)[0] + re = (torch.diagonal(confusion_matrix)[labels_presents].float() / max_ones_total_gts[labels_presents]).sum() + return re / float(len(labels_presents)) + + +def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> Union[int, torch.Tensor]: + """ + compute overall accuracy from confusion matrix + + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + """ + all_values = confusion_matrix.sum() + if all_values == 0: + return 0 + matrix_diagonal = torch.trace(confusion_matrix) + return matrix_diagonal.float() / all_values + + +def compute_intersection_union_per_class( + confusion_matrix: torch.Tensor, return_existing_mask: bool = False, eps: float = 1e-8 +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + compute intersection over union per class from confusion matrix + + Parameters + ---------- + confusion_matrix: torch.Tensor + square matrix + """ + + TP_plus_FN = confusion_matrix.sum(0) + TP_plus_FP = confusion_matrix.sum(1) + TP = torch.diagonal(confusion_matrix) + union = TP_plus_FN + TP_plus_FP - TP + iou = eps + TP / (union + eps) + existing_class_mask = union > 1e-3 + if return_existing_mask: + return iou, existing_class_mask + else: + return iou, None diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index aba02fa..4137772 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -4,85 +4,7 @@ from torchmetrics import Metric from torch_points3d.metrics.base_tracker import BaseTracker - - -def compute_average_intersection_union(confusion_matrix: torch.Tensor, missing_as_one: bool = False) -> torch.Tensor: - """ - compute intersection over union on average from confusion matrix - Parameters - Parameters - ---------- - confusion_matrix: torch.Tensor - square matrix - missing_as_one: bool, default: False - """ - - values, existing_classes_mask = compute_intersection_union_per_class(confusion_matrix, return_existing_mask=True) - if torch.sum(existing_classes_mask) == 0: - return torch.sum(existing_classes_mask) - if missing_as_one: - values[~existing_classes_mask] = 1 - existing_classes_mask[:] = True - return torch.sum(values[existing_classes_mask]) / torch.sum(existing_classes_mask) - - -def compute_mean_class_accuracy(confusion_matrix: torch.Tensor) -> torch.Tensor: - """ - compute intersection over union on average from confusion matrix - - Parameters - ---------- - confusion_matrix: torch.Tensor - square matrix - """ - total_gts = confusion_matrix.sum(1) - labels_presents = torch.where(total_gts > 0)[0] - if len(labels_presents) == 0: - return total_gts[0] - ones = torch.ones_like(total_gts) - max_ones_total_gts = torch.cat([total_gts[None, :], ones[None, :]], 0).max(0)[0] - re = (torch.diagonal(confusion_matrix)[labels_presents].float() / max_ones_total_gts[labels_presents]).sum() - return re / float(len(labels_presents)) - - -def compute_overall_accuracy(confusion_matrix: torch.Tensor) -> Union[int, torch.Tensor]: - """ - compute overall accuracy from confusion matrix - - Parameters - ---------- - confusion_matrix: torch.Tensor - square matrix - """ - all_values = confusion_matrix.sum() - if all_values == 0: - return 0 - matrix_diagonal = torch.trace(confusion_matrix) - return matrix_diagonal.float() / all_values - - -def compute_intersection_union_per_class( - confusion_matrix: torch.Tensor, return_existing_mask: bool = False, eps: float = 1e-8 -) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """ - compute intersection over union per class from confusion matrix - - Parameters - ---------- - confusion_matrix: torch.Tensor - square matrix - """ - - TP_plus_FN = confusion_matrix.sum(0) - TP_plus_FP = confusion_matrix.sum(1) - TP = torch.diagonal(confusion_matrix) - union = TP_plus_FN + TP_plus_FP - TP - iou = eps + TP / (union + eps) - existing_class_mask = union > 1e-3 - if return_existing_mask: - return iou, existing_class_mask - else: - return iou, None +import torch_points3d.metrics.segmentation.metrics as mt class SegmentationTracker(BaseTracker): @@ -104,10 +26,10 @@ def __init__( self.eps = eps def compute_metrics_from_cm(self, matrix: torch.Tensor) -> Dict[str, Any]: - acc = compute_overall_accuracy(matrix) - macc = compute_mean_class_accuracy(matrix) - miou = compute_average_intersection_union(matrix) - iou_per_class, _ = compute_intersection_union_per_class(matrix, eps=self.eps) + acc = mt.compute_overall_accuracy(matrix) + macc = mt.compute_mean_class_accuracy(matrix) + miou = mt.compute_average_intersection_union(matrix) + iou_per_class, _ = mt.compute_intersection_union_per_class(matrix, eps=self.eps) iou_per_class_dict = {f"{self.stage}_iou_class_{i}": (100 * v) for i, v in enumerate(iou_per_class)} res = { "{}_acc".format(self.stage): 100 * acc, diff --git a/torch_points3d/models/base_model.py b/torch_points3d/models/base_model.py index c08bc88..0ed49e9 100644 --- a/torch_points3d/models/base_model.py +++ b/torch_points3d/models/base_model.py @@ -23,14 +23,9 @@ def forward(self) -> Optional[torch.Tensor]: def compute_loss(self): raise (NotImplementedError("get_losses needs to be defined!")) - def get_losses(self, stage: Optional[str] = None) -> Optional[Dict["str", torch.Tensor]]: - if stage is None: - return self._losses - else: - losses = {} - for name, l in self._losses.items(): - losses[f"{stage}_{name}"] = l.item() - return losses + def get_losses(self) -> Optional[Dict["str", torch.Tensor]]: + return self._losses + def get_outputs(self) -> Dict[str, Optional[torch.Tensor]]: """ diff --git a/torch_points3d/models/segmentation/base_model.py b/torch_points3d/models/segmentation/base_model.py index 04d075c..42ac312 100644 --- a/torch_points3d/models/segmentation/base_model.py +++ b/torch_points3d/models/segmentation/base_model.py @@ -32,11 +32,8 @@ def forward(self) -> Optional[torch.Tensor]: features = self.backbone(self.input).x logits = self.head(features) self._output = F.log_softmax(logits, dim=-1) - self.compute_losses() - if "loss" in self._losses.keys(): - return self._losses["loss"] - else: - return None + loss = self.compute_losses() + return loss def compute_losses(self): """ @@ -44,6 +41,9 @@ def compute_losses(self): """ if self.labels is not None and self.criterion is not None: self._losses["loss"] = self.criterion(self._output, self.labels) + return self._losses["loss"] + else: + return None def get_outputs(self) -> Dict[str, torch.Tensor]: """ From 0f052d5cf44416fe7621e0e3fdbbfe3691d75625 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Wed, 28 Jul 2021 18:46:50 +0200 Subject: [PATCH 15/17] test + metrics --- test/test_confusion_matrix.py | 8 ++++---- torch_points3d/metrics/base_tracker.py | 6 +++--- torch_points3d/tasks/base_model.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_confusion_matrix.py b/test/test_confusion_matrix.py index 43bb549..2a04757 100644 --- a/test/test_confusion_matrix.py +++ b/test/test_confusion_matrix.py @@ -11,10 +11,10 @@ sys.path.insert(0, ROOT) sys.path.append('.') -from torch_points3d.metrics.segmentation.segmentation_tracker import compute_intersection_union_per_class -from torch_points3d.metrics.segmentation.segmentation_tracker import compute_average_intersection_union -from torch_points3d.metrics.segmentation.segmentation_tracker import compute_overall_accuracy -from torch_points3d.metrics.segmentation.segmentation_tracker import compute_mean_class_accuracy +from torch_points3d.metrics.segmentation.metrics import compute_intersection_union_per_class +from torch_points3d.metrics.segmentation.metrics import compute_average_intersection_union +from torch_points3d.metrics.segmentation.metrics import compute_overall_accuracy +from torch_points3d.metrics.segmentation.metrics import compute_mean_class_accuracy diff --git a/torch_points3d/metrics/base_tracker.py b/torch_points3d/metrics/base_tracker.py index 0aefead..9cc949b 100644 --- a/torch_points3d/metrics/base_tracker.py +++ b/torch_points3d/metrics/base_tracker.py @@ -21,9 +21,9 @@ def track(self, output_model, *args, **kwargs) -> Dict[str, Any]: def track_loss(self, losses: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: out_loss = dict() for key, loss in losses.items(): - loss_key = "%s_%s" % (self.stage, key) - if loss_key not in self.loss_metrics: - self.loss_metrics[loss_key] = AverageMeter() + loss_key = f"{self.stage}_{key}" + if loss_key not in self.loss_metrics.keys(): + self.loss_metrics[loss_key] = AverageMeter().to(loss) val = self.loss_metrics[loss_key](loss) out_loss[loss_key] = val return out_loss diff --git a/torch_points3d/tasks/base_model.py b/torch_points3d/tasks/base_model.py index b2c7e29..03fc2b3 100644 --- a/torch_points3d/tasks/base_model.py +++ b/torch_points3d/tasks/base_model.py @@ -98,7 +98,7 @@ def configure_metrics(self, stage: str) -> Optional[Any]: def _step(self, batch, batch_idx, stage: str): self.model.set_input(batch) loss = self.model.forward() - losses = self.model.get_losses(stage=stage) + losses = self.model.get_losses() outputs = self.model.get_outputs() metric_dict = self.tracker(outputs, losses) self.log_dict(metric_dict, prog_bar=True, on_step=False, on_epoch=True) From 413a2bd13601b3d0fef59519f38deaef4e6e7757 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Jul 2021 16:47:06 +0000 Subject: [PATCH 16/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torch_points3d/metrics/base_tracker.py | 2 +- torch_points3d/metrics/segmentation/metrics.py | 2 +- torch_points3d/metrics/segmentation/segmentation_tracker.py | 2 +- torch_points3d/models/base_model.py | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/torch_points3d/metrics/base_tracker.py b/torch_points3d/metrics/base_tracker.py index 9cc949b..f2e0568 100644 --- a/torch_points3d/metrics/base_tracker.py +++ b/torch_points3d/metrics/base_tracker.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, Callable, Optional +from typing import Any, Dict, Optional import torch from torch import nn from torchmetrics import AverageMeter diff --git a/torch_points3d/metrics/segmentation/metrics.py b/torch_points3d/metrics/segmentation/metrics.py index dd6cd48..a3aecc9 100644 --- a/torch_points3d/metrics/segmentation/metrics.py +++ b/torch_points3d/metrics/segmentation/metrics.py @@ -1,5 +1,5 @@ import torch -from typing import Dict, Optional, Tuple, Any, Union +from typing import Optional, Tuple, Union def compute_average_intersection_union(confusion_matrix: torch.Tensor, missing_as_one: bool = False) -> torch.Tensor: diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index 4137772..cb374f1 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -1,5 +1,5 @@ import torch -from typing import Dict, Optional, Tuple, Any, Union +from typing import Any, Dict from torchmetrics import ConfusionMatrix from torchmetrics import Metric diff --git a/torch_points3d/models/base_model.py b/torch_points3d/models/base_model.py index 0ed49e9..2845996 100644 --- a/torch_points3d/models/base_model.py +++ b/torch_points3d/models/base_model.py @@ -23,10 +23,9 @@ def forward(self) -> Optional[torch.Tensor]: def compute_loss(self): raise (NotImplementedError("get_losses needs to be defined!")) - def get_losses(self) -> Optional[Dict["str", torch.Tensor]]: + def get_losses(self) -> Optional[Dict["str", torch.Tensor]]: return self._losses - def get_outputs(self) -> Dict[str, Optional[torch.Tensor]]: """ return the outputs to track for the metrics From 11e8a5e6602525c9b2b4473fbbbe9a1b33493687 Mon Sep 17 00:00:00 2001 From: humanpose1 Date: Wed, 28 Jul 2021 18:51:34 +0200 Subject: [PATCH 17/17] remove reset --- torch_points3d/metrics/base_tracker.py | 5 ----- torch_points3d/metrics/segmentation/segmentation_tracker.py | 3 --- 2 files changed, 8 deletions(-) diff --git a/torch_points3d/metrics/base_tracker.py b/torch_points3d/metrics/base_tracker.py index f2e0568..dd65630 100644 --- a/torch_points3d/metrics/base_tracker.py +++ b/torch_points3d/metrics/base_tracker.py @@ -57,8 +57,3 @@ def get_final_loss_metrics(self): metrics[key] = m.compute() self.loss_metrics = nn.ModuleDict() return metrics - - def reset(self, stage: str = "train"): - self._finalised = False - self.stage = stage - self.loss_metrics = nn.ModuleDict() diff --git a/torch_points3d/metrics/segmentation/segmentation_tracker.py b/torch_points3d/metrics/segmentation/segmentation_tracker.py index cb374f1..d9c5a77 100644 --- a/torch_points3d/metrics/segmentation/segmentation_tracker.py +++ b/torch_points3d/metrics/segmentation/segmentation_tracker.py @@ -49,6 +49,3 @@ def _finalise(self): matrix = self.confusion_matrix_metric.compute() segmentation_metrics = self.compute_metrics_from_cm(matrix) return segmentation_metrics - - def reset(self, stage: str = "train"): - super().reset(stage)