You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Full logs or other relevant observations:
AssertionError Traceback (most recent call last)
Cell In[15], line 7
5 trainer.register_hooks([MemoryTrackingHook(period=1)])
6 trainer.resume_or_load(resume=False)
----> 7 trainer.train()
File ~/detectron_uw_repo/detectron2/engine/defaults.py:488, in DefaultTrainer.train(self)
481 def train(self):
482 """
483 Run training.
484
485 Returns:
486 OrderedDict of results, if evaluation is enabled. Otherwise None.
487 """
--> 488 super().train(self.start_iter, self.max_iter)
489 if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process():
490 assert hasattr(
491 self, "_last_eval_results"
492 ), "No evaluation results obtained during training!"
File ~/detectron_uw_repo/detectron2/engine/train_loop.py:155, in TrainerBase.train(self, start_iter, max_iter)
153 for self.iter in range(start_iter, max_iter):
154 self.before_step()
--> 155 self.run_step()
156 self.after_step()
157 # self.iter == max_iter can be used by after_train to
158 # tell whether the training successfully finished or failed
159 # due to exceptions.
File ~/anaconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs)
1536 # If we don't have any hooks, we want to skip the rest of the logic in
1537 # this function, and just call forward.
1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1539 or _global_backward_pre_hooks or _global_backward_hooks
1540 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1541 return forward_call(*args, **kwargs)
1543 try:
1544 result = None
File ~/detectron_uw_repo/detectron2/modeling/meta_arch/panoptic_fpn.py:119, in PanopticFPN.forward(self, batched_inputs)
116 images = self.preprocess_image(batched_inputs)
117 features = self.backbone(images.tensor)
--> 119 assert "sem_seg" in batched_inputs[0]
120 gt_sem_seg = [x["sem_seg"].to(self.device) for x in batched_inputs]
121 gt_sem_seg = ImageList.from_tensors(
122 gt_sem_seg,
123 self.backbone.size_divisibility,
124 self.sem_seg_head.ignore_value,
125 self.backbone.padding_constraints,
126 ).tensor
AssertionError:
Expected behavior:
Expected model to train as normal. Not sure what's causing this error. Followed some other examples I've seen on here on how others have implemented custom mappers and haven't found any examples of others getting this error.
Any and all help or advice would be greatly appreciated.
Environment:
sys.platform linux
Python 3.12.4 | packaged by Anaconda, Inc. | (main, Jun 18 2024, 15:12:24) [GCC 11.2.0]
numpy 1.26.4
detectron2 0.6 @/home/computational/anaconda3/lib/python3.12/site-packages/detectron2
Compiler GCC 11.2
CUDA compiler CUDA 12.5
detectron2 arch flags 8.6
DETECTRON2_ENV_MODULE
PyTorch 2.3.1+cu121 @/home/computational/anaconda3/lib/python3.12/site-packages/torch
PyTorch debug build False
torch._C._GLIBCXX_USE_CXX11_ABI False
GPU available Yes
GPU 0 NVIDIA RTX A2000 12GB (arch=8.6)
Driver version 555.58.02
CUDA_HOME /home/computational/anaconda3
Pillow 10.3.0
torchvision 0.18.1+cu121 @/home/computational/anaconda3/lib/python3.12/site-packages/torchvision
torchvision arch flags 5.0, 6.0, 7.0, 7.5, 8.0, 8.6, 9.0
fvcore 0.1.5.post20221221
iopath 0.1.9
cv2 4.10.0
PyTorch built with:
GCC 9.3
C++ Version: 201703
Intel(R) oneAPI Math Kernel Library Version 2023.1-Product Build 20230303 for Intel(R) 64 architecture applications
If you do not know the root cause of the problem, please post according to this template:
Instructions To Reproduce the Issue:
I tried to create a custom mapping function to increase the number of augmentations performed on my input images. Code for custom trainer below
def custom_mapper(dataset_dict):
dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below
image = utils.read_image(dataset_dict["file_name"], format="BGR")
transform_list = [
T.Resize((800,600)),
T.RandomBrightness(0.8, 1.8),
T.RandomContrast(0.6, 1.3),
T.RandomSaturation(0.8, 1.4),
T.RandomRotation(angle=[90, 90]),
T.RandomLighting(0.7),
T.RandomFlip(prob=0.4, horizontal=False, vertical=True),
]
image, transforms = T.apply_transform_gens(transform_list, image)
dataset_dict["image"] = torch.as_tensor(image.transpose(2, 0, 1).astype("float32"))
class CustomTrainer(DefaultTrainer):
@classmethod
def build_train_loader(cls, cfg):
return build_detection_train_loader(cfg, mapper=custom_mapper)
torch.cuda.empty_cache()
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
trainer = CustomTrainer(cfg)
trainer.register_hooks([MemoryTrackingHook(period=1)])
trainer.resume_or_load(resume=False)
trainer.train()
AssertionError Traceback (most recent call last)
Cell In[15], line 7
5 trainer.register_hooks([MemoryTrackingHook(period=1)])
6 trainer.resume_or_load(resume=False)
----> 7 trainer.train()
File ~/detectron_uw_repo/detectron2/engine/defaults.py:488, in DefaultTrainer.train(self)
481 def train(self):
482 """
483 Run training.
484
485 Returns:
486 OrderedDict of results, if evaluation is enabled. Otherwise None.
487 """
--> 488 super().train(self.start_iter, self.max_iter)
489 if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process():
490 assert hasattr(
491 self, "_last_eval_results"
492 ), "No evaluation results obtained during training!"
File ~/detectron_uw_repo/detectron2/engine/train_loop.py:155, in TrainerBase.train(self, start_iter, max_iter)
153 for self.iter in range(start_iter, max_iter):
154 self.before_step()
--> 155 self.run_step()
156 self.after_step()
157 # self.iter == max_iter can be used by
after_train
to158 # tell whether the training successfully finished or failed
159 # due to exceptions.
File ~/detectron_uw_repo/detectron2/engine/defaults.py:498, in DefaultTrainer.run_step(self)
496 def run_step(self):
497 self._trainer.iter = self.iter
--> 498 self._trainer.run_step()
File ~/detectron_uw_repo/detectron2/engine/train_loop.py:494, in AMPTrainer.run_step(self)
492 self.optimizer.zero_grad()
493 with autocast(dtype=self.precision):
--> 494 loss_dict = self.model(data)
495 if isinstance(loss_dict, torch.Tensor):
496 losses = loss_dict
File ~/anaconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs)
1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1531 else:
-> 1532 return self._call_impl(*args, **kwargs)
File ~/anaconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs)
1536 # If we don't have any hooks, we want to skip the rest of the logic in
1537 # this function, and just call forward.
1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1539 or _global_backward_pre_hooks or _global_backward_hooks
1540 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1541 return forward_call(*args, **kwargs)
1543 try:
1544 result = None
File ~/detectron_uw_repo/detectron2/modeling/meta_arch/panoptic_fpn.py:119, in PanopticFPN.forward(self, batched_inputs)
116 images = self.preprocess_image(batched_inputs)
117 features = self.backbone(images.tensor)
--> 119 assert "sem_seg" in batched_inputs[0]
120 gt_sem_seg = [x["sem_seg"].to(self.device) for x in batched_inputs]
121 gt_sem_seg = ImageList.from_tensors(
122 gt_sem_seg,
123 self.backbone.size_divisibility,
124 self.sem_seg_head.ignore_value,
125 self.backbone.padding_constraints,
126 ).tensor
AssertionError:
Expected behavior:
Expected model to train as normal. Not sure what's causing this error. Followed some other examples I've seen on here on how others have implemented custom mappers and haven't found any examples of others getting this error.
Any and all help or advice would be greatly appreciated.
Environment:
sys.platform linux
Python 3.12.4 | packaged by Anaconda, Inc. | (main, Jun 18 2024, 15:12:24) [GCC 11.2.0]
numpy 1.26.4
detectron2 0.6 @/home/computational/anaconda3/lib/python3.12/site-packages/detectron2
Compiler GCC 11.2
CUDA compiler CUDA 12.5
detectron2 arch flags 8.6
DETECTRON2_ENV_MODULE
PyTorch 2.3.1+cu121 @/home/computational/anaconda3/lib/python3.12/site-packages/torch
PyTorch debug build False
torch._C._GLIBCXX_USE_CXX11_ABI False
GPU available Yes
GPU 0 NVIDIA RTX A2000 12GB (arch=8.6)
Driver version 555.58.02
CUDA_HOME /home/computational/anaconda3
Pillow 10.3.0
torchvision 0.18.1+cu121 @/home/computational/anaconda3/lib/python3.12/site-packages/torchvision
torchvision arch flags 5.0, 6.0, 7.0, 7.5, 8.0, 8.6, 9.0
fvcore 0.1.5.post20221221
iopath 0.1.9
cv2 4.10.0
PyTorch built with:
The text was updated successfully, but these errors were encountered: