fix a minor typo bug in baseline

Author: jwyang

configs/sgg_res101_step.yaml

@@ -28,6 +28,6 @@ MODEL:
     GRCNN_FEATURE_UPDATE_STEP: 2
 SOLVER:
   BASE_LR: 5e-3
-  MAX_ITER: 40000
-  STEPS: (20000,30000)
+  MAX_ITER: 15000
+  STEPS: (8000,12000)
   CHECKPOINT_PERIOD: 1000

lib/config/defaults.py

@@ -37,8 +37,8 @@
 """"======================================="""
 _C.MODEL = CN()
 _C.MODEL.ALGORITHM = "sg_baseline"
+_C.MODEL.USE_RELPN = False
 _C.MODEL.USE_FREQ_PRIOR = False
-
 _C.MODEL.RPN_ONLY = False
 _C.MODEL.MASK_ON = False
 _C.MODEL.RETINANET_ON = False

lib/scene_parser/rcnn/modeling/relation_heads/baseline/baseline.py

@@ -14,7 +14,7 @@ def __init__(self, cfg, in_channels):
         super(Baseline, self).__init__()
         self.cfg = cfg
         self.pred_feature_extractor = make_roi_relation_feature_extractor(cfg, in_channels)
-        self.predictor = make_roi_relation_predictor(cfg, self.feature_extractor.out_channels)
+        self.predictor = make_roi_relation_predictor(cfg, self.pred_feature_extractor.out_channels)
 
     def forward(self, features, proposals, proposal_pairs):
         obj_class_logits = None # no need to predict object class again

lib/scene_parser/rcnn/modeling/relation_heads/grcnn/agcn/agcn.py

@@ -55,11 +55,11 @@ class _GraphConvolutionLayer_Collect(nn.Module):
     def __init__(self, dim_obj, dim_rel):
         super(_GraphConvolutionLayer_Collect, self).__init__()
         self.collect_units = nn.ModuleList()
-        # self.collect_units.append(_Collection_Unit(dim_obj, dim_obj)) # obj from obj
         self.collect_units.append(_Collection_Unit(dim_rel, dim_obj)) # obj (subject) from rel
         self.collect_units.append(_Collection_Unit(dim_rel, dim_obj)) # obj (object) from rel
         self.collect_units.append(_Collection_Unit(dim_obj, dim_rel)) # rel from obj (subject)
         self.collect_units.append(_Collection_Unit(dim_obj, dim_rel)) # rel from obj (object)
+        self.collect_units.append(_Collection_Unit(dim_obj, dim_obj)) # obj from obj
 
     def forward(self, target, source, attention, unit_id):
         collection = self.collect_units[unit_id](target, source, attention)

lib/scene_parser/rcnn/modeling/relation_heads/grcnn/grcnn.py

@@ -1,4 +1,4 @@
-# Scene Graph Generation by Iterative Message Passing
+# Graph R-CNN for scene graph generation
 # Reimnplemetned by Jianwei Yang ([email protected])
 
 import numpy as np
@@ -17,7 +17,7 @@ class GRCNN(nn.Module):
     def __init__(self, cfg, in_channels):
         super(GRCNN, self).__init__()
         self.cfg = cfg
-        self.dim = 512
+        self.dim = 1024
         self.update_step = cfg.MODEL.ROI_RELATION_HEAD.GRCNN_FEATURE_UPDATE_STEP
         num_classes_obj = cfg.MODEL.ROI_BOX_HEAD.NUM_CLASSES
         num_classes_pred = cfg.MODEL.ROI_RELATION_HEAD.NUM_CLASSES
@@ -42,30 +42,35 @@ def __init__(self, cfg, in_channels):
             self.gcn_collect_score = _GraphConvolutionLayer_Collect(num_classes_obj, num_classes_pred)
             self.gcn_update_score = _GraphConvolutionLayer_Update(num_classes_obj, num_classes_pred)
 
-        self.obj_predictor = make_roi_relation_box_predictor(cfg, 512)
-        self.pred_predictor = make_roi_relation_predictor(cfg, 512)
+        self.obj_predictor = make_roi_relation_box_predictor(cfg, self.dim)
+        self.pred_predictor = make_roi_relation_predictor(cfg, self.dim)
 
     def _get_map_idxs(self, proposals, proposal_pairs):
         rel_inds = []
         offset = 0
+        obj_num = sum([len(proposal) for proposal in proposals])
+        obj_obj_map = torch.FloatTensor(obj_num, obj_num).fill_(0)
         for proposal, proposal_pair in zip(proposals, proposal_pairs):
             rel_ind_i = proposal_pair.get_field("idx_pairs").detach()
+            obj_obj_map_i = (1 - torch.eye(len(proposal))).float()
+            obj_obj_map[offset:offset + len(proposal), offset:offset + len(proposal)] = obj_obj_map_i
             rel_ind_i += offset
             offset += len(proposal)
             rel_inds.append(rel_ind_i)
 
         rel_inds = torch.cat(rel_inds, 0)
 
-        subj_pred_map = rel_inds.new(sum([len(proposal) for proposal in proposals]), rel_inds.shape[0]).fill_(0).float().detach()
-        obj_pred_map = rel_inds.new(sum([len(proposal) for proposal in proposals]), rel_inds.shape[0]).fill_(0).float().detach()
+        subj_pred_map = rel_inds.new(obj_num, rel_inds.shape[0]).fill_(0).float().detach()
+        obj_pred_map = rel_inds.new(obj_num, rel_inds.shape[0]).fill_(0).float().detach()
 
         subj_pred_map.scatter_(0, (rel_inds[:, 0].contiguous().view(1, -1)), 1)
         obj_pred_map.scatter_(0, (rel_inds[:, 1].contiguous().view(1, -1)), 1)
+        obj_obj_map = obj_obj_map.type_as(obj_pred_map)
 
-        return rel_inds, subj_pred_map, obj_pred_map
+        return rel_inds, obj_obj_map, subj_pred_map, obj_pred_map
 
     def forward(self, features, proposals, proposal_pairs):
-        rel_inds, subj_pred_map, obj_pred_map = self._get_map_idxs(proposals, proposal_pairs)
+        rel_inds, obj_obj_map, subj_pred_map, obj_pred_map = self._get_map_idxs(proposals, proposal_pairs)
         x_obj = torch.cat([proposal.get_field("features").detach() for proposal in proposals], 0)
         obj_class_logits = torch.cat([proposal.get_field("logits").detach() for proposal in proposals], 0)
         # x_obj = self.avgpool(self.obj_feature_extractor(features, proposals))
@@ -78,9 +83,12 @@ def forward(self, features, proposals, proposal_pairs):
         pred_feats = [x_pred]
 
         for t in range(self.update_step):
+            # message from other objects
+            source_obj = self.gcn_collect_feat(obj_feats[t], obj_feats[t], obj_obj_map, 4)
+
             source_rel_sub = self.gcn_collect_feat(obj_feats[t], pred_feats[t], subj_pred_map, 0)
             source_rel_obj = self.gcn_collect_feat(obj_feats[t], pred_feats[t], obj_pred_map, 1)
-            source2obj_all = (source_rel_sub + source_rel_obj) / 2
+            source2obj_all = (source_obj + source_rel_sub + source_rel_obj) / 3
             obj_feats.append(self.gcn_update_feat(obj_feats[t], source2obj_all, 0))
 
             '''update predicate logits'''
@@ -100,12 +108,12 @@ def forward(self, features, proposals, proposal_pairs):
         for t in range(self.update_step):
             '''update object logits'''
             # message from other objects
-            # source_obj = self.gcn_collect(obj_class_logits, obj_class_logits, map_obj_obj, cfg.COLLECT_OBJ_FROM_OBJ)
+            source_obj = self.gcn_collect_score(obj_scores[t], obj_scores[t], obj_obj_map, 4)
 
             #essage from predicate
             source_rel_sub = self.gcn_collect_score(obj_scores[t], pred_scores[t], subj_pred_map, 0)
             source_rel_obj = self.gcn_collect_score(obj_scores[t], pred_scores[t], obj_pred_map, 1)
-            source2obj_all = (source_rel_sub + source_rel_obj) / 2
+            source2obj_all = (source_obj + source_rel_sub + source_rel_obj) / 3
             obj_scores.append(self.gcn_update_score(obj_scores[t], source2obj_all, 0))
 
             '''update predicate logits'''

lib/scene_parser/rcnn/modeling/relation_heads/relation_heads.py

@@ -4,12 +4,10 @@
 import numpy as np
 import torch
 from torch import nn
-
-# from .roi_relation_feature_extractors import make_roi_relation_feature_extractor
-# from .roi_relation_predictors import make_roi_relation_predictor
+from lib.scene_parser.rcnn.structures.bounding_box_pair import BoxPairList
 from .inference import make_roi_relation_post_processor
 from .loss import make_roi_relation_loss_evaluator
-from lib.scene_parser.rcnn.structures.bounding_box_pair import BoxPairList
+from .relpn.relpn import make_relation_proposal_network
 
 from .baseline.baseline import build_baseline_model
 from .imp.imp import build_imp_model
@@ -40,6 +38,9 @@ def __init__(self, cfg, in_channels):
         self.post_processor = make_roi_relation_post_processor(cfg)
         self.loss_evaluator = make_roi_relation_loss_evaluator(cfg)
 
+        if self.cfg.MODEL.USE_RELPN:
+            self.relpn = make_relation_proposal_network(cfg)
+
         self.freq_dist = None
         if self.cfg.MODEL.USE_FREQ_PRIOR:
             self.freq_dist = torch.from_numpy(np.load("freq_prior.npy"))
@@ -85,11 +86,16 @@ def forward(self, features, proposals, targets=None):
         if self.training:
             # Faster R-CNN subsamples during training the proposals with a fixed
             # positive / negative ratio
-            with torch.no_grad():
-                proposal_pairs = self.loss_evaluator.subsample(proposals, targets)
+            if self.cfg.MODEL.USE_RELPN:
+                proposal_pairs = self.relpn(proposals, targets)
+            else:
+                with torch.no_grad():
+                    proposal_pairs = self.loss_evaluator.subsample(proposals, targets)
         else:
-            # proposals = [proposal[:32] for proposal in proposals]
-            proposal_pairs = self._get_proposal_pairs(proposals)
+            if self.cfg.MODEL.USE_RELPN:
+                proposal_pairs = self.relpn(proposals, targets)
+            else:
+                proposal_pairs = self._get_proposal_pairs(proposals)
 
         if self.cfg.MODEL.USE_FREQ_PRIOR:
             """

lib/scene_parser/rcnn/modeling/relation_heads/reldn/reldn.py

@@ -40,20 +40,20 @@ def __init__(self, cfg, in_channels, eps=1e-10):
             nn.Linear(self.dim, self.dim),
         )
 
-        # self.rel_embedding = nn.Sequential(
-        #     nn.Linear(3 * self.dim, self.dim),
-        #     nn.ReLU(True),
-        #     nn.Linear(self.dim, self.dim),
-        #     nn.ReLU(True)
-        # )
+        self.rel_embedding = nn.Sequential(
+            nn.Linear(3 * self.dim, self.dim),
+            nn.ReLU(True),
+            nn.Linear(self.dim, self.dim),
+            nn.ReLU(True)
+        )
 
-        # self.rel_spatial_feat = build_spatial_feature(cfg, self.dim)
+        self.rel_spatial_feat = build_spatial_feature(cfg, self.dim)
 
-        # self.rel_subj_predictor = make_roi_relation_predictor(cfg, 512)
-        # self.rel_obj_predictor = make_roi_relation_predictor(cfg, 512)
+        self.rel_subj_predictor = make_roi_relation_predictor(cfg, 512)
+        self.rel_obj_predictor = make_roi_relation_predictor(cfg, 512)
         self.rel_pred_predictor = make_roi_relation_predictor(cfg, 512)
 
-        # self.rel_spt_predictor = nn.Linear(64, num_classes)
+        self.rel_spt_predictor = nn.Linear(64, num_classes)
 
 
         self.freq_dist = torch.from_numpy(np.load("freq_prior.npy"))
@@ -91,25 +91,24 @@ def forward(self, features, proposals, proposal_pairs):
         # x_obj = self.avgpool(self.obj_feature_extractor(features, proposals))
         x_pred = self.avgpool(self.pred_feature_extractor(features, proposal_pairs))
         x_obj = x_obj.view(x_obj.size(0), -1); x_pred = x_pred.view(x_pred.size(0), -1)
-        x_obj = self.obj_embedding(x_obj);
-        x_pred = self.pred_embedding(x_pred)
+        x_obj = self.obj_embedding(x_obj); x_pred = self.pred_embedding(x_pred)
 
         sub_vert = x_obj[rel_inds[:, 0]]  #
         obj_vert = x_obj[rel_inds[:, 1]]
 
         '''compute visual scores'''
-        # rel_subj_class_logits = self.rel_subj_predictor(sub_vert.unsqueeze(2).unsqueeze(3))
-        # rel_obj_class_logits = self.rel_obj_predictor(obj_vert.unsqueeze(2).unsqueeze(3))
+        rel_subj_class_logits = self.rel_subj_predictor(sub_vert.unsqueeze(2).unsqueeze(3))
+        rel_obj_class_logits = self.rel_obj_predictor(obj_vert.unsqueeze(2).unsqueeze(3))
 
-        x_rel = x_pred + sub_vert + obj_vert # torch.cat([sub_vert, obj_vert, x_pred], 1)
-        # x_rel = self.rel_embedding(x_rel)
+        x_rel = torch.cat([sub_vert, obj_vert, x_pred], 1)
+        x_rel = self.rel_embedding(x_rel)
         rel_pred_class_logits = self.rel_pred_predictor(x_rel.unsqueeze(2).unsqueeze(3))
-        # rel_vis_class_logits = rel_pred_class_logits + rel_subj_class_logits + rel_obj_class_logits
-        rel_vis_class_logits = rel_pred_class_logits # + rel_subj_class_logits + rel_obj_class_logits
+        rel_vis_class_logits = rel_pred_class_logits + rel_subj_class_logits + rel_obj_class_logits
+        # rel_vis_class_logits = rel_pred_class_logits # + rel_subj_class_logits + rel_obj_class_logits
 
         '''compute spatial scores'''
-        # edge_spt_feats = self.rel_spatial_feat(proposal_pairs)
-        # rel_spt_class_logits = self.rel_spt_predictor(edge_spt_feats)
+        edge_spt_feats = self.rel_spatial_feat(proposal_pairs)
+        rel_spt_class_logits = self.rel_spt_predictor(edge_spt_feats)
 
         '''compute semantic scores'''
         rel_sem_class_logits = []
@@ -129,7 +128,7 @@ def forward(self, features, proposals, proposal_pairs):
             rel_sem_class_logits.append(class_logits_per_image)
         rel_sem_class_logits = torch.cat(rel_sem_class_logits, 0)
 
-        rel_class_logits = rel_vis_class_logits + rel_sem_class_logits # + rel_spt_class_logits #
+        rel_class_logits = rel_vis_class_logits + rel_sem_class_logits + rel_spt_class_logits #
         return (x_obj, x_pred), obj_class_logits, rel_class_logits
 
 def build_reldn_model(cfg, in_channels):