fix normal

main.py

@@ -52,7 +52,7 @@
     parser.add_argument('--radius_range', type=float, nargs='*', default=[1.0, 1.5], help="training camera radius range")
     parser.add_argument('--fovy_range', type=float, nargs='*', default=[40, 70], help="training camera fovy range")
     parser.add_argument('--dir_text', action='store_true', help="direction-encode the text prompt, by appending front/side/back/overhead view")
-    parser.add_argument('--negative_dir_text', action='store_true', help="also use negative dir text prompt.")
+    parser.add_argument('--suppress_face', action='store_true', help="also use negative dir text prompt.")
     parser.add_argument('--angle_overhead', type=float, default=30, help="[0, angle_overhead] is the overhead region")
     parser.add_argument('--angle_front', type=float, default=60, help="[0, angle_front] is the front region, [180, 180+angle_front] the back region, otherwise the side region.")
 
@@ -76,7 +76,7 @@
     if opt.O:
         opt.fp16 = True
         opt.dir_text = True
-        opt.negative_dir_text = True
+        # opt.suppress_face = True
         opt.cuda_ray = True
 
         # opt.lambda_entropy = 1e-4
@@ -85,7 +85,7 @@
     elif opt.O2:
         opt.fp16 = True
         opt.dir_text = True
-        opt.negative_dir_text = True
+        # opt.suppress_face = True
 
         # opt.lambda_entropy = 1e-4 # necessary to keep non-empty
         # opt.lambda_opacity = 3e-3 # no occupancy grid, so use a stronger opacity loss.

nerf/network.py

@@ -95,7 +95,7 @@ def finite_difference_normal(self, x, epsilon=1e-2):
             0.5 * (dz_pos - dz_neg) / epsilon
         ], dim=-1)
 
-        return normal
+        return -normal
 
     def normal(self, x):
 
@@ -138,7 +138,7 @@ def forward(self, x, d, l=None, ratio=1, shading='albedo'):
             normal[torch.isnan(normal)] = 0
 
             # lambertian shading
-            lambertian = ratio + (1 - ratio) * (normal @ -l).clamp(min=0) # [N,]
+            lambertian = ratio + (1 - ratio) * (normal @ l).clamp(min=0) # [N,]
 
             if shading == 'textureless':
                 color = lambertian.unsqueeze(-1).repeat(1, 3)

nerf/network_grid.py

@@ -102,7 +102,7 @@ def finite_difference_normal(self, x, epsilon=1e-2):
             0.5 * (dz_pos - dz_neg) / epsilon
         ], dim=-1)
 
-        return normal
+        return -normal
 
 
     def normal(self, x):
@@ -132,7 +132,7 @@ def forward(self, x, d, l=None, ratio=1, shading='albedo'):
             normal = self.normal(x)
 
             # lambertian shading
-            lambertian = ratio + (1 - ratio) * (normal @ -l).clamp(min=0) # [N,]
+            lambertian = ratio + (1 - ratio) * (normal @ l).clamp(min=0) # [N,]
 
             if shading == 'textureless':
                 color = lambertian.unsqueeze(-1).repeat(1, 3)

nerf/renderer.py

@@ -403,7 +403,7 @@ def run(self, rays_o, rays_d, num_steps=128, upsample_steps=128, light_d=None, a
             # orientation loss
             normals = normals.view(N, -1, 3)
             loss_orient = weights.detach() * (normals * dirs).sum(-1).clamp(min=0) ** 2
-            results['loss_orient'] = loss_orient.mean()
+            results['loss_orient'] = loss_orient.sum(-1).mean()
 
             # surface normal smoothness
             normals_perturb = self.normal(xyzs + torch.randn_like(xyzs) * 1e-2).view(N, -1, 3)
@@ -483,7 +483,7 @@ def run_cuda(self, rays_o, rays_d, dt_gamma=0, light_d=None, ambient_ratio=1.0,
 
             # normals related regularizations
             if normals is not None:
-                # orientation loss
+                # orientation loss (not very exact in cuda ray mode)
                 weights = 1 - torch.exp(-sigmas)
                 loss_orient = weights.detach() * (normals * dirs).sum(-1).clamp(min=0) ** 2
                 results['loss_orient'] = loss_orient.mean()

nerf/utils.py

@@ -306,14 +306,14 @@ def prepare_text_embeddings(self):
                 negative_text = f"{self.opt.negative}"
 
                 # explicit negative dir-encoded text
-                if self.opt.negative_dir_text:
+                if self.opt.suppress_face:
                     if negative_text != '': negative_text += ', '
 
-                    if d == 'back': negative_text += "front view"
-                    elif d == 'front': negative_text += "back view"
-                    elif d == 'side': negative_text += "front view, back view"
-                    elif d == 'overhead': negative_text += "bottom view"
-                    elif d == 'bottom': negative_text += "overhead view"
+                    if d == 'back': negative_text += "face"
+                    # elif d == 'front': negative_text += ""
+                    elif d == 'side': negative_text += "face"
+                    elif d == 'overhead': negative_text += "face"
+                    elif d == 'bottom': negative_text += "face"
 
                 text_z = self.guidance.get_text_embeds([text], [negative_text])
                 self.text_z.append(text_z)
@@ -351,9 +351,9 @@ def train_step(self, data):
             if rand > 0.8: 
                 shading = 'albedo'
                 ambient_ratio = 1.0
-            # elif rand > 0.4: 
-            #     shading = 'textureless'
-            #     ambient_ratio = 0.1
+            elif rand > 0.4: 
+                shading = 'textureless'
+                ambient_ratio = 0.1
             else: 
                 shading = 'lambertian'
                 ambient_ratio = 0.1

readme.md

@@ -80,7 +80,7 @@ python main.py --text "a hamburger" --workspace trial -O
 # we also support negative text prompt now:
 python main.py --text "a rose" --negative "red" --workspace trial -O
 
-# if the above command fails to generate things (learns an empty scene), maybe try:
+# if the above command fails to generate meaningful things (learns an empty scene), maybe try:
 # 1. disable random lambertian shading, simply use albedo as color:
 python main.py --text "a hamburger" --workspace trial -O --albedo_iters 10000 # i.e., set --albedo_iters >= --iters, which is default to 10000
 # 2. use a smaller density regularization weight: