turn grad accum off by default, decrease background mlp size, add overhead text embedding

nerfstudio/engine/dreamfusion_trainer.py

@@ -35,7 +35,7 @@ class DreamfusionTrainerConfig(TrainerConfig):
     """target class to instantiate"""
     pipeline: DreamfusionPipelineConfig = DreamfusionPipelineConfig()
     """specifies the pipeline config"""
-    gradient_accumulation_steps: int = 4
+    gradient_accumulation_steps: int = 1
     """number of gradient accumulation steps before optimizer step"""
 
 

nerfstudio/fields/dreamfusion_field.py

@@ -140,8 +140,8 @@ class DreamFusionField(Field):
                 "otype": "FullyFusedMLP",
                 "activation": "ReLU",
                 "output_activation": "Sigmoid",
-                "n_neurons": 64,
-                "n_hidden_layers": 2,
+                "n_neurons": 32,
+                "n_hidden_layers": 1,
             },
         )
 

nerfstudio/models/dreamfusion.py

@@ -224,7 +224,6 @@ class DreamFusionModel(Model):
         accumulation = self.renderer_accumulation(weights)
         depth = self.renderer_depth(weights, ray_samples_uniform)
         rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
-        # rgb = torch.clamp(rgb, 0.0, 1.0)
 
         accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0)
         accum_mask_inv = 1.0 - accum_mask

nerfstudio/pipelines/dreamfusion_pipeline.py

@@ -110,26 +110,8 @@ class DreamfusionPipeline(VanillaPipeline):
             .permute(0, 3, 1, 2)
         )
 
-        accumulation = (
-            model_outputs["accumulation"]
-            .view(self.config.datamanager.train_resolution, self.config.datamanager.train_resolution)
-            .detach()
-            .cpu()
-            .numpy()
-        )
-        accumulation = np.clip(accumulation, 0.0, 1.0)
-        plt.imsave("nerf_accumulation.jpg", accumulation)
-
-        # background = model_outputs["background"].view(res, res, 3).detach().cpu().numpy()
-        # background = np.clip(background, 0.0, 1.0)
-        # plt.imsave("nerf_background.jpg", background)
-
-        # shaded = model_outputs["shaded"].view(res, res, 3).detach().cpu().numpy()
-        # shaded = np.clip(shaded, 0.0, 1.0)
-        # plt.imsave("nerf_textureless.jpg", shaded)
-
         if self.config.location_based_prompting:
-            if batch["vertical"] > 60:
+            if batch["vertical"] < 30:
                 text_embedding = self.top_text_embedding
             elif batch["central"] > 315 or batch["central"] <= 45:
                 text_embedding = self.front_text_embedding