pose and appearance opt working

nerfstudio/cameras/camera_optimizers.py

@@ -78,24 +78,28 @@ class CameraOptimizer(nn.Module):
         self.num_cameras = num_cameras
         self.device = device
 
+        self.initialize_parameters()
+
+    def initialize_parameters(self) -> None:
         # Initialize learnable parameters.
         if self.config.mode == "off":
             pass
         elif self.config.mode in ("SO3xR3", "SE3"):
-            self.pose_adjustment = torch.nn.Parameter(torch.zeros((num_cameras, 6), device=device))
+            self.pose_adjustment = torch.nn.Parameter(torch.zeros((self.num_cameras, 6), device=self.device))
         else:
             assert_never(self.config.mode)
 
         # Initialize pose noise; useful for debugging.
-        if config.position_noise_std != 0.0 or config.orientation_noise_std != 0.0:
-            assert config.position_noise_std >= 0.0 and config.orientation_noise_std >= 0.0
+        if self.config.position_noise_std != 0.0 or self.config.orientation_noise_std != 0.0:
+            assert self.config.position_noise_std >= 0.0 and self.config.orientation_noise_std >= 0.0
             std_vector = torch.tensor(
-                [config.position_noise_std] * 3 + [config.orientation_noise_std] * 3, device=device
+                [self.config.position_noise_std] * 3 + [self.config.orientation_noise_std] * 3, device=self.device
             )
-            self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((num_cameras, 6), device=device), std_vector))
+            self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((self.num_cameras, 6), device=self.device), std_vector))
         else:
             self.pose_noise = None
 
+
     def forward(
         self,
         indices: TensorType["num_cameras"],

nerfstudio/data/datamanagers/base_datamanager.py

@@ -418,12 +418,10 @@ class VanillaDataManager(DataManager):  # pylint: disable=abstract-method
         image_batch = get_dict_to_torch(image_batch, device=self.device, exclude=["image"])
         # image_batch["image"].shape == (1, H, W, 3)
         # this will sample the entire image
-        # we downsample the image so it fits in memory. TODO: make this a parameter
-        print(image_batch["image"].shape)
+        # we downsample the image so it fits in memory
         image_batch["image"] = torch.nn.functional.interpolate(
             image_batch["image"].permute(0, 3, 1, 2), scale_factor=scale_factor, mode="bilinear", align_corners=False
         ).permute(0, 2, 3, 1)
-        print(image_batch["image"].shape)
 
         batch = self.eval_pixel_sampler.sample(image_batch, sample_all_pixels=True)
         ray_indices = batch["indices"]

nerfstudio/pipelines/base_pipeline.py

@@ -355,6 +355,8 @@ class VanillaPipeline(Pipeline):
 
                     # get the eval camera optimizer's parameters
                     camera_opt_param_group = self.config.datamanager.camera_optimizer.param_group + "_eval"
+                    # reinitialize the parameters
+                    self.datamanager.eval_camera_optimizer.initialize_parameters()
                     param_groups = self.datamanager.get_param_groups()
                     optimizer = torch.optim.Adam(param_groups[camera_opt_param_group], lr=1e-3, eps=1e-15)
 

scripts/datasets/process_mipnerf360.py

@@ -86,3 +86,8 @@ with Pool() as p:
 
 # --load-config outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_194940/config.yml
 # start train but run eval on all images at the start...
+
+# DATASET="plane" METHOD="nerfacto-none" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras False --pipeline.eval_optimize_appearance False
+# DATASET="plane" METHOD="nerfacto-pose" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False
+# DATASET="plane" METHOD="nerfacto-pose-app" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True
+# DATASET="plane" METHOD="nerfacto-app" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras False --pipeline.eval_optimize_appearance True