diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index 891cd272a..e3052c813 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -78,24 +78,28 @@ class CameraOptimizer(nn.Module): self.num_cameras = num_cameras self.device = device + self.initialize_parameters() + + def initialize_parameters(self) -> None: # Initialize learnable parameters. if self.config.mode == "off": pass elif self.config.mode in ("SO3xR3", "SE3"): - self.pose_adjustment = torch.nn.Parameter(torch.zeros((num_cameras, 6), device=device)) + self.pose_adjustment = torch.nn.Parameter(torch.zeros((self.num_cameras, 6), device=self.device)) else: assert_never(self.config.mode) # Initialize pose noise; useful for debugging. - if config.position_noise_std != 0.0 or config.orientation_noise_std != 0.0: - assert config.position_noise_std >= 0.0 and config.orientation_noise_std >= 0.0 + if self.config.position_noise_std != 0.0 or self.config.orientation_noise_std != 0.0: + assert self.config.position_noise_std >= 0.0 and self.config.orientation_noise_std >= 0.0 std_vector = torch.tensor( - [config.position_noise_std] * 3 + [config.orientation_noise_std] * 3, device=device + [self.config.position_noise_std] * 3 + [self.config.orientation_noise_std] * 3, device=self.device ) - self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((num_cameras, 6), device=device), std_vector)) + self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((self.num_cameras, 6), device=self.device), std_vector)) else: self.pose_noise = None + def forward( self, indices: TensorType["num_cameras"], diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 7aa873aee..9f4a51824 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -418,12 +418,10 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method image_batch = get_dict_to_torch(image_batch, device=self.device, exclude=["image"]) # image_batch["image"].shape == (1, H, W, 3) # this will sample the entire image - # we downsample the image so it fits in memory. TODO: make this a parameter - print(image_batch["image"].shape) + # we downsample the image so it fits in memory image_batch["image"] = torch.nn.functional.interpolate( image_batch["image"].permute(0, 3, 1, 2), scale_factor=scale_factor, mode="bilinear", align_corners=False ).permute(0, 2, 3, 1) - print(image_batch["image"].shape) batch = self.eval_pixel_sampler.sample(image_batch, sample_all_pixels=True) ray_indices = batch["indices"] diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 9babe3113..88b512789 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -355,6 +355,8 @@ class VanillaPipeline(Pipeline): # get the eval camera optimizer's parameters camera_opt_param_group = self.config.datamanager.camera_optimizer.param_group + "_eval" + # reinitialize the parameters + self.datamanager.eval_camera_optimizer.initialize_parameters() param_groups = self.datamanager.get_param_groups() optimizer = torch.optim.Adam(param_groups[camera_opt_param_group], lr=1e-3, eps=1e-15) diff --git a/scripts/datasets/process_mipnerf360.py b/scripts/datasets/process_mipnerf360.py index e06210738..fbc235eb9 100644 --- a/scripts/datasets/process_mipnerf360.py +++ b/scripts/datasets/process_mipnerf360.py @@ -86,3 +86,8 @@ with Pool() as p: # --load-config outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_194940/config.yml # start train but run eval on all images at the start... + +# DATASET="plane" METHOD="nerfacto-none" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras False --pipeline.eval_optimize_appearance False +# DATASET="plane" METHOD="nerfacto-pose" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False +# DATASET="plane" METHOD="nerfacto-pose-app" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True +# DATASET="plane" METHOD="nerfacto-app" && ns-train nerfacto --vis wandb --data "data/nerfstudio/${DATASET}" --wandb-name "${METHOD}" --experiment-name "${METHOD}-${DATASET}" --trainer.load-dir outputs/nerfacto-no-pose-plane/nerfacto/2022-12-16_220419/nerfstudio_models --trainer.steps-per-eval-all-images 10 --pipeline.eval_optimize_cameras False --pipeline.eval_optimize_appearance True