Cam optimized off, ns-render fix

experiments.md

@@ -0,0 +1,28 @@
+
+
+``
+renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto
+``
+
+No camera optim
+
+outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-11_141957/config.yml
+
+Train:
+```
+export SCENE=lab_downstairs_ns
+
+ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
+```
+
+Eval (GT+normal):
+```
+export SCENE=lab_downstairs_ns
+export MODEL_NAME=hdr-nerfacto/2024-05-11_141957
+
+ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \
+ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \
+ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \
+ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr
+```
+

nerfstudio/hdr_nerfacto/config.py

@@ -23,9 +23,7 @@ def get_hdr_nerfacto_config():
                 train_num_rays_per_batch=4096,
                 eval_num_rays_per_batch=4096,
                 camera_optimizer=CameraOptimizerConfig(
-                    mode="SO3xR3",
-                    optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2),
-                    scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000),
+                    mode="off"
                 ),
             ),
             model=HdrNerfactoModelConfig(

nerfstudio/hdr_nerfacto/hdr_nerf_model.py

@@ -403,7 +403,7 @@ class HdrNerfactoModel(Model):
             accumulation=outputs["accumulation"],
         )
 
-        combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1)
+        combined_rgb = predicted_rgb
         combined_acc = torch.cat([acc], dim=1)
         combined_depth = torch.cat([depth], dim=1)
 

nerfstudio/lantern/dataset.py

@@ -102,6 +102,11 @@ class HDRNerfactoInputDataset(InputDataset):
         """
         image = self.get_image(image_idx)
         data = {"image_idx": image_idx, "image": image}
+        
+         # TODO: this doesnt work for training
+        image_filename = self._dataparser_outputs.image_filenames[image_idx]
+        data["image_filename"] = str(image_filename.name)
+
         if self._dataparser_outputs.mask_filenames is not None:
             mask_filepath = self._dataparser_outputs.mask_filenames[image_idx]
             data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor)

nerfstudio/pipelines/base_pipeline.py

@@ -394,9 +394,22 @@ class VanillaPipeline(Pipeline):
                     camera_indices = camera_ray_bundle.camera_indices
                     assert camera_indices is not None
                     for key, val in images_dict.items():
-                        Image.fromarray((val * 255).byte().cpu().numpy()).save(
-                            output_path / "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key)
-                        )
+                        if 'image_filename' in batch.keys():
+                            output_name = batch['image_filename'].split('.')[0] + '-' + key
+                            if key == 'img':
+                                output_name = output_name + '.' + batch['image_filename'].split('.')[-1]
+                            else:
+                                output_name = output_name + '.jpg'
+                        else:
+                            output_name = "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key)
+                        if output_name.endswith('.exr'):
+                            import cv2
+                            render_image = cv2.cvtColor(val.cpu().numpy(), cv2.COLOR_BGR2RGB)
+                            cv2.imwrite(str(output_path / output_name), render_image)
+                        else:
+                            Image.fromarray((val * 255).byte().cpu().numpy()).save(
+                                output_path / output_name
+                            )
                 assert "num_rays_per_sec" not in metrics_dict
                 metrics_dict["num_rays_per_sec"] = num_rays / (time() - inner_start)
                 fps_str = "fps"

nerfstudio/scripts/render.py

@@ -169,6 +169,10 @@ def _render_trajectory_video(
                             .cpu()
                             .numpy()
                         )
+                    elif rendered_output_name == "rgb_fast" or rendered_output_name == "rgb" or rendered_output_name == "rgb_hdr":
+                        output_image = (
+                            output_image.cpu().numpy()
+                        )
                     elif is_validity:
                         mask_f = torch.clip(outputs["validity_f"], 0, 1)
                         mask_f = torch.ones(mask_f.shape).to(mask_f.get_device()) - mask_f
@@ -186,9 +190,6 @@ def _render_trajectory_video(
                             .numpy()
                         )
                     elif image_format == "exr":
-                        u = 5000.
-                        output_image = torch.exp(output_image * torch.log(torch.tensor(u+1.))) - 1.
-                        output_image /= u
                         output_image = (
                             output_image.cpu().numpy()
                         )