From 745bfa56e73427a09db1771005dadcafeefe920c Mon Sep 17 00:00:00 2001 From: lefreud Date: Sat, 11 May 2024 23:15:42 +0000 Subject: [PATCH] Cam optimized off, ns-render fix --- experiments.md | 28 +++++++++++++++++++++++ nerfstudio/hdr_nerfacto/config.py | 4 +--- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 2 +- nerfstudio/lantern/dataset.py | 5 ++++ nerfstudio/pipelines/base_pipeline.py | 19 ++++++++++++--- nerfstudio/scripts/render.py | 7 +++--- 6 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 experiments.md diff --git a/experiments.md b/experiments.md new file mode 100644 index 000000000..0941de33b --- /dev/null +++ b/experiments.md @@ -0,0 +1,28 @@ + + +`` +renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto +`` + +No camera optim + +outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-11_141957/config.yml + +Train: +``` +export SCENE=lab_downstairs_ns + +ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +``` + +Eval (GT+normal): +``` +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-11_141957 + +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr +``` + diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py index 5f223c917..e7b4c4b22 100644 --- a/nerfstudio/hdr_nerfacto/config.py +++ b/nerfstudio/hdr_nerfacto/config.py @@ -23,9 +23,7 @@ def get_hdr_nerfacto_config(): train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + mode="off" ), ), model=HdrNerfactoModelConfig( diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 65d6c6291..2b3eedebf 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -403,7 +403,7 @@ class HdrNerfactoModel(Model): accumulation=outputs["accumulation"], ) - combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + combined_rgb = predicted_rgb combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index a0dc0061a..f5f0b362d 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -102,6 +102,11 @@ class HDRNerfactoInputDataset(InputDataset): """ image = self.get_image(image_idx) data = {"image_idx": image_idx, "image": image} + + # TODO: this doesnt work for training + image_filename = self._dataparser_outputs.image_filenames[image_idx] + data["image_filename"] = str(image_filename.name) + if self._dataparser_outputs.mask_filenames is not None: mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 5e202b06c..3b58f1306 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -394,9 +394,22 @@ class VanillaPipeline(Pipeline): camera_indices = camera_ray_bundle.camera_indices assert camera_indices is not None for key, val in images_dict.items(): - Image.fromarray((val * 255).byte().cpu().numpy()).save( - output_path / "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key) - ) + if 'image_filename' in batch.keys(): + output_name = batch['image_filename'].split('.')[0] + '-' + key + if key == 'img': + output_name = output_name + '.' + batch['image_filename'].split('.')[-1] + else: + output_name = output_name + '.jpg' + else: + output_name = "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key) + if output_name.endswith('.exr'): + import cv2 + render_image = cv2.cvtColor(val.cpu().numpy(), cv2.COLOR_BGR2RGB) + cv2.imwrite(str(output_path / output_name), render_image) + else: + Image.fromarray((val * 255).byte().cpu().numpy()).save( + output_path / output_name + ) assert "num_rays_per_sec" not in metrics_dict metrics_dict["num_rays_per_sec"] = num_rays / (time() - inner_start) fps_str = "fps" diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index bd53890c8..da6856147 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -169,6 +169,10 @@ def _render_trajectory_video( .cpu() .numpy() ) + elif rendered_output_name == "rgb_fast" or rendered_output_name == "rgb" or rendered_output_name == "rgb_hdr": + output_image = ( + output_image.cpu().numpy() + ) elif is_validity: mask_f = torch.clip(outputs["validity_f"], 0, 1) mask_f = torch.ones(mask_f.shape).to(mask_f.get_device()) - mask_f @@ -186,9 +190,6 @@ def _render_trajectory_video( .numpy() ) elif image_format == "exr": - u = 5000. - output_image = torch.exp(output_image * torch.log(torch.tensor(u+1.))) - 1. - output_image /= u output_image = ( output_image.cpu().numpy() )