diff --git a/README-pandora.md b/README-pandora.md index ae849bac3..5a4012072 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -115,7 +115,7 @@ After convergence, go to the next step. ns-train lantern-nerfacto --help ``` -If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30` +If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30` > Thing to keep in mind: the way Nerfstudio splits train and test images may not be optimal: left might be in one dataset and right in another... that's a bit of a data leak. @@ -134,6 +134,22 @@ TODO: validate this part ## 4. Evaluation +``` +ns-train hdr-nerfacto --data data/lab_downstairs_ns --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +``` + ``` ns-eval --load-config outputs/LVSN_LAB_OpenSFM/hdr-nerfacto/2024-05-10_032723/config.yml --output-path ablations_out/hdr_nerfacto/output.json --render-output-path ablations_out/hdr-nerfacto -``` \ No newline at end of file +``` + +``` +ns-render camera-path --load-config outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-10_211317/config.yml --camera-path-filename data/lab_downstairs_ns/GT_transforms.json --output-path renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto --output-format images --image-format exr +``` + +``` +docker run -it --rm -v /gel/usr/$USER:/mnt/workspace/ -v /home-local2/$USER.extra.nobkp/:/mnt/data/ --gpus '"device=1"' -p 7007:7007 lantern_docker:latest /bin/bash +export PYTHONPATH=$PYTHONPATH:$(pwd) +``` + +docker exec -it 57d2bf549c9a /bin/bash +ns-train hdr-nerfacto --data data/lab_downstairs_ns/ --pipeline.datamanager.train-num-images-to-sample-from 1800 --viewer.websocket-port 9009 diff --git a/experiments.md b/experiments.md index 0941de33b..ae9abdc05 100644 --- a/experiments.md +++ b/experiments.md @@ -12,7 +12,7 @@ Train: ``` export SCENE=lab_downstairs_ns -ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 ``` Eval (GT+normal): @@ -21,8 +21,25 @@ export SCENE=lab_downstairs_ns export MODEL_NAME=hdr-nerfacto/2024-05-11_141957 ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ -ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \ -ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr ``` +Move (outside docker) +``` +mkdir -p /gel/usr/mokad6/ml_class/fred-out/ ; \ +cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/ablations_out /gel/usr/mokad6/ml_class/fred-out/ns-eval ; \ +cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/renders /gel/usr/mokad6/ml_class/fred-out/ns-render +``` + +1800 samples: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 + + + +Test wo CRF + +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 563d7e2df..38776f3da 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -78,6 +78,7 @@ descriptions = { "nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.", "lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", "hdr-nerfacto": "HDR-Nerfacto, adapted to the Nerfacto model.", + "hdr-nerfacto-wo-crf": "HDR-Nerfacto without CRF, adapted to the Nerfacto model.", "depth-nerfacto": "Nerfacto with depth supervision.", "volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", @@ -128,7 +129,8 @@ method_configs["nerfacto"] = TrainerConfig( method_configs["lantern-nerfacto"] = Get_lantern_config() -method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config() +method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True) +method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False) method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py index e7b4c4b22..ecb9168dc 100644 --- a/nerfstudio/hdr_nerfacto/config.py +++ b/nerfstudio/hdr_nerfacto/config.py @@ -6,19 +6,27 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.hdr_nerfacto.hdr_nerf_model import HdrNerfactoModelConfig -from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig +from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig, HDRNerfactoWoCrfDataManagerConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig -def get_hdr_nerfacto_config(): +def get_hdr_nerfacto_config(*, use_crf=True): + if use_crf: + method_name = "hdr-nerfacto" + else: + method_name = "hdr-nerfacto-wo-crf" + + datamanager_config_class = HDRNerfactoDataManagerConfig if use_crf else HDRNerfactoWoCrfDataManagerConfig + return TrainerConfig( - method_name="hdr-nerfacto", + method_name=method_name, steps_per_eval_batch=500, steps_per_save=2000, + # max_num_iterations=1000, max_num_iterations=30000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=HDRNerfactoDataManagerConfig( + datamanager=datamanager_config_class( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, @@ -27,7 +35,8 @@ def get_hdr_nerfacto_config(): ), ), model=HdrNerfactoModelConfig( - eval_num_rays_per_chunk=1 << 15 + eval_num_rays_per_chunk=1 << 15, + use_crf=use_crf, ), ), optimizers={ diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index aff4aa922..9143339bc 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -96,6 +96,7 @@ class HdrNerfactoField(Field): use_average_appearance_embedding: bool = False, spatial_distortion: Optional[SpatialDistortion] = None, implementation: Literal["tcnn", "torch"] = "tcnn", + use_crf: bool = True, ) -> None: super().__init__() @@ -200,17 +201,19 @@ class HdrNerfactoField(Field): implementation=implementation, ) - # HDR NeRF, 1 MLP per channel - self.crf_mlps = nn.ModuleList([ - MLP(in_dim=1, - num_layers=2, - layer_width=128, - out_dim=1, - activation=nn.ReLU(), - out_activation=nn.Sigmoid(), - implementation=implementation, - ) for _ in range(3) - ]) + self.use_crf = use_crf + if use_crf: + # HDR NeRF, 1 MLP per channel + self.crf_mlps = nn.ModuleList([ + MLP(in_dim=1, + num_layers=2, + layer_width=128, + out_dim=1, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) for _ in range(3) + ]) def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: @@ -316,29 +319,37 @@ class HdrNerfactoField(Field): dim=-1, ) log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions) - rgb_hdr = torch.exp(log_rgb_hdr).view(*outputs_shape, -1).to(directions) + log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures) - log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure + #print('exposure_min', exposures.min(), 'exposure_max', exposures.max()) + # TODO: validate this + rgb_hdr = torch.exp(log_rgb_hdr_exposed).view(*outputs_shape, -1).to(directions) + #print('rgb_hdr_min', rgb_hdr.min(), 'rgb_hdr_max', rgb_hdr.max()) - pred_rgb_fast = torch.cat([ - self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3) - ], dim=-1).view(*outputs_shape, -1).to(directions) + # outputs[FieldHeadNames.RGB_HDR] = torch.clamp(rgb_hdr, 0, 1) + outputs[FieldHeadNames.RGB_HDR] = rgb_hdr - pred_rgb_ldr = torch.cat([ - self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) - ], dim=-1) - pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions) + if self.use_crf: + log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure + pred_rgb_fast = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3) + ], dim=-1).view(*outputs_shape, -1).to(directions) - zero_radiance_crf = torch.cat([ - self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3) - ], dim=-1) - zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions) + pred_rgb_ldr = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) + ], dim=-1) + pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions) - outputs.update({ - FieldHeadNames.RGB: pred_rgb_ldr, - FieldHeadNames.RGB_FAST: pred_rgb_fast, - FieldHeadNames.RGB_HDR: rgb_hdr, - FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, - }) + zero_radiance_crf = torch.cat([ + self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3) + ], dim=-1) + zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions) + + outputs.update({ + FieldHeadNames.RGB: pred_rgb_ldr, + FieldHeadNames.RGB_FAST: pred_rgb_fast, + FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, + }) return outputs + diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 2b3eedebf..991ca55df 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -142,6 +142,8 @@ class HdrNerfactoModelConfig(ModelConfig): unit_exposure_loss_mult: float = 0.5 unit_exposure_expected_value: float = 0.5 + use_crf: bool = True + class HdrNerfactoModel(Model): """Nerfacto model @@ -178,6 +180,7 @@ class HdrNerfactoModel(Model): use_average_appearance_embedding=self.config.use_average_appearance_embedding, appearance_embedding_dim=self.config.appearance_embed_dim, implementation=self.config.implementation, + use_crf=self.config.use_crf, ) self.density_fns = [] @@ -301,24 +304,25 @@ class HdrNerfactoModel(Model): weights_list.append(weights) ray_samples_list.append(ray_samples) - rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) - rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) - depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) - accumulation = self.renderer_accumulation(weights=weights) - # HDR-Nerfacto rgb_hdr = self.renderer_rgb( rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights ) + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + accumulation = self.renderer_accumulation(weights=weights) + outputs = { - "rgb": rgb, - "rgb_fast": rgb_fast, "accumulation": accumulation, "depth": depth, "rgb_hdr": rgb_hdr, - "zero_radiance_crf": field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF], } + if self.config.use_crf: + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) + outputs["rgb"] = rgb + outputs["rgb_fast"] = rgb_fast + outputs['zero_radiance_crf'] = field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF] if self.config.predict_normals: normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) @@ -350,7 +354,10 @@ class HdrNerfactoModel(Model): metrics_dict = {} gt_rgb = batch["image"].to(self.device) # RGB or RGBA image gt_rgb = self.renderer_rgb.blend_background(gt_rgb) # Blend if RGBA - predicted_rgb = outputs["rgb"] + if self.config.use_crf: + predicted_rgb = outputs["rgb"] + else: + predicted_rgb = outputs["rgb_hdr"] metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb) if self.training: @@ -361,17 +368,24 @@ class HdrNerfactoModel(Model): loss_dict = {} image = batch["image"].to(self.device) - pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( - pred_image=outputs["rgb"], - pred_accumulation=outputs["accumulation"], - gt_image=image, - ) - - loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss( - outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value - ) + if self.config.use_crf: + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) + loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss( + outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value + ) + else: + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb_hdr"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + # TODO: check that # if self.training: # loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( @@ -395,7 +409,10 @@ class HdrNerfactoModel(Model): self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: gt_rgb = batch["image"].to(self.device) - predicted_rgb = outputs["rgb"] # Blended with background (black if random background) + if self.config.use_crf: + predicted_rgb = outputs["rgb"] + else: + predicted_rgb = outputs["rgb_hdr"] gt_rgb = self.renderer_rgb.blend_background(gt_rgb) acc = colormaps.apply_colormap(outputs["accumulation"]) depth = colormaps.apply_depth_colormap( diff --git a/nerfstudio/lantern/datamanager.py b/nerfstudio/lantern/datamanager.py index be88eb6be..b9e2a5d3f 100644 --- a/nerfstudio/lantern/datamanager.py +++ b/nerfstudio/lantern/datamanager.py @@ -9,7 +9,11 @@ from nerfstudio.data.datamanagers.base_datamanager import ( VanillaDataManager, VanillaDataManagerConfig, ) -from nerfstudio.lantern.dataset import HDRInputDataset, HDRNerfactoInputDataset +from nerfstudio.lantern.dataset import ( + HDRInputDataset, + HDRNerfactoInputDataset, + HDRNerfactoWoCrfInputDataset, +) @dataclass @@ -19,6 +23,13 @@ class HDRNerfactoDataManagerConfig(VanillaDataManagerConfig): _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoInputDataset]) """Target class to instantiate.""" +@dataclass +class HDRNerfactoWoCrfDataManagerConfig(VanillaDataManagerConfig): + """A HDR data manager, based on VanillaDataManager""" + + _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoWoCrfInputDataset]) + """Target class to instantiate.""" + @dataclass class HDRVanillaDataManagerConfig(VanillaDataManagerConfig):