From 687906a6d9db3d381119e48f3a593a72fea25db4 Mon Sep 17 00:00:00 2001 From: lefreud Date: Mon, 29 Jan 2024 19:16:47 +0000 Subject: [PATCH 01/20] Quick hacky fix for pixel sampler --- nerfstudio/data/pixel_samplers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 27480dcc6..c2fe2407d 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -97,14 +97,14 @@ class PixelSampler: mask: mask of possible pixels in an image to sample from. """ # TODO Defining a flag for sampling even when there is a mask - if isinstance(mask, torch.Tensor): + if isinstance(mask, torch.Tensor) and mask.shape[0] >= 4: # mask size: batch * h * w * 1 # Init state, cache masks indices. if self.states_of_mask == 0: with profiler.time_function("mask squeeze: whole"): mask = mask.squeeze(-1) num_masks = mask.shape[0] - size_of_disk = 100 + size_of_disk = 2 num_disks = ceil(num_masks / size_of_disk) # slice mask to 100 disk random_idx_disk = int(torch.randint(0, num_disks, (1,))) From 95cb1c7e2ec34aa63d47478209f8c2c3bae7ff7a Mon Sep 17 00:00:00 2001 From: lefreud Date: Mon, 29 Jan 2024 19:17:47 +0000 Subject: [PATCH 02/20] Removed unused file --- nerfstudio/configs/method_configs.py | 1 - .../data/datamanagers/base_datamanager.py | 1 - nerfstudio/data/datasets/hdr_dataset.py | 137 ------------------ 3 files changed, 139 deletions(-) delete mode 100644 nerfstudio/data/datasets/hdr_dataset.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 50357be3e..0675b172c 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -41,7 +41,6 @@ from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParser from nerfstudio.data.datasets.depth_dataset import DepthDataset from nerfstudio.data.datasets.sdf_dataset import SDFDataset from nerfstudio.data.datasets.semantic_dataset import SemanticDataset -from nerfstudio.data.datasets.hdr_dataset import HDRInputDataset from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import ( diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 0d73c0459..7571c0477 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -54,7 +54,6 @@ from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset -from nerfstudio.data.datasets.hdr_dataset import HDRInputDataset from nerfstudio.data.pixel_samplers import ( PixelSampler, diff --git a/nerfstudio/data/datasets/hdr_dataset.py b/nerfstudio/data/datasets/hdr_dataset.py deleted file mode 100644 index c6f1c8954..000000000 --- a/nerfstudio/data/datasets/hdr_dataset.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Dataset. -""" -from __future__ import annotations - -from copy import deepcopy -from pathlib import Path -from typing import Dict, List -import os -os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" - -import numpy as np -import numpy.typing as npt -import torch -from jaxtyping import Float -from PIL import Image -from torch import Tensor -from torch.utils.data import Dataset -import cv2 - - -from nerfstudio.cameras.cameras import Cameras -from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs -from nerfstudio.data.utils.data_utils import get_image_mask_tensor_from_path -from nerfstudio.data.datasets.base_dataset import InputDataset - -class HDRInputDataset(InputDataset): - """Dataset that returns HDR images, OpenEXR. - - Args: - dataparser_outputs: description of where and how to read input images. - scale_factor: The scaling factor for the dataparser outputs - """ - - # exclude_batch_keys_from_device: List[str] = ["image", "mask"] - # cameras: Cameras - - def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): - super().__init__(dataparser_outputs, scale_factor) - - def __len__(self): - return len(self._dataparser_outputs.image_filenames) - - def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: - """Returns the image of shape (H, W, 3 or 4), with np.float32 values. - - Args: - image_idx: The image index in the dataset. - """ - image_filename = self._dataparser_outputs.image_filenames[image_idx] - hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) - if self.scale_factor != 1.0: - # Here cv2 img shape is (height, width, BGR) - height, width = hdr_image.shape[:2] - newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) - hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) - hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) - hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) - if len(hdr_image.shape) == 2: - hdr_image = hdr_image[:, :, None].repeat(3, axis=2) - assert len(hdr_image.shape) == 3 - assert hdr_image.dtype == np.float32 - assert hdr_image.shape[2] in [3, 4], f"Image shape of {hdr_image.shape} is in correct." - # dfdffdd - return hdr_image - - def get_image(self, image_idx: int) -> Float[Tensor, "image_height image_width num_channels"]: - """Returns a 3 channel image. - - Args: - image_idx: The image index in the dataset. - """ - # Compress HDR using: pixel = log(pixel + 1.) - - image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) - image = (image) ** (1 / 2.2) - image = torch.clip(image, 0, 1) - # image = torch.log(1. + image) - - - if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: - image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) - return image - - def get_data(self, image_idx: int) -> Dict: - """Returns the ImageDataset data as a dictionary. - - Args: - image_idx: The image index in the dataset. - """ - image = self.get_image(image_idx) - data = {"image_idx": image_idx, "image": image} - if self._dataparser_outputs.mask_filenames is not None: - mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] - data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) - assert ( - data["mask"].shape[:2] == data["image"].shape[:2] - ), f"Mask and image have different shapes. Got {data['mask'].shape[:2]} and {data['image'].shape[:2]}" - metadata = self.get_metadata(data) - data.update(metadata) - return data - - def get_metadata(self, data: Dict) -> Dict: - """Method that can be used to process any additional metadata that may be part of the model inputs. - - Args: - image_idx: The image index in the dataset. - """ - del data - return {} - - def __getitem__(self, image_idx: int) -> Dict: - data = self.get_data(image_idx) - return data - - @property - def image_filenames(self) -> List[Path]: - """ - Returns image filenames for this dataset. - The order of filenames is the same as in the Cameras object for easy mapping. - """ - - return self._dataparser_outputs.image_filenames From d89d988c81e537fd4d0aa927d814103d17aeb566 Mon Sep 17 00:00:00 2001 From: lefreud Date: Mon, 29 Jan 2024 19:18:08 +0000 Subject: [PATCH 03/20] Readme --- README-pandora.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README-pandora.md b/README-pandora.md index 462910199..c7a16dd89 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -99,8 +99,7 @@ After that, you'll need to copy manually (for now), the folder to `images`, for Examples ``` -ns-process-data lantern --data ./data/lab_ground_floor/trimmed_videos_smart_subset --mask-dir ./data/lab_ground_floor/trimmed_videos_masks --output-dir ./data/lab_ground_floor/colmap_out_smart --skip-image-processing --camera-type equirectangular --images-per --equirect 8 --skip-colmap --skip-perspective-transform +ns-process-data lantern --data ./data/lab_ground_floor/trimmed_videos_smart_subset --mask-dir ./data/lab_ground_floor/trimmed_videos_masks --output-dir ./data/lab_ground_floor/colmap_out_smart --skip-image-processing --camera-type equirectangular --images-per-equirect 8 --skip-colmap --skip-perspective-transform ``` ## 2. Pre-training NeRF @@ -120,7 +119,7 @@ If you need to use less frames, add this argument: `--pipeline.datamanager.train Examples ``` -ns-train lantern-nerfacto --data data/lab_ground_floor/colmap_out_smart --pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30 +ns-train lantern-nerfacto --data data/lab_ground_floor/colmap_out_smart --pipeline.datamanager.eval-num-images-to-sample-from 0 ``` ## 3. Running MomoNet From 1af19580627c1f80e7feacf57454135eb2c533da Mon Sep 17 00:00:00 2001 From: lefreud Date: Tue, 30 Jan 2024 14:39:39 +0000 Subject: [PATCH 04/20] Added hdr-nerfacto from nerfacto --- nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 301 ++++++++++++++++ nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 401 ++++++++++++++++++++++ 2 files changed, 702 insertions(+) create mode 100644 nerfstudio/hdr_nerfacto/hdr_nerf_field.py create mode 100644 nerfstudio/hdr_nerfacto/hdr_nerf_model.py diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py new file mode 100644 index 000000000..34c6346fc --- /dev/null +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -0,0 +1,301 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Field for compound nerf model, adds scene contraction and image embeddings to instant ngp +""" + + +from typing import Dict, Literal, Optional, Tuple + +import torch +from torch import Tensor, nn + +from nerfstudio.cameras.rays import RaySamples +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.field_components.activations import trunc_exp +from nerfstudio.field_components.embedding import Embedding +from nerfstudio.field_components.encodings import HashEncoding, NeRFEncoding, SHEncoding +from nerfstudio.field_components.field_heads import ( + FieldHeadNames, + PredNormalsFieldHead, + SemanticFieldHead, + TransientDensityFieldHead, + TransientRGBFieldHead, + UncertaintyFieldHead, +) +from nerfstudio.field_components.mlp import MLP +from nerfstudio.field_components.spatial_distortions import SpatialDistortion +from nerfstudio.fields.base_field import Field, get_normalized_directions + + +class NerfactoField(Field): + """Compound Field that uses TCNN + + Args: + aabb: parameters of scene aabb bounds + num_images: number of images in the dataset + num_layers: number of hidden layers + hidden_dim: dimension of hidden layers + geo_feat_dim: output geo feat dimensions + num_levels: number of levels of the hashmap for the base mlp + base_res: base resolution of the hashmap for the base mlp + max_res: maximum resolution of the hashmap for the base mlp + log2_hashmap_size: size of the hashmap for the base mlp + num_layers_color: number of hidden layers for color network + num_layers_transient: number of hidden layers for transient network + features_per_level: number of features per level for the hashgrid + hidden_dim_color: dimension of hidden layers for color network + hidden_dim_transient: dimension of hidden layers for transient network + appearance_embedding_dim: dimension of appearance embedding + transient_embedding_dim: dimension of transient embedding + use_transient_embedding: whether to use transient embedding + use_semantics: whether to use semantic segmentation + num_semantic_classes: number of semantic classes + use_pred_normals: whether to use predicted normals + use_average_appearance_embedding: whether to use average appearance embedding or zeros for inference + spatial_distortion: spatial distortion to apply to the scene + """ + + aabb: Tensor + + def __init__( + self, + aabb: Tensor, + num_images: int, + num_layers: int = 2, + hidden_dim: int = 64, + geo_feat_dim: int = 15, + num_levels: int = 16, + base_res: int = 16, + max_res: int = 2048, + log2_hashmap_size: int = 19, + num_layers_color: int = 3, + num_layers_transient: int = 2, + features_per_level: int = 2, + hidden_dim_color: int = 64, + hidden_dim_transient: int = 64, + appearance_embedding_dim: int = 32, + transient_embedding_dim: int = 16, + use_transient_embedding: bool = False, + use_semantics: bool = False, + num_semantic_classes: int = 100, + pass_semantic_gradients: bool = False, + use_pred_normals: bool = False, + use_average_appearance_embedding: bool = False, + spatial_distortion: Optional[SpatialDistortion] = None, + implementation: Literal["tcnn", "torch"] = "tcnn", + ) -> None: + super().__init__() + + self.register_buffer("aabb", aabb) + self.geo_feat_dim = geo_feat_dim + + self.register_buffer("max_res", torch.tensor(max_res)) + self.register_buffer("num_levels", torch.tensor(num_levels)) + self.register_buffer("log2_hashmap_size", torch.tensor(log2_hashmap_size)) + + self.spatial_distortion = spatial_distortion + self.num_images = num_images + self.appearance_embedding_dim = appearance_embedding_dim + self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) + self.use_average_appearance_embedding = use_average_appearance_embedding + self.use_transient_embedding = use_transient_embedding + self.use_semantics = use_semantics + self.use_pred_normals = use_pred_normals + self.pass_semantic_gradients = pass_semantic_gradients + self.base_res = base_res + + self.direction_encoding = SHEncoding( + levels=4, + implementation=implementation, + ) + + self.position_encoding = NeRFEncoding( + in_dim=3, num_frequencies=2, min_freq_exp=0, max_freq_exp=2 - 1, implementation=implementation + ) + + self.mlp_base_grid = HashEncoding( + num_levels=num_levels, + min_res=base_res, + max_res=max_res, + log2_hashmap_size=log2_hashmap_size, + features_per_level=features_per_level, + implementation=implementation, + ) + self.mlp_base_mlp = MLP( + in_dim=self.mlp_base_grid.get_out_dim(), + num_layers=num_layers, + layer_width=hidden_dim, + out_dim=1 + self.geo_feat_dim, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.mlp_base = torch.nn.Sequential(self.mlp_base_grid, self.mlp_base_mlp) + + # transients + if self.use_transient_embedding: + self.transient_embedding_dim = transient_embedding_dim + self.embedding_transient = Embedding(self.num_images, self.transient_embedding_dim) + self.mlp_transient = MLP( + in_dim=self.geo_feat_dim + self.transient_embedding_dim, + num_layers=num_layers_transient, + layer_width=hidden_dim_transient, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.get_out_dim()) + self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim()) + self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim()) + + # semantics + if self.use_semantics: + self.mlp_semantics = MLP( + in_dim=self.geo_feat_dim, + num_layers=2, + layer_width=64, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.field_head_semantics = SemanticFieldHead( + in_dim=self.mlp_semantics.get_out_dim(), num_classes=num_semantic_classes + ) + + # predicted normals + if self.use_pred_normals: + self.mlp_pred_normals = MLP( + in_dim=self.geo_feat_dim + self.position_encoding.get_out_dim(), + num_layers=3, + layer_width=64, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.get_out_dim()) + + self.mlp_head = MLP( + in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim, + num_layers=num_layers_color, + layer_width=hidden_dim_color, + out_dim=3, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) + + def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: + """Computes and returns the densities.""" + if self.spatial_distortion is not None: + positions = ray_samples.frustums.get_positions() + positions = self.spatial_distortion(positions) + positions = (positions + 2.0) / 4.0 + else: + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + # Make sure the tcnn gets inputs between 0 and 1. + selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) + positions = positions * selector[..., None] + self._sample_locations = positions + if not self._sample_locations.requires_grad: + self._sample_locations.requires_grad = True + positions_flat = positions.view(-1, 3) + h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) + density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) + self._density_before_activation = density_before_activation + + # Rectifying the density with an exponential is much more stable than a ReLU or + # softplus, because it enables high post-activation (float32) density outputs + # from smaller internal (float16) parameters. + density = trunc_exp(density_before_activation.to(positions)) + density = density * selector[..., None] + return density, base_mlp_out + + def get_outputs( + self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None + ) -> Dict[FieldHeadNames, Tensor]: + assert density_embedding is not None + outputs = {} + if ray_samples.camera_indices is None: + raise AttributeError("Camera indices are not provided.") + camera_indices = ray_samples.camera_indices.squeeze() + directions = get_normalized_directions(ray_samples.frustums.directions) + directions_flat = directions.view(-1, 3) + d = self.direction_encoding(directions_flat) + + outputs_shape = ray_samples.frustums.directions.shape[:-1] + + # appearance + if self.training: + embedded_appearance = self.embedding_appearance(camera_indices) + else: + if self.use_average_appearance_embedding: + embedded_appearance = torch.ones( + (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + ) * self.embedding_appearance.mean(dim=0) + else: + embedded_appearance = torch.zeros( + (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + ) + + # transients + if self.use_transient_embedding and self.training: + embedded_transient = self.embedding_transient(camera_indices) + transient_input = torch.cat( + [ + density_embedding.view(-1, self.geo_feat_dim), + embedded_transient.view(-1, self.transient_embedding_dim), + ], + dim=-1, + ) + x = self.mlp_transient(transient_input).view(*outputs_shape, -1).to(directions) + outputs[FieldHeadNames.UNCERTAINTY] = self.field_head_transient_uncertainty(x) + outputs[FieldHeadNames.TRANSIENT_RGB] = self.field_head_transient_rgb(x) + outputs[FieldHeadNames.TRANSIENT_DENSITY] = self.field_head_transient_density(x) + + # semantics + if self.use_semantics: + semantics_input = density_embedding.view(-1, self.geo_feat_dim) + if not self.pass_semantic_gradients: + semantics_input = semantics_input.detach() + + x = self.mlp_semantics(semantics_input).view(*outputs_shape, -1).to(directions) + outputs[FieldHeadNames.SEMANTICS] = self.field_head_semantics(x) + + # predicted normals + if self.use_pred_normals: + positions = ray_samples.frustums.get_positions() + + positions_flat = self.position_encoding(positions.view(-1, 3)) + pred_normals_inp = torch.cat([positions_flat, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) + + x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions) + outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x) + + h = torch.cat( + [ + d, + density_embedding.view(-1, self.geo_feat_dim), + embedded_appearance.view(-1, self.appearance_embedding_dim), + ], + dim=-1, + ) + rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions) + outputs.update({FieldHeadNames.RGB: rgb}) + + return outputs diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py new file mode 100644 index 000000000..1d243632b --- /dev/null +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -0,0 +1,401 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NeRF implementation that combines many recent advancements. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Literal, Tuple, Type + +import numpy as np +import torch +from torch.nn import Parameter +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image import PeakSignalNoiseRatio +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity + +from nerfstudio.cameras.rays import RayBundle, RaySamples +from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.fields.nerfacto_field import NerfactoField +from nerfstudio.model_components.losses import ( + MSELoss, + distortion_loss, + interlevel_loss, + orientation_loss, + pred_normal_loss, + scale_gradients_by_distance_squared, +) +from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler, UniformSampler +from nerfstudio.model_components.renderers import AccumulationRenderer, DepthRenderer, NormalsRenderer, RGBRenderer +from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps + + +@dataclass +class NerfactoModelConfig(ModelConfig): + """Nerfacto Model Config""" + + _target: Type = field(default_factory=lambda: NerfactoModel) + near_plane: float = 0.05 + """How far along the ray to start sampling.""" + far_plane: float = 1000.0 + """How far along the ray to stop sampling.""" + background_color: Literal["random", "last_sample", "black", "white"] = "last_sample" + """Whether to randomize the background color.""" + hidden_dim: int = 64 + """Dimension of hidden layers""" + hidden_dim_color: int = 64 + """Dimension of hidden layers for color network""" + hidden_dim_transient: int = 64 + """Dimension of hidden layers for transient network""" + num_levels: int = 16 + """Number of levels of the hashmap for the base mlp.""" + base_res: int = 16 + """Resolution of the base grid for the hashgrid.""" + max_res: int = 2048 + """Maximum resolution of the hashmap for the base mlp.""" + log2_hashmap_size: int = 19 + """Size of the hashmap for the base mlp""" + features_per_level: int = 2 + """How many hashgrid features per level""" + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" + num_nerf_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128, "use_linear": False}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256, "use_linear": False}, + ] + ) + """Arguments for the proposal density fields.""" + proposal_initial_sampler: Literal["piecewise", "uniform"] = "piecewise" + """Initial sampler for the proposal network. Piecewise is preferred for unbounded scenes.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + distortion_loss_mult: float = 0.002 + """Distortion loss multiplier.""" + orientation_loss_mult: float = 0.0001 + """Orientation loss multiplier on computed normals.""" + pred_normal_loss_mult: float = 0.001 + """Predicted normal loss multiplier.""" + use_proposal_weight_anneal: bool = True + """Whether to use proposal weight annealing.""" + use_average_appearance_embedding: bool = True + """Whether to use average appearance embedding or zeros for inference.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 1000 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + predict_normals: bool = False + """Whether to predict normals or not.""" + disable_scene_contraction: bool = False + """Whether to disable scene contraction or not.""" + use_gradient_scaling: bool = False + """Use gradient scaler where the gradients are lower for points closer to the camera.""" + implementation: Literal["tcnn", "torch"] = "tcnn" + """Which implementation to use for the model.""" + appearance_embed_dim: int = 32 + """Dimension of the appearance embedding.""" + + +class NerfactoModel(Model): + """Nerfacto model + + Args: + config: Nerfacto configuration to instantiate model + """ + + config: NerfactoModelConfig + + def populate_modules(self): + """Set the fields and modules.""" + super().populate_modules() + + if self.config.disable_scene_contraction: + scene_contraction = None + else: + scene_contraction = SceneContraction(order=float("inf")) + + # Fields + self.field = NerfactoField( + self.scene_box.aabb, + hidden_dim=self.config.hidden_dim, + num_levels=self.config.num_levels, + max_res=self.config.max_res, + base_res=self.config.base_res, + features_per_level=self.config.features_per_level, + log2_hashmap_size=self.config.log2_hashmap_size, + hidden_dim_color=self.config.hidden_dim_color, + hidden_dim_transient=self.config.hidden_dim_transient, + spatial_distortion=scene_contraction, + num_images=self.num_train_data, + use_pred_normals=self.config.predict_normals, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, + appearance_embedding_dim=self.config.appearance_embed_dim, + implementation=self.config.implementation, + ) + + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + if self.config.use_same_proposal_network: + assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed." + prop_net_args = self.config.proposal_net_args_list[0] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)]) + else: + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + # Samplers + def update_schedule(step): + return np.clip( + np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]), + 1, + self.config.proposal_update_every, + ) + + # Change proposal network initial sampler if uniform + initial_sampler = None # None is for piecewise as default (see ProposalNetworkSampler) + if self.config.proposal_initial_sampler == "uniform": + initial_sampler = UniformSampler(single_jitter=self.config.use_single_jitter) + + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + initial_sampler=initial_sampler, + ) + + # Collider + self.collider = NearFarCollider(near_plane=self.config.near_plane, far_plane=self.config.far_plane) + + # renderers + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_normals = NormalsRenderer() + + # shaders + self.normals_shader = NormalsShader() + + # losses + self.rgb_loss = MSELoss() + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + param_groups = {} + param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) + param_groups["fields"] = list(self.field.parameters()) + return param_groups + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + callbacks = [] + if self.config.use_proposal_weight_anneal: + # anneal the weights of the proposal network before doing PDF sampling + N = self.config.proposal_weights_anneal_max_num_iters + + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / N, 0, 1) + + def bias(x, b): + return b * x / ((b - 1) * x + 1) + + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ) + ) + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.proposal_sampler.step_cb, + ) + ) + return callbacks + + def get_outputs(self, ray_bundle: RayBundle): + ray_samples: RaySamples + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals) + if self.config.use_gradient_scaling: + field_outputs = scale_gradients_by_distance_squared(field_outputs, ray_samples) + + weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) + weights_list.append(weights) + ray_samples_list.append(ray_samples) + + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + accumulation = self.renderer_accumulation(weights=weights) + + outputs = { + "rgb": rgb, + "accumulation": accumulation, + "depth": depth, + } + + if self.config.predict_normals: + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + outputs["normals"] = self.normals_shader(normals) + outputs["pred_normals"] = self.normals_shader(pred_normals) + # These use a lot of GPU memory, so we avoid storing them for eval. + if self.training: + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + if self.training and self.config.predict_normals: + outputs["rendered_orientation_loss"] = orientation_loss( + weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions + ) + + outputs["rendered_pred_normal_loss"] = pred_normal_loss( + weights.detach(), + field_outputs[FieldHeadNames.NORMALS].detach(), + field_outputs[FieldHeadNames.PRED_NORMALS], + ) + + for i in range(self.config.num_proposal_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + + return outputs + + def get_metrics_dict(self, outputs, batch): + metrics_dict = {} + gt_rgb = batch["image"].to(self.device) # RGB or RGBA image + gt_rgb = self.renderer_rgb.blend_background(gt_rgb) # Blend if RGBA + predicted_rgb = outputs["rgb"] + metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb) + + if self.training: + metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"]) + return metrics_dict + + def get_loss_dict(self, outputs, batch, metrics_dict=None): + loss_dict = {} + image = batch["image"].to(self.device) + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) + + loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + if self.training: + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + assert metrics_dict is not None and "distortion" in metrics_dict + loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] + if self.config.predict_normals: + # orientation loss for computed normals + loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( + outputs["rendered_orientation_loss"] + ) + + # ground truth supervision for normals + loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( + outputs["rendered_pred_normal_loss"] + ) + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + gt_rgb = batch["image"].to(self.device) + predicted_rgb = outputs["rgb"] # Blended with background (black if random background) + gt_rgb = self.renderer_rgb.blend_background(gt_rgb) + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + + combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + combined_acc = torch.cat([acc], dim=1) + combined_depth = torch.cat([depth], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + gt_rgb = torch.moveaxis(gt_rgb, -1, 0)[None, ...] + predicted_rgb = torch.moveaxis(predicted_rgb, -1, 0)[None, ...] + + psnr = self.psnr(gt_rgb, predicted_rgb) + ssim = self.ssim(gt_rgb, predicted_rgb) + lpips = self.lpips(torch.clamp(gt_rgb, 0, 1), torch.clamp(predicted_rgb, 0, 1)) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + + for i in range(self.config.num_proposal_iterations): + key = f"prop_depth_{i}" + prop_depth_i = colormaps.apply_depth_colormap( + outputs[key], + accumulation=outputs["accumulation"], + ) + images_dict[key] = prop_depth_i + + return metrics_dict, images_dict From 7aa202e476a88a61a0827d612b19ebc993250c9a Mon Sep 17 00:00:00 2001 From: lefreud Date: Tue, 30 Jan 2024 18:54:51 +0000 Subject: [PATCH 05/20] "Working" HDR-nerfacto --- nerfstudio/configs/method_configs.py | 28 +++-- .../data/dataparsers/nerfstudio_dataparser.py | 11 +- nerfstudio/field_components/field_heads.py | 4 + nerfstudio/hdr_nerfacto/__init__.py | 1 + nerfstudio/hdr_nerfacto/config.py | 47 +++++++ nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 79 ++++++++---- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 79 ++++++++---- nerfstudio/lantern/datamanager.py | 19 ++- nerfstudio/lantern/dataset.py | 115 +++++++++++++++++- nerfstudio/utils/colormaps.py | 20 +-- nerfstudio/viewer/server/control_panel.py | 6 +- 11 files changed, 328 insertions(+), 81 deletions(-) create mode 100644 nerfstudio/hdr_nerfacto/__init__.py create mode 100644 nerfstudio/hdr_nerfacto/config.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 0675b172c..563d7e2df 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -20,28 +20,33 @@ from __future__ import annotations from collections import OrderedDict from typing import Dict -from dataclasses import dataclass, field import tyro from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.configs.external_methods import get_external_methods - -from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig - +from nerfstudio.data.datamanagers.base_datamanager import ( + VanillaDataManager, + VanillaDataManagerConfig, +) +from nerfstudio.data.datamanagers.random_cameras_datamanager import ( + RandomCamerasDataManagerConfig, +) from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig -from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig +from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( + InstantNGPDataParserConfig, +) from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig -from nerfstudio.data.dataparsers.phototourism_dataparser import PhototourismDataParserConfig +from nerfstudio.data.dataparsers.phototourism_dataparser import ( + PhototourismDataParserConfig, +) from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig from nerfstudio.data.datasets.depth_dataset import DepthDataset from nerfstudio.data.datasets.sdf_dataset import SDFDataset from nerfstudio.data.datasets.semantic_dataset import SemanticDataset - from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import ( CosineDecaySchedulerConfig, @@ -51,6 +56,8 @@ from nerfstudio.engine.schedulers import ( from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.fields.sdf_field import SDFFieldConfig +from nerfstudio.hdr_nerfacto.config import get_hdr_nerfacto_config +from nerfstudio.lantern.config import Get_lantern_config from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig from nerfstudio.models.generfacto import GenerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig @@ -65,13 +72,12 @@ from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig from nerfstudio.plugins.registry import discover_methods -from nerfstudio.lantern.config import Get_lantern_config - method_configs: Dict[str, TrainerConfig] = {} descriptions = { "nerfacto": "Recommended real-time model tuned for real captures. This model will be continually updated.", "nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.", "lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", + "hdr-nerfacto": "HDR-Nerfacto, adapted to the Nerfacto model.", "depth-nerfacto": "Nerfacto with depth supervision.", "volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", @@ -122,6 +128,8 @@ method_configs["nerfacto"] = TrainerConfig( method_configs["lantern-nerfacto"] = Get_lantern_config() +method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config() + method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", steps_per_eval_batch=500, diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index e6b08b2d2..b2d080cbe 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -16,18 +16,17 @@ from __future__ import annotations import math +import os from dataclasses import dataclass, field from pathlib import Path from typing import Literal, Optional, Type -import os + os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" +import cv2 import numpy as np import torch -from PIL import Image -import cv2 - from nerfstudio.cameras import camera_utils from nerfstudio.cameras.cameras import CAMERA_MODEL_TO_TYPE, Cameras, CameraType @@ -294,6 +293,7 @@ class Nerfstudio(DataParser): else: distortion_params = torch.stack(distort, dim=0)[idx_tensor] + exposure_tensor = torch.tensor([exposures], dtype=torch.float32).T cameras = Cameras( fx=fx, fy=fy, @@ -304,6 +304,9 @@ class Nerfstudio(DataParser): width=width, camera_to_worlds=poses[:, :3, :4], camera_type=camera_type, + metadata={ + 'exposures': exposure_tensor + } ) assert self.downscale_factor is not None diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index 44df9f602..ff9831079 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -41,6 +41,10 @@ class FieldHeadNames(Enum): SDF = "sdf" ALPHA = "alpha" GRADIENT = "gradient" + + RGB_HDR = "rgb_hdr" # HDR-Nerfacto + PRED_RGB_LDR = "pred_rgb_ldr" # HDR-Nerfacto + ZERO_RADIANCE_CRF = "zero_radiance_crf" # HDR-Nerfacto class FieldHead(FieldComponent): diff --git a/nerfstudio/hdr_nerfacto/__init__.py b/nerfstudio/hdr_nerfacto/__init__.py new file mode 100644 index 000000000..84952a816 --- /dev/null +++ b/nerfstudio/hdr_nerfacto/__init__.py @@ -0,0 +1 @@ +# init \ No newline at end of file diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py new file mode 100644 index 000000000..5f223c917 --- /dev/null +++ b/nerfstudio/hdr_nerfacto/config.py @@ -0,0 +1,47 @@ + +from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig +from nerfstudio.configs.base_config import ViewerConfig +from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig +from nerfstudio.engine.optimizers import AdamOptimizerConfig +from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.hdr_nerfacto.hdr_nerf_model import HdrNerfactoModelConfig +from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig + + +def get_hdr_nerfacto_config(): + return TrainerConfig( + method_name="hdr-nerfacto", + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + datamanager=HDRNerfactoDataManagerConfig( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=4096, + eval_num_rays_per_batch=4096, + camera_optimizer=CameraOptimizerConfig( + mode="SO3xR3", + optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), + scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + ), + ), + model=HdrNerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15 + ), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), + } + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", + ) diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index 34c6346fc..3f726d115 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -40,7 +40,7 @@ from nerfstudio.field_components.spatial_distortions import SpatialDistortion from nerfstudio.fields.base_field import Field, get_normalized_directions -class NerfactoField(Field): +class HdrNerfactoField(Field): """Compound Field that uses TCNN Args: @@ -108,9 +108,9 @@ class NerfactoField(Field): self.spatial_distortion = spatial_distortion self.num_images = num_images - self.appearance_embedding_dim = appearance_embedding_dim - self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) - self.use_average_appearance_embedding = use_average_appearance_embedding + # self.appearance_embedding_dim = appearance_embedding_dim + # self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) + # self.use_average_appearance_embedding = use_average_appearance_embedding self.use_transient_embedding = use_transient_embedding self.use_semantics = use_semantics self.use_pred_normals = use_pred_normals @@ -191,15 +191,28 @@ class NerfactoField(Field): self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.get_out_dim()) self.mlp_head = MLP( - in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim, + in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim, num_layers=num_layers_color, layer_width=hidden_dim_color, out_dim=3, activation=nn.ReLU(), - out_activation=nn.Sigmoid(), + out_activation=None, implementation=implementation, ) + # HDR NeRF, 1 MLP per channel + self.crf_mlps = nn.ModuleList([ + MLP(in_dim=1, + num_layers=2, + layer_width=128, + out_dim=1, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) for _ in range(3) + ]) + + def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: """Computes and returns the densities.""" if self.spatial_distortion is not None: @@ -229,6 +242,13 @@ class NerfactoField(Field): def get_outputs( self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None ) -> Dict[FieldHeadNames, Tensor]: + + if ray_samples.metadata is None or 'exposures' not in ray_samples.metadata.keys(): + exposures = torch.ones(ray_samples.frustums.directions.shape[:-1], device=ray_samples.frustums.directions.device)[..., None] + else: + exposures = ray_samples.metadata['exposures'].to(ray_samples.frustums.directions.device) + exposures = exposures.reshape(-1, 1).clone().detach() / 200 + assert density_embedding is not None outputs = {} if ray_samples.camera_indices is None: @@ -240,18 +260,18 @@ class NerfactoField(Field): outputs_shape = ray_samples.frustums.directions.shape[:-1] - # appearance - if self.training: - embedded_appearance = self.embedding_appearance(camera_indices) - else: - if self.use_average_appearance_embedding: - embedded_appearance = torch.ones( - (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device - ) * self.embedding_appearance.mean(dim=0) - else: - embedded_appearance = torch.zeros( - (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device - ) + # # appearance + # if self.training: + # embedded_appearance = self.embedding_appearance(camera_indices) + # else: + # if self.use_average_appearance_embedding: + # embedded_appearance = torch.ones( + # (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + # ) * self.embedding_appearance.mean(dim=0) + # else: + # embedded_appearance = torch.zeros( + # (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + # ) # transients if self.use_transient_embedding and self.training: @@ -291,11 +311,28 @@ class NerfactoField(Field): [ d, density_embedding.view(-1, self.geo_feat_dim), - embedded_appearance.view(-1, self.appearance_embedding_dim), + # embedded_appearance.view(-1, self.appearance_embedding_dim), ], dim=-1, ) - rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions) - outputs.update({FieldHeadNames.RGB: rgb}) + log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions) + rgb_hdr = torch.exp(log_rgb_hdr).view(*outputs_shape, -1).to(directions) + log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures) + + pred_rgb_ldr = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) + ], dim=-1) + pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions) + + zero_radiance_crf = torch.cat([ + self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3) + ], dim=-1) + zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions) + + outputs.update({ + FieldHeadNames.RGB: pred_rgb_ldr, + FieldHeadNames.RGB_HDR: rgb_hdr, + FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, + }) return outputs diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 1d243632b..f10b1e005 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -29,21 +29,32 @@ from torchmetrics.image import PeakSignalNoiseRatio from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle, RaySamples -from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField -from nerfstudio.fields.nerfacto_field import NerfactoField +from nerfstudio.hdr_nerfacto.hdr_nerf_field import HdrNerfactoField from nerfstudio.model_components.losses import ( MSELoss, distortion_loss, - interlevel_loss, orientation_loss, pred_normal_loss, scale_gradients_by_distance_squared, ) -from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler, UniformSampler -from nerfstudio.model_components.renderers import AccumulationRenderer, DepthRenderer, NormalsRenderer, RGBRenderer +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) from nerfstudio.model_components.scene_colliders import NearFarCollider from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model, ModelConfig @@ -51,10 +62,10 @@ from nerfstudio.utils import colormaps @dataclass -class NerfactoModelConfig(ModelConfig): +class HdrNerfactoModelConfig(ModelConfig): """Nerfacto Model Config""" - _target: Type = field(default_factory=lambda: NerfactoModel) + _target: Type = field(default_factory=lambda: HdrNerfactoModel) near_plane: float = 0.05 """How far along the ray to start sampling.""" far_plane: float = 1000.0 @@ -127,15 +138,19 @@ class NerfactoModelConfig(ModelConfig): appearance_embed_dim: int = 32 """Dimension of the appearance embedding.""" + # HDR-Nerfacto + unit_exposure_loss_mult: float = 0.5 + unit_exposure_expected_value: float = 0.5 -class NerfactoModel(Model): + +class HdrNerfactoModel(Model): """Nerfacto model Args: config: Nerfacto configuration to instantiate model """ - config: NerfactoModelConfig + config: HdrNerfactoModelConfig def populate_modules(self): """Set the fields and modules.""" @@ -147,7 +162,7 @@ class NerfactoModel(Model): scene_contraction = SceneContraction(order=float("inf")) # Fields - self.field = NerfactoField( + self.field = HdrNerfactoField( self.scene_box.aabb, hidden_dim=self.config.hidden_dim, num_levels=self.config.num_levels, @@ -228,6 +243,7 @@ class NerfactoModel(Model): # losses self.rgb_loss = MSELoss() + self.unit_exposure_loss = MSELoss() # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) @@ -289,10 +305,17 @@ class NerfactoModel(Model): depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) accumulation = self.renderer_accumulation(weights=weights) + # HDR-Nerfacto + rgb_hdr = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights + ) + outputs = { "rgb": rgb, "accumulation": accumulation, "depth": depth, + "rgb_hdr": rgb_hdr, + "zero_radiance_crf": field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF], } if self.config.predict_normals: @@ -341,23 +364,27 @@ class NerfactoModel(Model): gt_image=image, ) - loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) - if self.training: - loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( - outputs["weights_list"], outputs["ray_samples_list"] - ) - assert metrics_dict is not None and "distortion" in metrics_dict - loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] - if self.config.predict_normals: - # orientation loss for computed normals - loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( - outputs["rendered_orientation_loss"] - ) + loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss( + outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value + ) - # ground truth supervision for normals - loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( - outputs["rendered_pred_normal_loss"] - ) + loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + # if self.training: + # loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + # outputs["weights_list"], outputs["ray_samples_list"] + # ) + # assert metrics_dict is not None and "distortion" in metrics_dict + # loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] + # if self.config.predict_normals: + # # orientation loss for computed normals + # loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( + # outputs["rendered_orientation_loss"] + # ) +# + # # ground truth supervision for normals + # loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( + # outputs["rendered_pred_normal_loss"] + # ) return loss_dict def get_image_metrics_and_images( diff --git a/nerfstudio/lantern/datamanager.py b/nerfstudio/lantern/datamanager.py index 90d8885aa..be88eb6be 100644 --- a/nerfstudio/lantern/datamanager.py +++ b/nerfstudio/lantern/datamanager.py @@ -3,16 +3,23 @@ Lantern HDR DataManager """ from dataclasses import dataclass, field -from typing import Dict, Literal, Tuple, Type, Union +from typing import Type -import torch - -from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datamanagers.base_datamanager import ( - VanillaDataManager, VanillaDataManagerConfig) -from nerfstudio.lantern.dataset import HDRInputDataset + VanillaDataManager, + VanillaDataManagerConfig, +) +from nerfstudio.lantern.dataset import HDRInputDataset, HDRNerfactoInputDataset +@dataclass +class HDRNerfactoDataManagerConfig(VanillaDataManagerConfig): + """A HDR data manager, based on VanillaDataManager""" + + _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoInputDataset]) + """Target class to instantiate.""" + + @dataclass class HDRVanillaDataManagerConfig(VanillaDataManagerConfig): """A HDR data manager, based on VanillaDataManager""" diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index 07a46063d..b3d493941 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -18,7 +18,6 @@ HDR Dataset of .exr format from __future__ import annotations import os -from copy import deepcopy from pathlib import Path from typing import Dict, List @@ -29,16 +28,124 @@ import numpy as np import numpy.typing as npt import torch from jaxtyping import Float -from PIL import Image from torch import Tensor -from torch.utils.data import Dataset -from nerfstudio.cameras.cameras import Cameras from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.utils.data_utils import get_image_mask_tensor_from_path +class HDRNerfactoInputDataset(InputDataset): + """Dataset that returns HDR images, OpenEXR. + + Args: + dataparser_outputs: description of where and how to read input images. + scale_factor: The scaling factor for the dataparser outputs + """ + + # exclude_batch_keys_from_device: List[str] = ["image", "mask"] + # cameras: Cameras + + def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): + super().__init__(dataparser_outputs, scale_factor) + + def __len__(self): + return len(self._dataparser_outputs.image_filenames) + + def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: + """Returns the image of shape (H, W, 3 or 4), with np.float32 values. + + Args: + image_idx: The image index in the dataset. + """ + image_filename = self._dataparser_outputs.image_filenames[image_idx] + #print('image_filename', image_filename) + hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) + # print('hdr_image', hdr_image.shape, hdr_image.dtype) + # print('hdr_image', hdr_image.max(), hdr_image.min()) + # print('hdr_image', hdr_image[0,0,:]) + + if self.scale_factor != 1.0: + # Here cv2 img shape is (height, width, BGR) + height, width = hdr_image.shape[:2] + newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) + hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) + hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) + hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) + if len(hdr_image.shape) == 2: + hdr_image = hdr_image[:, :, None].repeat(3, axis=2) + assert len(hdr_image.shape) == 3 + assert hdr_image.dtype == np.float32 + assert hdr_image.shape[2] in [3, 4], f"Image shape of {hdr_image.shape} is in correct." + return hdr_image + + def get_image(self, image_idx: int) -> Float[Tensor, "image_height image_width num_channels"]: + """Returns a 3 channel image. + + Args: + image_idx: The image index in the dataset. + """ + image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) + #print('a', image.max()) + image = image * self._dataparser_outputs.exposures[image_idx] + # print('b', image.max()) + + if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: + image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) + return image + + def get_data(self, image_idx: int) -> Dict: + """Returns the ImageDataset data as a dictionary. + + Args: + image_idx: The image index in the dataset. + """ + image = self.get_image(image_idx) + data = {"image_idx": image_idx, "image": image} + if self._dataparser_outputs.mask_filenames is not None: + mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] + data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) + assert ( + data["mask"].shape[:2] == data["image"].shape[:2] + ), f"Mask and image have different shapes. Got {data['mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.saturation_mask_filenames is not None: + saturation_mask_filepath = self._dataparser_outputs.saturation_mask_filenames[image_idx] + data["saturation_mask"] = get_image_mask_tensor_from_path(filepath=saturation_mask_filepath, scale_factor=self.scale_factor) + assert ( + data["saturation_mask"].shape[:2] == data["image"].shape[:2] + ), f"Saturation mask and image have different shapes. Got {data['saturation_mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.exposures is not None: + exposure_value = self._dataparser_outputs.exposures[image_idx] + data["exposure"] = exposure_value + metadata = self.get_metadata(data) + data.update(metadata) + return data + + def get_metadata(self, data: Dict) -> Dict: + """Method that can be used to process any additional metadata that may be part of the model inputs. + + Args: + image_idx: The image index in the dataset. + """ + del data + return {} + + def __getitem__(self, image_idx: int) -> Dict: + data = self.get_data(image_idx) + return data + + @property + def image_filenames(self) -> List[Path]: + """ + Returns image filenames for this dataset. + The order of filenames is the same as in the Cameras object for easy mapping. + """ + + return self._dataparser_outputs.image_filenames + + class HDRInputDataset(InputDataset): """Dataset that returns HDR images, OpenEXR. diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index 89ddba073..cf15150cb 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -43,6 +43,11 @@ class ColormapOptions: """ Whether to invert the output colormap """ exposure_scale: float = 0 """Scale the brightness prior to tonemapping by 2 ^ Exposure""" + use_mu_law: bool = True + """Use mu-law de-compression for HDR images""" + use_gamma: bool = True + """Use gamma for HDR images""" + def apply_colormap( image: Float[Tensor, "*bs channels"], @@ -66,15 +71,16 @@ def apply_colormap( # default for rgb images if image.shape[-1] == 3: if(colormap_options.exposure_scale != 0 or image.max() > 1): - # Log - # img_gamma_22 = torch.exp(image) - 1. - # u-law - u = 5000. - img_gamma_22 = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1. - img_gamma_22 /= u + img_gamma_22 = image + if colormap_options.use_mu_law: + u = 5000. + img_gamma_22 = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1. + img_gamma_22 /= u + img_gamma_22 *= 2.**(colormap_options.exposure_scale) - img_gamma_22 = torch.pow(img_gamma_22, 1./2.2) + if colormap_options.use_gamma: + img_gamma_22 = torch.pow(img_gamma_22, 1./2.2) return torch.clamp(img_gamma_22, 0., 1.) else: return image diff --git a/nerfstudio/viewer/server/control_panel.py b/nerfstudio/viewer/server/control_panel.py index 8f2e1bc34..9e36fb2ef 100644 --- a/nerfstudio/viewer/server/control_panel.py +++ b/nerfstudio/viewer/server/control_panel.py @@ -79,8 +79,8 @@ class ControlPanel: self._exposure = ViewerSlider( "Exposure scale", default_value=0.0, - min_value=-10.0, - max_value=10.0, + min_value=-20.0, + max_value=20.0, step=0.1, cb_hook=rerender_cb, hint="Scale the brightness prior to tonemapping by 2 ^ Exposure" @@ -224,7 +224,7 @@ class ControlPanel: Sets elements to be hidden or not based on the current state of the control panel """ self._colormap.set_disabled(self.output_render == "rgb") - self._exposure.set_disabled(self.output_render != "rgb") + self._exposure.set_disabled(self.output_render not in ["rgb", "rgb_hdr"]) for e in self._elements_by_tag["colormap"]: e.set_hidden(self.output_render == "rgb") for e in self._elements_by_tag["split_colormap"]: From e155a4658fb470e760e31bc9ba68095a76e27e98 Mon Sep 17 00:00:00 2001 From: lefreud Date: Tue, 7 May 2024 14:52:10 -0400 Subject: [PATCH 06/20] Added momo's train_test_split --- lantern_scripts/train_test_split.py | 80 +++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 lantern_scripts/train_test_split.py diff --git a/lantern_scripts/train_test_split.py b/lantern_scripts/train_test_split.py new file mode 100644 index 000000000..34d36e285 --- /dev/null +++ b/lantern_scripts/train_test_split.py @@ -0,0 +1,80 @@ +import argparse +import os +from tqdm import tqdm +from pathlib import Path +import json +import numpy as np +import math + + +def load_from_json(filename: Path): + """Load a dictionary from a JSON filename. + + Args: + filename: The filename to load from. + """ + assert filename.suffix == ".json" + with open(filename, encoding="UTF-8") as file: + return json.load(file) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--input_json', type=str) + parser.add_argument('--output_json', type=str) + parser.add_argument('--train_split_fraction', type=float, default= 0.8) + parser.add_argument('--eval_split_fraction', type=float, default= 0.1) + args = parser.parse_args() + + assert os.path.isfile(args.input_json) and args.input_json.endswith('.json'), 'The input JSON file is missing!' + + json_data = load_from_json( Path(args.input_json) ) + + + new_json_data = {} + all_keys = json_data.keys() + for key in all_keys: + new_json_data[key] = json_data[key] + + + num_images = len(json_data['frames']) + num_train_images = math.ceil(num_images * args.train_split_fraction) + num_eval_test_images = num_images - num_train_images + i_all = np.arange(num_images) + i_train = np.linspace( + 0, num_images - 1, num_train_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval_test = np.setdiff1d(i_all, i_train) # eval images are the remaining images + assert len(i_eval_test) == num_eval_test_images + + num_eval_images = math.ceil(num_images * args.eval_split_fraction) + num_test_images = num_eval_test_images - num_train_images + i_eval = np.linspace( + 0, num_eval_test_images - 1, num_eval_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval = i_eval_test[i_eval] + i_test = np.setdiff1d(i_eval_test, i_eval) # eval images are the remaining images + assert len(i_eval) == num_eval_images + + + # has_split_files_spec = any(f"{split}_filenames" in meta for split in ("train", "val", "test")) + + train_list = [] + for i in i_train: + train_list.append(json_data['frames'][i]["file_path"]) + new_json_data["train_filenames"] = train_list + + test_list = [] + for i in i_test: + test_list.append(json_data['frames'][i]["file_path"]) + new_json_data["test_filenames"] = test_list + + eval_list = [] + for i in i_eval: + eval_list.append(json_data['frames'][i]["file_path"]) + new_json_data["val_filenames"] = eval_list + + with open(args.output_json, "w") as outfile: + json.dump(new_json_data, outfile, indent=4) + + print("All done!") \ No newline at end of file From 2719a4457bc098454702dca56de8d5bd7cd1c25d Mon Sep 17 00:00:00 2001 From: lefreud Date: Tue, 7 May 2024 23:02:36 +0000 Subject: [PATCH 07/20] Generate small subset for HDR Nerf --- README-pandora.md | 3 + .../generate_small_subset_hdr_nerf.py | 135 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 lantern_scripts/generate_small_subset_hdr_nerf.py diff --git a/README-pandora.md b/README-pandora.md index c7a16dd89..06a749fed 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -4,6 +4,9 @@ ## 0. Setup + +Open this folder in VS Code Dev Container extension! + ``` export PYTHONPATH=$pwd:$PYTHONPATH pip install skylibs equilib diff --git a/lantern_scripts/generate_small_subset_hdr_nerf.py b/lantern_scripts/generate_small_subset_hdr_nerf.py new file mode 100644 index 000000000..36f34b8bc --- /dev/null +++ b/lantern_scripts/generate_small_subset_hdr_nerf.py @@ -0,0 +1,135 @@ +import argparse +import json +import math +import os +import re +import shutil +from pathlib import Path + +import cv2 +import numpy as np + + +def load_from_json(filename: Path): + """Load a dictionary from a JSON filename. + + Args: + filename: The filename to load from. + """ + assert filename.suffix == ".json" + with open(filename, encoding="UTF-8") as file: + return json.load(file) + +def is_mask_all_valid(mask_path): + mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE) + return np.all(mask > 200) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--input_json', type=str, required=True) + parser.add_argument('--output_dir', type=str, required=True) + parser.add_argument('--train_split_fraction', type=float, default= 0.8) + parser.add_argument('--eval_split_fraction', type=float, default= 0.1) + parser.add_argument('--min_frame_number', type=int, default=100) + parser.add_argument('--frames_per_direction', type=int, default=100) + args = parser.parse_args() + + assert os.path.isfile(args.input_json) and args.input_json.endswith('.json'), 'The input JSON file is missing!' + + os.makedirs(args.output_dir, exist_ok=True) + + transforms_data = load_from_json( Path(args.input_json) ) + exposures_data = load_from_json( Path(args.input_json).parent / 'exposures.json' ) + + + selected_left_frames = [] + selected_right_frames = [] + new_exposures = dict() + + os.makedirs(os.path.join(args.output_dir, 'images'), exist_ok=True) + os.makedirs(os.path.join(args.output_dir, 'masks'), exist_ok=True) + for i, frame in enumerate(transforms_data['frames']): + file_path = frame['file_path'] # e.g. images/left_e1_output_0010_0_linear.exr (frame num is 0010) + mask_path = str(Path(args.input_json).parent / frame['mask_path']) # e.g. masks/left_e1_output_0010_0.png + # get frame number + frame_num = int(re.search(r'(\d{4})', file_path).group(0)) + direction = re.search(r'(left|right)', file_path, re.IGNORECASE).group(0).lower() + + if int(frame_num) >= args.min_frame_number: + if direction == 'left': + if len(selected_left_frames) < args.frames_per_direction and is_mask_all_valid(mask_path): + selected_left_frames.append(frame) + + shutil.copy(str(Path(args.input_json).parent / file_path), os.path.join(args.output_dir, file_path)) + shutil.copy(str(Path(args.input_json).parent / file_path.replace('_linear.exr', '.png')), os.path.join(args.output_dir, file_path.replace('_linear.exr', '.png'))) + shutil.copy(mask_path, os.path.join(args.output_dir, frame['mask_path'])) + shutil.copy(mask_path.replace('.png', '_saturation_mask.png'), os.path.join(args.output_dir, frame['mask_path'].replace('.png', '_saturation_mask.png'))) + + new_exposures[file_path] = exposures_data[file_path] + else: + if len(selected_right_frames) < args.frames_per_direction and is_mask_all_valid(mask_path): + selected_right_frames.append(frame) + + shutil.copy(str(Path(args.input_json).parent / file_path), os.path.join(args.output_dir, file_path)) + shutil.copy(str(Path(args.input_json).parent / file_path.replace('_linear.exr', '.png')), os.path.join(args.output_dir, file_path.replace('_linear.exr', '.png'))) + shutil.copy(mask_path, os.path.join(args.output_dir, frame['mask_path'])) + shutil.copy(mask_path.replace('.png', '_saturation_mask.png'), os.path.join(args.output_dir, frame['mask_path'].replace('.png', '_saturation_mask.png'))) + + new_exposures[file_path] = exposures_data[file_path] + + + new_transforms_data = {} + all_keys = transforms_data.keys() + for key in all_keys: + new_transforms_data[key] = transforms_data[key] + + new_transforms_data['frames'] = selected_left_frames + selected_right_frames + + num_images_per_direction = len(selected_left_frames) + + num_train_images = math.ceil(num_images_per_direction * args.train_split_fraction) + num_eval_test_images = num_images_per_direction - num_train_images + i_all = np.arange(num_images_per_direction) + i_train = np.linspace( + 0, num_images_per_direction - 1, num_train_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval_test = np.setdiff1d(i_all, i_train) # eval images are the remaining images + assert len(i_eval_test) == num_eval_test_images + + num_eval_images = math.ceil(num_images_per_direction * args.eval_split_fraction) + num_test_images = num_eval_test_images - num_train_images + i_eval = np.linspace( + 0, num_eval_test_images - 1, num_eval_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval = i_eval_test[i_eval] + i_test = np.setdiff1d(i_eval_test, i_eval) # eval images are the remaining images + assert len(i_eval) == num_eval_images + + + # has_split_files_spec = any(f"{split}_filenames" in meta for split in ("train", "val", "test")) + + train_list = [] + for i in i_train: + train_list.append(selected_left_frames[i]["file_path"]) + train_list.append(selected_right_frames[i]["file_path"]) + new_transforms_data["train_filenames"] = train_list + + test_list = [] + for i in i_test: + test_list.append(selected_left_frames[i]["file_path"]) + test_list.append(selected_right_frames[i]["file_path"]) + new_transforms_data["test_filenames"] = test_list + + eval_list = [] + for i in i_eval: + eval_list.append(selected_left_frames[i]["file_path"]) + eval_list.append(selected_right_frames[i]["file_path"]) + new_transforms_data["val_filenames"] = eval_list + + with open(os.path.join(args.output_dir, 'transforms.json'), "w") as outfile: + json.dump(new_transforms_data, outfile, indent=4) + + with open(os.path.join(args.output_dir, 'exposures.json'), "w") as outfile: + json.dump(new_exposures, outfile, indent=4) + + print("All done!") From faa8a7dfb63609da561f3ea924df96f0e967792c Mon Sep 17 00:00:00 2001 From: lefreud Date: Wed, 8 May 2024 00:27:35 +0000 Subject: [PATCH 08/20] Fix gen subset --- lantern_scripts/generate_small_subset_hdr_nerf.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lantern_scripts/generate_small_subset_hdr_nerf.py b/lantern_scripts/generate_small_subset_hdr_nerf.py index 36f34b8bc..e9ab61010 100644 --- a/lantern_scripts/generate_small_subset_hdr_nerf.py +++ b/lantern_scripts/generate_small_subset_hdr_nerf.py @@ -4,6 +4,7 @@ import math import os import re import shutil +from copy import deepcopy from pathlib import Path import cv2 @@ -105,6 +106,18 @@ if __name__ == "__main__": i_test = np.setdiff1d(i_eval_test, i_eval) # eval images are the remaining images assert len(i_eval) == num_eval_images + hdr_nerf_transforms_data = dict() + hdr_nerf_exposures_data = dict() + for split, indices in zip(("train", "val", "test"), (i_train, i_eval, i_test)): + hdr_nerf_transforms_data[split] = deepcopy(transforms_data) + hdr_nerf_transforms_data[split]["frames"] = [selected_left_frames[i] for i in indices] + [selected_right_frames[i] for i in indices] + hdr_nerf_exposures_data[split] = {k: v for k, v in new_exposures.items() if k in [frame["file_path"] for frame in hdr_nerf_transforms_data[split]["frames"]]} + + with open(os.path.join(args.output_dir, f'transforms_{split}.json'), "w") as outfile: + json.dump(hdr_nerf_transforms_data[split], outfile, indent=4) + + with open(os.path.join(args.output_dir, f'exposure_{split}.json'), "w") as outfile: + json.dump(hdr_nerf_exposures_data[split], outfile, indent=4) # has_split_files_spec = any(f"{split}_filenames" in meta for split in ("train", "val", "test")) From be2b4360248ea5450fc9c5d5565d0144c9262921 Mon Sep 17 00:00:00 2001 From: lefreud Date: Fri, 10 May 2024 01:08:43 +0000 Subject: [PATCH 09/20] WIP hdr-nerfacto --- .gitignore | 1 + .../generate_small_subset_hdr_nerf.py | 4 ++-- nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 2 +- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 2 ++ nerfstudio/lantern/dataset.py | 14 +++++++------- nerfstudio/models/nerfacto.py | 18 +++++++++++++++--- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 661bf0136..6ec98f95b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Ours Dual_Cameras gt_test +hdr-nerfacto-subset-out # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/lantern_scripts/generate_small_subset_hdr_nerf.py b/lantern_scripts/generate_small_subset_hdr_nerf.py index e9ab61010..45133ed6d 100644 --- a/lantern_scripts/generate_small_subset_hdr_nerf.py +++ b/lantern_scripts/generate_small_subset_hdr_nerf.py @@ -29,8 +29,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--input_json', type=str, required=True) parser.add_argument('--output_dir', type=str, required=True) - parser.add_argument('--train_split_fraction', type=float, default= 0.8) - parser.add_argument('--eval_split_fraction', type=float, default= 0.1) + parser.add_argument('--train_split_fraction', type=float, default= 0.60) + parser.add_argument('--eval_split_fraction', type=float, default= 0.01) parser.add_argument('--min_frame_number', type=int, default=100) parser.add_argument('--frames_per_direction', type=int, default=100) args = parser.parse_args() diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index 3f726d115..362d5d2c5 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -247,7 +247,7 @@ class HdrNerfactoField(Field): exposures = torch.ones(ray_samples.frustums.directions.shape[:-1], device=ray_samples.frustums.directions.device)[..., None] else: exposures = ray_samples.metadata['exposures'].to(ray_samples.frustums.directions.device) - exposures = exposures.reshape(-1, 1).clone().detach() / 200 + exposures = exposures.reshape(-1, 1).clone().detach() assert density_embedding is not None outputs = {} diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index f10b1e005..662ab9f80 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -358,6 +358,7 @@ class HdrNerfactoModel(Model): def get_loss_dict(self, outputs, batch, metrics_dict=None): loss_dict = {} image = batch["image"].to(self.device) + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( pred_image=outputs["rgb"], pred_accumulation=outputs["accumulation"], @@ -369,6 +370,7 @@ class HdrNerfactoModel(Model): ) loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + # TODO: check that # if self.training: # loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( # outputs["weights_list"], outputs["ray_samples_list"] diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index b3d493941..a0dc0061a 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -58,9 +58,9 @@ class HDRNerfactoInputDataset(InputDataset): Args: image_idx: The image index in the dataset. """ - image_filename = self._dataparser_outputs.image_filenames[image_idx] + image_filename = str(self._dataparser_outputs.image_filenames[image_idx]).replace('_linear.exr', '.png') #print('image_filename', image_filename) - hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) + hdr_image = cv2.imread(str(image_filename)) # print('hdr_image', hdr_image.shape, hdr_image.dtype) # print('hdr_image', hdr_image.max(), hdr_image.min()) # print('hdr_image', hdr_image[0,0,:]) @@ -70,8 +70,8 @@ class HDRNerfactoInputDataset(InputDataset): height, width = hdr_image.shape[:2] newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) - hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) - hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) + hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) + hdr_image = hdr_image.astype("float32") / 255 # shape is (h, w) or (h, w, 3 or 4) if len(hdr_image.shape) == 2: hdr_image = hdr_image[:, :, None].repeat(3, axis=2) assert len(hdr_image.shape) == 3 @@ -87,11 +87,11 @@ class HDRNerfactoInputDataset(InputDataset): """ image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) #print('a', image.max()) - image = image * self._dataparser_outputs.exposures[image_idx] + # image = image * self._dataparser_outputs.exposures[image_idx] # print('b', image.max()) - if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: - image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) + # if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: + # image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) return image def get_data(self, image_idx: int) -> Dict: diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 1d243632b..c09b58c93 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -29,7 +29,11 @@ from torchmetrics.image import PeakSignalNoiseRatio from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle, RaySamples -from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField @@ -42,8 +46,16 @@ from nerfstudio.model_components.losses import ( pred_normal_loss, scale_gradients_by_distance_squared, ) -from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler, UniformSampler -from nerfstudio.model_components.renderers import AccumulationRenderer, DepthRenderer, NormalsRenderer, RGBRenderer +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) from nerfstudio.model_components.scene_colliders import NearFarCollider from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model, ModelConfig From 8b0668188fc41a2ddcb564ef64131ff71303df22 Mon Sep 17 00:00:00 2001 From: lefreud Date: Fri, 10 May 2024 12:55:31 +0000 Subject: [PATCH 10/20] Working hdr-nerfacto --- .gitignore | 1 + README-pandora.md | 5 ++--- nerfstudio/field_components/field_heads.py | 1 + nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 6 ++++++ nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 2 ++ nerfstudio/model_components/renderers.py | 4 ++-- nerfstudio/utils/colormaps.py | 6 ++++-- 7 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 6ec98f95b..9ea2dafce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ Dual_Cameras gt_test hdr-nerfacto-subset-out +ablations_out # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README-pandora.md b/README-pandora.md index 06a749fed..ae849bac3 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -132,9 +132,8 @@ It will be trained on the Laval indoor HDR dataset to do this task. TODO: validate this part -## 4. Train from scratch or fine-tune the NeRF model using MomoNet's predictions - - +## 4. Evaluation ``` +ns-eval --load-config outputs/LVSN_LAB_OpenSFM/hdr-nerfacto/2024-05-10_032723/config.yml --output-path ablations_out/hdr_nerfacto/output.json --render-output-path ablations_out/hdr-nerfacto ``` \ No newline at end of file diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index ff9831079..5ed438b15 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -43,6 +43,7 @@ class FieldHeadNames(Enum): GRADIENT = "gradient" RGB_HDR = "rgb_hdr" # HDR-Nerfacto + RGB_FAST = "rgb_fast" # HDR-Nerfacto PRED_RGB_LDR = "pred_rgb_ldr" # HDR-Nerfacto ZERO_RADIANCE_CRF = "zero_radiance_crf" # HDR-Nerfacto diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index 362d5d2c5..aff4aa922 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -318,6 +318,11 @@ class HdrNerfactoField(Field): log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions) rgb_hdr = torch.exp(log_rgb_hdr).view(*outputs_shape, -1).to(directions) log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures) + log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure + + pred_rgb_fast = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3) + ], dim=-1).view(*outputs_shape, -1).to(directions) pred_rgb_ldr = torch.cat([ self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) @@ -331,6 +336,7 @@ class HdrNerfactoField(Field): outputs.update({ FieldHeadNames.RGB: pred_rgb_ldr, + FieldHeadNames.RGB_FAST: pred_rgb_fast, FieldHeadNames.RGB_HDR: rgb_hdr, FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, }) diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 662ab9f80..65d6c6291 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -302,6 +302,7 @@ class HdrNerfactoModel(Model): ray_samples_list.append(ray_samples) rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) accumulation = self.renderer_accumulation(weights=weights) @@ -312,6 +313,7 @@ class HdrNerfactoModel(Model): outputs = { "rgb": rgb, + "rgb_fast": rgb_fast, "accumulation": accumulation, "depth": depth, "rgb_hdr": rgb_hdr, diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 24090872e..fbf2fc5a9 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -224,8 +224,8 @@ class RGBRenderer(nn.Module): rgb = self.combine_rgb( rgb, weights, background_color=background_color, ray_indices=ray_indices, num_rays=num_rays ) - if not self.training: - torch.clamp_(rgb, min=0.0, max=1.0) + # if not self.training: + # torch.clamp_(rgb, min=0.0, max=1.0) return rgb diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index cf15150cb..bb85a6005 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -43,9 +43,9 @@ class ColormapOptions: """ Whether to invert the output colormap """ exposure_scale: float = 0 """Scale the brightness prior to tonemapping by 2 ^ Exposure""" - use_mu_law: bool = True + use_mu_law: bool = False """Use mu-law de-compression for HDR images""" - use_gamma: bool = True + use_gamma: bool = False """Use gamma for HDR images""" @@ -74,12 +74,14 @@ def apply_colormap( img_gamma_22 = image if colormap_options.use_mu_law: + raise Exception('this shouldnt be used') u = 5000. img_gamma_22 = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1. img_gamma_22 /= u img_gamma_22 *= 2.**(colormap_options.exposure_scale) if colormap_options.use_gamma: + raise Exception('this shouldnt be used') img_gamma_22 = torch.pow(img_gamma_22, 1./2.2) return torch.clamp(img_gamma_22, 0., 1.) else: From 745bfa56e73427a09db1771005dadcafeefe920c Mon Sep 17 00:00:00 2001 From: lefreud Date: Sat, 11 May 2024 23:15:42 +0000 Subject: [PATCH 11/20] Cam optimized off, ns-render fix --- experiments.md | 28 +++++++++++++++++++++++ nerfstudio/hdr_nerfacto/config.py | 4 +--- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 2 +- nerfstudio/lantern/dataset.py | 5 ++++ nerfstudio/pipelines/base_pipeline.py | 19 ++++++++++++--- nerfstudio/scripts/render.py | 7 +++--- 6 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 experiments.md diff --git a/experiments.md b/experiments.md new file mode 100644 index 000000000..0941de33b --- /dev/null +++ b/experiments.md @@ -0,0 +1,28 @@ + + +`` +renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto +`` + +No camera optim + +outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-11_141957/config.yml + +Train: +``` +export SCENE=lab_downstairs_ns + +ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +``` + +Eval (GT+normal): +``` +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-11_141957 + +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr +``` + diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py index 5f223c917..e7b4c4b22 100644 --- a/nerfstudio/hdr_nerfacto/config.py +++ b/nerfstudio/hdr_nerfacto/config.py @@ -23,9 +23,7 @@ def get_hdr_nerfacto_config(): train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + mode="off" ), ), model=HdrNerfactoModelConfig( diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 65d6c6291..2b3eedebf 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -403,7 +403,7 @@ class HdrNerfactoModel(Model): accumulation=outputs["accumulation"], ) - combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + combined_rgb = predicted_rgb combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index a0dc0061a..f5f0b362d 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -102,6 +102,11 @@ class HDRNerfactoInputDataset(InputDataset): """ image = self.get_image(image_idx) data = {"image_idx": image_idx, "image": image} + + # TODO: this doesnt work for training + image_filename = self._dataparser_outputs.image_filenames[image_idx] + data["image_filename"] = str(image_filename.name) + if self._dataparser_outputs.mask_filenames is not None: mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 5e202b06c..3b58f1306 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -394,9 +394,22 @@ class VanillaPipeline(Pipeline): camera_indices = camera_ray_bundle.camera_indices assert camera_indices is not None for key, val in images_dict.items(): - Image.fromarray((val * 255).byte().cpu().numpy()).save( - output_path / "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key) - ) + if 'image_filename' in batch.keys(): + output_name = batch['image_filename'].split('.')[0] + '-' + key + if key == 'img': + output_name = output_name + '.' + batch['image_filename'].split('.')[-1] + else: + output_name = output_name + '.jpg' + else: + output_name = "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key) + if output_name.endswith('.exr'): + import cv2 + render_image = cv2.cvtColor(val.cpu().numpy(), cv2.COLOR_BGR2RGB) + cv2.imwrite(str(output_path / output_name), render_image) + else: + Image.fromarray((val * 255).byte().cpu().numpy()).save( + output_path / output_name + ) assert "num_rays_per_sec" not in metrics_dict metrics_dict["num_rays_per_sec"] = num_rays / (time() - inner_start) fps_str = "fps" diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index bd53890c8..da6856147 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -169,6 +169,10 @@ def _render_trajectory_video( .cpu() .numpy() ) + elif rendered_output_name == "rgb_fast" or rendered_output_name == "rgb" or rendered_output_name == "rgb_hdr": + output_image = ( + output_image.cpu().numpy() + ) elif is_validity: mask_f = torch.clip(outputs["validity_f"], 0, 1) mask_f = torch.ones(mask_f.shape).to(mask_f.get_device()) - mask_f @@ -186,9 +190,6 @@ def _render_trajectory_video( .numpy() ) elif image_format == "exr": - u = 5000. - output_image = torch.exp(output_image * torch.log(torch.tensor(u+1.))) - 1. - output_image /= u output_image = ( output_image.cpu().numpy() ) From 3ba1d6b767b982c891df6b2670e050e89b864897 Mon Sep 17 00:00:00 2001 From: lefreud Date: Mon, 13 May 2024 00:05:46 +0000 Subject: [PATCH 12/20] wo_CRF --- README-pandora.md | 20 ++++++- experiments.md | 23 +++++++- nerfstudio/configs/method_configs.py | 4 +- nerfstudio/hdr_nerfacto/config.py | 19 ++++-- nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 71 +++++++++++++---------- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 55 ++++++++++++------ nerfstudio/lantern/datamanager.py | 13 ++++- 7 files changed, 144 insertions(+), 61 deletions(-) diff --git a/README-pandora.md b/README-pandora.md index ae849bac3..5a4012072 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -115,7 +115,7 @@ After convergence, go to the next step. ns-train lantern-nerfacto --help ``` -If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30` +If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30` > Thing to keep in mind: the way Nerfstudio splits train and test images may not be optimal: left might be in one dataset and right in another... that's a bit of a data leak. @@ -134,6 +134,22 @@ TODO: validate this part ## 4. Evaluation +``` +ns-train hdr-nerfacto --data data/lab_downstairs_ns --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +``` + ``` ns-eval --load-config outputs/LVSN_LAB_OpenSFM/hdr-nerfacto/2024-05-10_032723/config.yml --output-path ablations_out/hdr_nerfacto/output.json --render-output-path ablations_out/hdr-nerfacto -``` \ No newline at end of file +``` + +``` +ns-render camera-path --load-config outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-10_211317/config.yml --camera-path-filename data/lab_downstairs_ns/GT_transforms.json --output-path renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto --output-format images --image-format exr +``` + +``` +docker run -it --rm -v /gel/usr/$USER:/mnt/workspace/ -v /home-local2/$USER.extra.nobkp/:/mnt/data/ --gpus '"device=1"' -p 7007:7007 lantern_docker:latest /bin/bash +export PYTHONPATH=$PYTHONPATH:$(pwd) +``` + +docker exec -it 57d2bf549c9a /bin/bash +ns-train hdr-nerfacto --data data/lab_downstairs_ns/ --pipeline.datamanager.train-num-images-to-sample-from 1800 --viewer.websocket-port 9009 diff --git a/experiments.md b/experiments.md index 0941de33b..ae9abdc05 100644 --- a/experiments.md +++ b/experiments.md @@ -12,7 +12,7 @@ Train: ``` export SCENE=lab_downstairs_ns -ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 ``` Eval (GT+normal): @@ -21,8 +21,25 @@ export SCENE=lab_downstairs_ns export MODEL_NAME=hdr-nerfacto/2024-05-11_141957 ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ -ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \ -ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr ``` +Move (outside docker) +``` +mkdir -p /gel/usr/mokad6/ml_class/fred-out/ ; \ +cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/ablations_out /gel/usr/mokad6/ml_class/fred-out/ns-eval ; \ +cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/renders /gel/usr/mokad6/ml_class/fred-out/ns-render +``` + +1800 samples: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 + + + +Test wo CRF + +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 563d7e2df..38776f3da 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -78,6 +78,7 @@ descriptions = { "nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.", "lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", "hdr-nerfacto": "HDR-Nerfacto, adapted to the Nerfacto model.", + "hdr-nerfacto-wo-crf": "HDR-Nerfacto without CRF, adapted to the Nerfacto model.", "depth-nerfacto": "Nerfacto with depth supervision.", "volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", @@ -128,7 +129,8 @@ method_configs["nerfacto"] = TrainerConfig( method_configs["lantern-nerfacto"] = Get_lantern_config() -method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config() +method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True) +method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False) method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py index e7b4c4b22..ecb9168dc 100644 --- a/nerfstudio/hdr_nerfacto/config.py +++ b/nerfstudio/hdr_nerfacto/config.py @@ -6,19 +6,27 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.hdr_nerfacto.hdr_nerf_model import HdrNerfactoModelConfig -from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig +from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig, HDRNerfactoWoCrfDataManagerConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig -def get_hdr_nerfacto_config(): +def get_hdr_nerfacto_config(*, use_crf=True): + if use_crf: + method_name = "hdr-nerfacto" + else: + method_name = "hdr-nerfacto-wo-crf" + + datamanager_config_class = HDRNerfactoDataManagerConfig if use_crf else HDRNerfactoWoCrfDataManagerConfig + return TrainerConfig( - method_name="hdr-nerfacto", + method_name=method_name, steps_per_eval_batch=500, steps_per_save=2000, + # max_num_iterations=1000, max_num_iterations=30000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=HDRNerfactoDataManagerConfig( + datamanager=datamanager_config_class( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, @@ -27,7 +35,8 @@ def get_hdr_nerfacto_config(): ), ), model=HdrNerfactoModelConfig( - eval_num_rays_per_chunk=1 << 15 + eval_num_rays_per_chunk=1 << 15, + use_crf=use_crf, ), ), optimizers={ diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index aff4aa922..9143339bc 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -96,6 +96,7 @@ class HdrNerfactoField(Field): use_average_appearance_embedding: bool = False, spatial_distortion: Optional[SpatialDistortion] = None, implementation: Literal["tcnn", "torch"] = "tcnn", + use_crf: bool = True, ) -> None: super().__init__() @@ -200,17 +201,19 @@ class HdrNerfactoField(Field): implementation=implementation, ) - # HDR NeRF, 1 MLP per channel - self.crf_mlps = nn.ModuleList([ - MLP(in_dim=1, - num_layers=2, - layer_width=128, - out_dim=1, - activation=nn.ReLU(), - out_activation=nn.Sigmoid(), - implementation=implementation, - ) for _ in range(3) - ]) + self.use_crf = use_crf + if use_crf: + # HDR NeRF, 1 MLP per channel + self.crf_mlps = nn.ModuleList([ + MLP(in_dim=1, + num_layers=2, + layer_width=128, + out_dim=1, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) for _ in range(3) + ]) def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: @@ -316,29 +319,37 @@ class HdrNerfactoField(Field): dim=-1, ) log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions) - rgb_hdr = torch.exp(log_rgb_hdr).view(*outputs_shape, -1).to(directions) + log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures) - log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure + #print('exposure_min', exposures.min(), 'exposure_max', exposures.max()) + # TODO: validate this + rgb_hdr = torch.exp(log_rgb_hdr_exposed).view(*outputs_shape, -1).to(directions) + #print('rgb_hdr_min', rgb_hdr.min(), 'rgb_hdr_max', rgb_hdr.max()) - pred_rgb_fast = torch.cat([ - self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3) - ], dim=-1).view(*outputs_shape, -1).to(directions) + # outputs[FieldHeadNames.RGB_HDR] = torch.clamp(rgb_hdr, 0, 1) + outputs[FieldHeadNames.RGB_HDR] = rgb_hdr - pred_rgb_ldr = torch.cat([ - self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) - ], dim=-1) - pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions) + if self.use_crf: + log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure + pred_rgb_fast = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3) + ], dim=-1).view(*outputs_shape, -1).to(directions) - zero_radiance_crf = torch.cat([ - self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3) - ], dim=-1) - zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions) + pred_rgb_ldr = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) + ], dim=-1) + pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions) - outputs.update({ - FieldHeadNames.RGB: pred_rgb_ldr, - FieldHeadNames.RGB_FAST: pred_rgb_fast, - FieldHeadNames.RGB_HDR: rgb_hdr, - FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, - }) + zero_radiance_crf = torch.cat([ + self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3) + ], dim=-1) + zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions) + + outputs.update({ + FieldHeadNames.RGB: pred_rgb_ldr, + FieldHeadNames.RGB_FAST: pred_rgb_fast, + FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, + }) return outputs + diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 2b3eedebf..991ca55df 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -142,6 +142,8 @@ class HdrNerfactoModelConfig(ModelConfig): unit_exposure_loss_mult: float = 0.5 unit_exposure_expected_value: float = 0.5 + use_crf: bool = True + class HdrNerfactoModel(Model): """Nerfacto model @@ -178,6 +180,7 @@ class HdrNerfactoModel(Model): use_average_appearance_embedding=self.config.use_average_appearance_embedding, appearance_embedding_dim=self.config.appearance_embed_dim, implementation=self.config.implementation, + use_crf=self.config.use_crf, ) self.density_fns = [] @@ -301,24 +304,25 @@ class HdrNerfactoModel(Model): weights_list.append(weights) ray_samples_list.append(ray_samples) - rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) - rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) - depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) - accumulation = self.renderer_accumulation(weights=weights) - # HDR-Nerfacto rgb_hdr = self.renderer_rgb( rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights ) + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + accumulation = self.renderer_accumulation(weights=weights) + outputs = { - "rgb": rgb, - "rgb_fast": rgb_fast, "accumulation": accumulation, "depth": depth, "rgb_hdr": rgb_hdr, - "zero_radiance_crf": field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF], } + if self.config.use_crf: + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) + outputs["rgb"] = rgb + outputs["rgb_fast"] = rgb_fast + outputs['zero_radiance_crf'] = field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF] if self.config.predict_normals: normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) @@ -350,7 +354,10 @@ class HdrNerfactoModel(Model): metrics_dict = {} gt_rgb = batch["image"].to(self.device) # RGB or RGBA image gt_rgb = self.renderer_rgb.blend_background(gt_rgb) # Blend if RGBA - predicted_rgb = outputs["rgb"] + if self.config.use_crf: + predicted_rgb = outputs["rgb"] + else: + predicted_rgb = outputs["rgb_hdr"] metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb) if self.training: @@ -361,17 +368,24 @@ class HdrNerfactoModel(Model): loss_dict = {} image = batch["image"].to(self.device) - pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( - pred_image=outputs["rgb"], - pred_accumulation=outputs["accumulation"], - gt_image=image, - ) - - loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss( - outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value - ) + if self.config.use_crf: + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) + loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss( + outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value + ) + else: + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb_hdr"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + # TODO: check that # if self.training: # loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( @@ -395,7 +409,10 @@ class HdrNerfactoModel(Model): self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: gt_rgb = batch["image"].to(self.device) - predicted_rgb = outputs["rgb"] # Blended with background (black if random background) + if self.config.use_crf: + predicted_rgb = outputs["rgb"] + else: + predicted_rgb = outputs["rgb_hdr"] gt_rgb = self.renderer_rgb.blend_background(gt_rgb) acc = colormaps.apply_colormap(outputs["accumulation"]) depth = colormaps.apply_depth_colormap( diff --git a/nerfstudio/lantern/datamanager.py b/nerfstudio/lantern/datamanager.py index be88eb6be..b9e2a5d3f 100644 --- a/nerfstudio/lantern/datamanager.py +++ b/nerfstudio/lantern/datamanager.py @@ -9,7 +9,11 @@ from nerfstudio.data.datamanagers.base_datamanager import ( VanillaDataManager, VanillaDataManagerConfig, ) -from nerfstudio.lantern.dataset import HDRInputDataset, HDRNerfactoInputDataset +from nerfstudio.lantern.dataset import ( + HDRInputDataset, + HDRNerfactoInputDataset, + HDRNerfactoWoCrfInputDataset, +) @dataclass @@ -19,6 +23,13 @@ class HDRNerfactoDataManagerConfig(VanillaDataManagerConfig): _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoInputDataset]) """Target class to instantiate.""" +@dataclass +class HDRNerfactoWoCrfDataManagerConfig(VanillaDataManagerConfig): + """A HDR data manager, based on VanillaDataManager""" + + _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoWoCrfInputDataset]) + """Target class to instantiate.""" + @dataclass class HDRVanillaDataManagerConfig(VanillaDataManagerConfig): From 9bac48382f2641ba07a8b82bb93cf7a593dcf5e8 Mon Sep 17 00:00:00 2001 From: lefreud Date: Mon, 13 May 2024 00:06:17 +0000 Subject: [PATCH 13/20] missin file --- nerfstudio/lantern/dataset.py | 117 ++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index f5f0b362d..f0e1fd49c 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -151,6 +151,123 @@ class HDRNerfactoInputDataset(InputDataset): return self._dataparser_outputs.image_filenames +class HDRNerfactoWoCrfInputDataset(InputDataset): + """Dataset that returns HDR images, OpenEXR. + + Args: + dataparser_outputs: description of where and how to read input images. + scale_factor: The scaling factor for the dataparser outputs + """ + + # exclude_batch_keys_from_device: List[str] = ["image", "mask"] + # cameras: Cameras + + def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): + super().__init__(dataparser_outputs, scale_factor) + + def __len__(self): + return len(self._dataparser_outputs.image_filenames) + + def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: + """Returns the image of shape (H, W, 3 or 4), with np.float32 values. + + Args: + image_idx: The image index in the dataset. + """ + image_filename = self._dataparser_outputs.image_filenames[image_idx] + #print('image_filename', image_filename) + hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) + # print('hdr_image', hdr_image.shape, hdr_image.dtype) + # print('hdr_image', hdr_image.max(), hdr_image.min()) + # print('hdr_image', hdr_image[0,0,:]) + + if self.scale_factor != 1.0: + # Here cv2 img shape is (height, width, BGR) + height, width = hdr_image.shape[:2] + newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) + hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) + hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) + hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) + if len(hdr_image.shape) == 2: + hdr_image = hdr_image[:, :, None].repeat(3, axis=2) + assert len(hdr_image.shape) == 3 + assert hdr_image.dtype == np.float32 + assert hdr_image.shape[2] in [3, 4], f"Image shape of {hdr_image.shape} is in correct." + return hdr_image + + def get_image(self, image_idx: int) -> Float[Tensor, "image_height image_width num_channels"]: + """Returns a 3 channel image. + + Args: + image_idx: The image index in the dataset. + """ + image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) + #print('a', image.max()) + # image = image * self._dataparser_outputs.exposures[image_idx] + # print('b', image.max()) + + # if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: + # image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) + return image + + def get_data(self, image_idx: int) -> Dict: + """Returns the ImageDataset data as a dictionary. + + Args: + image_idx: The image index in the dataset. + """ + image = self.get_image(image_idx) + data = {"image_idx": image_idx, "image": image} + + # TODO: this doesnt work for training + image_filename = self._dataparser_outputs.image_filenames[image_idx] + data["image_filename"] = str(image_filename.name) + + if self._dataparser_outputs.mask_filenames is not None: + mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] + data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) + assert ( + data["mask"].shape[:2] == data["image"].shape[:2] + ), f"Mask and image have different shapes. Got {data['mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.saturation_mask_filenames is not None: + saturation_mask_filepath = self._dataparser_outputs.saturation_mask_filenames[image_idx] + data["saturation_mask"] = get_image_mask_tensor_from_path(filepath=saturation_mask_filepath, scale_factor=self.scale_factor) + assert ( + data["saturation_mask"].shape[:2] == data["image"].shape[:2] + ), f"Saturation mask and image have different shapes. Got {data['saturation_mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.exposures is not None: + exposure_value = self._dataparser_outputs.exposures[image_idx] + data["exposure"] = exposure_value + metadata = self.get_metadata(data) + data.update(metadata) + return data + + def get_metadata(self, data: Dict) -> Dict: + """Method that can be used to process any additional metadata that may be part of the model inputs. + + Args: + image_idx: The image index in the dataset. + """ + del data + return {} + + def __getitem__(self, image_idx: int) -> Dict: + data = self.get_data(image_idx) + return data + + @property + def image_filenames(self) -> List[Path]: + """ + Returns image filenames for this dataset. + The order of filenames is the same as in the Cameras object for easy mapping. + """ + + return self._dataparser_outputs.image_filenames + + + class HDRInputDataset(InputDataset): """Dataset that returns HDR images, OpenEXR. From 1fa5cba2d7773ec28c944480929ad2ea20750a51 Mon Sep 17 00:00:00 2001 From: lefreud Date: Mon, 13 May 2024 02:09:24 +0000 Subject: [PATCH 14/20] Clipped HDR-nerfacto --- nerfstudio/configs/method_configs.py | 5 +++-- nerfstudio/data/pixel_samplers.py | 2 +- nerfstudio/field_components/field_heads.py | 3 ++- nerfstudio/hdr_nerfacto/config.py | 9 +++++++-- nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 2 +- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 19 ++++++++++++++++--- nerfstudio/model_components/renderers.py | 4 ++-- 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 38776f3da..5b092a56f 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -129,8 +129,9 @@ method_configs["nerfacto"] = TrainerConfig( method_configs["lantern-nerfacto"] = Get_lantern_config() -method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True) -method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False) +method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True, clip_before_accumulation=False) +method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False, clip_before_accumulation=False) +method_configs["hdr-nerfacto-wo-crf-clip-bf-acc"] = get_hdr_nerfacto_config(use_crf=False, clip_before_accumulation=True) method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index c2fe2407d..1b8533e9f 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -213,7 +213,7 @@ class PixelSampler: c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) c, y, x = c.cpu(), y.cpu(), x.cpu() collated_batch = { - key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and value is not None + key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and key != "image_filename" and value is not None } for key, value in batch.items(): diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index 5ed438b15..8f238912a 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -42,7 +42,8 @@ class FieldHeadNames(Enum): ALPHA = "alpha" GRADIENT = "gradient" - RGB_HDR = "rgb_hdr" # HDR-Nerfacto + RGB_HDR = "rgb_hdr" # HDR-Nerfacto` + RGB_LINEAR_CLIPPED = "rgb_linear_clipped" # HDR-Nerfacto RGB_FAST = "rgb_fast" # HDR-Nerfacto PRED_RGB_LDR = "pred_rgb_ldr" # HDR-Nerfacto ZERO_RADIANCE_CRF = "zero_radiance_crf" # HDR-Nerfacto diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py index ecb9168dc..b9a4876bd 100644 --- a/nerfstudio/hdr_nerfacto/config.py +++ b/nerfstudio/hdr_nerfacto/config.py @@ -10,11 +10,15 @@ from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig, HDRNerf from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig -def get_hdr_nerfacto_config(*, use_crf=True): +def get_hdr_nerfacto_config(*, use_crf=True, clip_before_accumulation=False): if use_crf: method_name = "hdr-nerfacto" + assert not clip_before_accumulation else: - method_name = "hdr-nerfacto-wo-crf" + if clip_before_accumulation: + method_name = "hdr-nerfacto-wo-crf-clip-bf-acc" + else: + method_name = "hdr-nerfacto-wo-crf" datamanager_config_class = HDRNerfactoDataManagerConfig if use_crf else HDRNerfactoWoCrfDataManagerConfig @@ -37,6 +41,7 @@ def get_hdr_nerfacto_config(*, use_crf=True): model=HdrNerfactoModelConfig( eval_num_rays_per_chunk=1 << 15, use_crf=use_crf, + clip_before_accumulation=clip_before_accumulation, ), ), optimizers={ diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index 9143339bc..e5af2dff5 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -326,7 +326,7 @@ class HdrNerfactoField(Field): rgb_hdr = torch.exp(log_rgb_hdr_exposed).view(*outputs_shape, -1).to(directions) #print('rgb_hdr_min', rgb_hdr.min(), 'rgb_hdr_max', rgb_hdr.max()) - # outputs[FieldHeadNames.RGB_HDR] = torch.clamp(rgb_hdr, 0, 1) + outputs[FieldHeadNames.RGB_LINEAR_CLIPPED] = torch.clamp(rgb_hdr, 0, 1) outputs[FieldHeadNames.RGB_HDR] = rgb_hdr if self.use_crf: diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index 991ca55df..5b14835b4 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -38,6 +38,7 @@ from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField from nerfstudio.hdr_nerfacto.hdr_nerf_field import HdrNerfactoField +from nerfstudio.lantern.renderer import RGBRenderer_HDR from nerfstudio.model_components.losses import ( MSELoss, distortion_loss, @@ -143,6 +144,7 @@ class HdrNerfactoModelConfig(ModelConfig): unit_exposure_expected_value: float = 0.5 use_crf: bool = True + clip_before_accumulation: bool = False class HdrNerfactoModel(Model): @@ -237,6 +239,7 @@ class HdrNerfactoModel(Model): # renderers self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) + self.renderer_rgb_hdr = RGBRenderer_HDR(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer() self.renderer_normals = NormalsRenderer() @@ -305,9 +308,18 @@ class HdrNerfactoModel(Model): ray_samples_list.append(ray_samples) # HDR-Nerfacto - rgb_hdr = self.renderer_rgb( + rgb_hdr = self.renderer_rgb_hdr( rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights ) + if self.config.clip_before_accumulation: + rgb_linear_clipped = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.RGB_LINEAR_CLIPPED], weights=weights + ) + else: + # clipping only after accumulation (in the renderer) + rgb_linear_clipped = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights + ) depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) accumulation = self.renderer_accumulation(weights=weights) @@ -316,6 +328,7 @@ class HdrNerfactoModel(Model): "accumulation": accumulation, "depth": depth, "rgb_hdr": rgb_hdr, + "rgb_linear_clipped": rgb_linear_clipped, } if self.config.use_crf: rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) @@ -379,7 +392,7 @@ class HdrNerfactoModel(Model): ) else: pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( - pred_image=outputs["rgb_hdr"], + pred_image=outputs["rgb_linear_clipped"], pred_accumulation=outputs["accumulation"], gt_image=image, ) @@ -412,7 +425,7 @@ class HdrNerfactoModel(Model): if self.config.use_crf: predicted_rgb = outputs["rgb"] else: - predicted_rgb = outputs["rgb_hdr"] + predicted_rgb = outputs["rgb_linear_clipped"] gt_rgb = self.renderer_rgb.blend_background(gt_rgb) acc = colormaps.apply_colormap(outputs["accumulation"]) depth = colormaps.apply_depth_colormap( diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index fbf2fc5a9..24090872e 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -224,8 +224,8 @@ class RGBRenderer(nn.Module): rgb = self.combine_rgb( rgb, weights, background_color=background_color, ray_indices=ray_indices, num_rays=num_rays ) - # if not self.training: - # torch.clamp_(rgb, min=0.0, max=1.0) + if not self.training: + torch.clamp_(rgb, min=0.0, max=1.0) return rgb From fb0c4e1da0ef12c3f30a151b6d3caabc1e990216 Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Thu, 16 May 2024 13:27:20 -0400 Subject: [PATCH 15/20] Update experiments --- experiments.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) mode change 100644 => 100755 experiments.md diff --git a/experiments.md b/experiments.md old mode 100644 new mode 100755 index ae9abdc05..9c5ee8a26 --- a/experiments.md +++ b/experiments.md @@ -37,9 +37,68 @@ cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/renders /gel/usr/mokad6/ml_ export SCENE=lab_downstairs_ns export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 +1600 samples: +export SCENE=JF_office_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_152107 + + Test wo CRF export SCENE=lab_downstairs_ns export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 + + + +## Fix pixel sampler + +1800 samples, fix pixel sampler: +export SCENE=Henry_Gagnon_blue_et_rouge_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_175753 +Exp: 0.002..... +DONE + +1800 samples, fix pixel sampler: +export SCENE=JF_office_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_182719 +Exp: 0.008 + +1800 samples, fix pixel sampler: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_191020 +Exp: 0.01 + +1800 samples, fix pixel sampler: +export SCENE=meeting_room_ns +export MODEL_NAME=hdr-nerfacto/2024-05-15_203121 +Exp: 0.008 + +1800 samples, w/o CRF, fix pixel sampler: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto-wo-crf/2024-05-14_194316 +Exp: 0.01 + +1800 samples, w/o CRF, clip before acc, fix pixel sampler: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-14_200611 +Exp: 0.01 + + +## Export all + +export SCENE=Henry_Gagnon_blue_et_rouge_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-14_175753 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=JF_office_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-14_182719 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-14_191020 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf/2024-05-14_194316 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-14_200611 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME From 54afcad79936282c0a540760d64ffa6f67595c87 Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Thu, 16 May 2024 13:27:33 -0400 Subject: [PATCH 16/20] gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 .gitignore diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 9ea2dafce..19e4e736a --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ Dual_Cameras gt_test hdr-nerfacto-subset-out ablations_out +ablations* +renders* # Byte-compiled / optimized / DLL files __pycache__/ From d7d766c0d75652a08d0f1882def3110da66f9ccb Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Fri, 17 May 2024 21:28:26 -0400 Subject: [PATCH 17/20] Fast pixel sampler --- nerfstudio/data/pixel_samplers.py | 109 ++++++++++++++++-------------- 1 file changed, 58 insertions(+), 51 deletions(-) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 1b8533e9f..62af4c768 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -61,6 +61,8 @@ class PixelSampler: config: PixelSamplerConfig states_of_mask: int=0 masks_indices: torch.Tensor + is_well_exposed_computed: bool = False + well_exposed_indices: torch.Tensor def __init__(self, config: PixelSamplerConfig, **kwargs) -> None: self.kwargs = kwargs self.config = config @@ -77,7 +79,40 @@ class PixelSampler: num_rays_per_batch: number of rays to sample per batch """ self.num_rays_per_batch = num_rays_per_batch + + def sample_considering_mask(self, mask, device, batch_size): + num_masks = mask.shape[0] + indices = torch.tensor([]).cuda().to(device) + nonzero_indices_whole = torch.tensor([]).to(device) + + if num_masks > 0: + size_of_disk = 50 + num_disks = ceil(num_masks / size_of_disk) + # slice mask to 100 disk + random_idx_disk = int(torch.randint(0, num_disks, (1,))) + + for current_disk, start in enumerate(range(0, num_masks, size_of_disk)): + is_last_disk = ((start//size_of_disk) == num_disks-1) + end = start + size_of_disk if not is_last_disk else num_masks + current_masks = mask[start:end,...].to(device) + nonzero_indices = torch.nonzero(current_masks, as_tuple=False) + # Add offset to indices:local indices --> global + nonzero_indices[:,0] += start + + num_samples = batch_size // (num_disks-1) if current_disk != random_idx_disk \ + else batch_size % (num_disks-1) + chosen_indices = torch.randint(0, nonzero_indices.shape[0], (num_samples,), device=device) + + indice_this_disk = nonzero_indices[chosen_indices] + indices = torch.cat((indices, indice_this_disk), dim=0).long() + nonzero_indices_whole = torch.cat((nonzero_indices_whole, nonzero_indices.cpu()), dim=0) + + del current_masks + del nonzero_indices + del indice_this_disk + return indices, nonzero_indices_whole + @profiler.time_function def sample_method( self, @@ -97,49 +132,21 @@ class PixelSampler: mask: mask of possible pixels in an image to sample from. """ # TODO Defining a flag for sampling even when there is a mask - if isinstance(mask, torch.Tensor) and mask.shape[0] >= 4: + if isinstance(mask, torch.Tensor): # mask size: batch * h * w * 1 # Init state, cache masks indices. - if self.states_of_mask == 0: - with profiler.time_function("mask squeeze: whole"): - mask = mask.squeeze(-1) - num_masks = mask.shape[0] - size_of_disk = 2 - num_disks = ceil(num_masks / size_of_disk) - # slice mask to 100 disk - random_idx_disk = int(torch.randint(0, num_disks, (1,))) - indices = torch.tensor([]).to(device) - nonzero_indices_whole = torch.tensor([]) - - for current_disk, start in enumerate(range(0, num_masks, size_of_disk)): - is_last_disk = ((start//size_of_disk) == num_disks-1) - end = start + size_of_disk if not is_last_disk else num_masks - current_masks = mask[start:end,...].to(device) - nonzero_indices = torch.nonzero(current_masks, as_tuple=False) - # Add offset to indices:local indices --> global - nonzero_indices[:,0] += start - - num_samples = batch_size // (num_disks-1) if current_disk != random_idx_disk \ - else batch_size % (num_disks-1) - chosen_indices = torch.randint(0, nonzero_indices.shape[0], (num_samples,), device=device) - - indice_this_disk = nonzero_indices[chosen_indices] - indices = torch.cat((indices, indice_this_disk), dim=0).long() - nonzero_indices_whole = torch.cat((nonzero_indices_whole, nonzero_indices.cpu()), dim=0).long() + with profiler.time_function("mask squeeze: whole"): + mask = mask.squeeze(-1) - del current_masks - del nonzero_indices - del indice_this_disk - - self.masks_indices = nonzero_indices_whole.cpu() - self.states_of_mask = 1 + if self.is_well_exposed_computed: + chosen_indices = torch.randint(0, self.well_exposed_indices.shape[0], (batch_size,)) + return self.well_exposed_indices[chosen_indices].to(device) + else: + indices, nonzero_indices_whole = self.sample_considering_mask(mask, device, batch_size) + self.well_exposed_indices = nonzero_indices_whole.long().cpu() + self.is_well_exposed_computed = True return indices - - elif self.states_of_mask == 1: - chosen_indices = torch.randint(0, self.masks_indices.shape[0], (batch_size,)) - return self.masks_indices[chosen_indices].to(device) - - + else: indices = torch.floor( torch.rand((batch_size, 3), device=device) @@ -210,22 +217,20 @@ class PixelSampler: else: indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device) - c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) - c, y, x = c.cpu(), y.cpu(), x.cpu() + chan, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) + chan, y, x = chan.cpu(), y.cpu(), x.cpu() collated_batch = { - key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and key != "image_filename" and value is not None + key: value[chan, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and key != "image_filename" and value is not None } - - for key, value in batch.items(): - if key == "exposure": - collated_batch["exposure"] = value[c] - assert collated_batch["image"].shape[0] == num_rays_per_batch # Needed to correct the random indices to their actual camera idx locations. - indices[:, 0] = batch["image_idx"][c] - collated_batch["indices"] = indices # with the abs camera indices + collated_batch["indices"] = indices.clone() # with the abs camera indices + collated_batch["indices"][:, 0] = batch["image_idx"][chan].clone() + + if "exposure" in batch: + collated_batch["exposure"] = batch["exposure"][chan] if keep_full_image: collated_batch["full_image"] = batch["image"] @@ -369,7 +374,8 @@ class PatchPixelSampler(PixelSampler): if isinstance(mask, Tensor): sub_bs = batch_size // (self.config.patch_size**2) half_patch_size = int(self.config.patch_size / 2) - m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=half_patch_size) + # m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=half_patch_size) + m = mask.permute(0, 3, 1, 2).float() nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device) chosen_indices = random.sample(range(len(nonzero_indices)), k=sub_bs) indices = nonzero_indices[chosen_indices] @@ -448,7 +454,8 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods device: Union[torch.device, str] = "cpu", ) -> Int[Tensor, "batch_size 3"]: if isinstance(mask, Tensor): - m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius) + # m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius) + m = mask.permute(0, 3, 1, 2).float() nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device) chosen_indices = random.sample(range(len(nonzero_indices)), k=self.rays_to_sample) indices = nonzero_indices[chosen_indices] From b990b60dc2b34cea1a020185681142b4fdc45a00 Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Fri, 17 May 2024 21:28:37 -0400 Subject: [PATCH 18/20] 60K iterations --- nerfstudio/hdr_nerfacto/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py index b9a4876bd..7f3af16ba 100644 --- a/nerfstudio/hdr_nerfacto/config.py +++ b/nerfstudio/hdr_nerfacto/config.py @@ -27,7 +27,7 @@ def get_hdr_nerfacto_config(*, use_crf=True, clip_before_accumulation=False): steps_per_eval_batch=500, steps_per_save=2000, # max_num_iterations=1000, - max_num_iterations=30000, + max_num_iterations=60000, mixed_precision=True, pipeline=VanillaPipelineConfig( datamanager=datamanager_config_class( From f3d34d442f1fa5205b3b31ac704dc579c8872d4a Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Sat, 18 May 2024 19:48:08 -0400 Subject: [PATCH 19/20] Exp --- experiments.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/experiments.md b/experiments.md index 9c5ee8a26..652b78597 100755 --- a/experiments.md +++ b/experiments.md @@ -12,7 +12,7 @@ Train: ``` export SCENE=lab_downstairs_ns -ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30 +ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 1800 ``` Eval (GT+normal): @@ -102,3 +102,47 @@ ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablati export SCENE=lab_downstairs_ns ; \ export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-14_200611 ; \ ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME + +### Models trained for 60K steps + +outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-18_012547/config.yml +outputs/meeting_room_ns/hdr-nerfacto/2024-05-18_020406/config.yml + +fast_exp=0.01 +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-18_012547 ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +export SCENE=meeting_room_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-18_020406 ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +fast_exp=0.004 +export SCENE=my_apartment_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-18_133026 ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +fast_exp=0.004 +export SCENE=LVSN_LAB_OpenSFM +export MODEL_NAME=hdr-nerfacto/2024-05-18_143132 +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + + +### Ablations 60K + + +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf/2024-05-18_171020 +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-18_174929 +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr From 8ee91791d952b1720f257d94605a0b6eb4bb5bed Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Sun, 19 May 2024 17:08:41 -0400 Subject: [PATCH 20/20] Deadline results --- lantern_scripts/combine.py | 106 ++++++++++++++++++ linearize-ricoh-z10-wb-3500.json | 2 +- linearize-ricoh-z10-wb-5000.json | 1 - video_rendering_code/hdr_blender.py | 58 ++++++++++ video_rendering_code/render_video.sh | 160 +++++++++++++++++++++++++++ 5 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 lantern_scripts/combine.py delete mode 100644 linearize-ricoh-z10-wb-5000.json create mode 100644 video_rendering_code/hdr_blender.py create mode 100644 video_rendering_code/render_video.sh diff --git a/lantern_scripts/combine.py b/lantern_scripts/combine.py new file mode 100644 index 000000000..7c2035113 --- /dev/null +++ b/lantern_scripts/combine.py @@ -0,0 +1,106 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json + + +FAST_EXPOSURE_CUTOFF = 0.1 # good for real data (0.1 for synthetic) +WELL_EXPOSURE_CUTOFF = 0.98 # good for real data (0.9 for synthetic) + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + +def cut_weighting_function(pixels, exposures): + weights = np.zeros(pixels.shape, dtype=np.float32) + + if exposures == WELL_EXPOSURE: + weights = np.clip(-20.0 * (pixels - WELL_EXPOSURE_CUTOFF) + 1, 0.0, 1.0) + else: + weights = np.clip(100.0 * (pixels - FAST_EXPOSURE_CUTOFF), 0.0, 1.0) + + return weights + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--well_dir', type=str) + parser.add_argument('--fast_dir', type=str) + parser.add_argument('--out_dir', type=str) + parser.add_argument('--do_linearization', action='store_true') + parser.add_argument('--experiment_location', type=str, required=True) + args = parser.parse_args() + + experiment_setup = args.experiment_location + '/capture_settings.json' + assert os.path.exists(experiment_setup), "!!!! Cannot find the files for exposure setup. !!!" + with open(experiment_setup) as f: + data = json.load(f) + WELL_EXPOSURE = 1.0 + FAST_EXPOSURE = data["right"]["shutter_speed"] / data["left"]["shutter_speed"] + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + well_addr = args.well_dir + + with open(f'linearize-ricoh-z10-wb-3500.json') as f: + camera_data = json.load(f) + CORRECTION_CURVE_TYPE = 'gamma' + + well_images = [] + if args.do_linearization: + for path in Path(well_addr).rglob('*.jpg'): + well_images.append(path) + else: + for path in Path(well_addr).rglob('*.exr'): + well_images.append(path) + + for well_addr in tqdm(well_images): + + well_addr = str(well_addr) + + hdr_well_image = cv2.imread(well_addr, cv2.IMREAD_UNCHANGED) + hdr_fast_image = cv2.imread(well_addr.replace(args.well_dir, args.fast_dir), cv2.IMREAD_UNCHANGED) + + + # hdr_well_image = np.uint8(hdr_well_image * 255.0) + # hdr_fast_image = np.uint8(hdr_fast_image * 255.0) + + if args.do_linearization: + hdr_well_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_well_image), CORRECTION_CURVE_TYPE) + hdr_fast_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_fast_image), CORRECTION_CURVE_TYPE) + + + # well_mask_addr = well_addr.replace('/well_expo/', '/well_mask/') + # well_mask_addr = well_mask_addr.replace('.exr', '.png') + # well_mask = cv2.imread(well_mask_addr) + # well_mask = well_mask.astype(np.float32) / 255.0 + # well_mask[well_mask > 0.1] = 1.0 + + weights_well = cut_weighting_function(hdr_well_image, WELL_EXPOSURE) + weights_fast = cut_weighting_function(hdr_fast_image, FAST_EXPOSURE) + + # fast_mask = cv2.imread(well_mask_addr.replace('/well_mask/', '/fast_mask/')) + # fast_mask = fast_mask.astype(np.float32) / 255.0 + + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + weights_fast * (hdr_fast_image / FAST_EXPOSURE) + # final_output = final_output / (weights_well+weights_well) + final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + (hdr_fast_image / FAST_EXPOSURE) + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + + output_addr = well_addr.replace(args.well_dir, args.out_dir) + if args.do_linearization: + output_addr = output_addr.replace('jpg', '.exr') + cv2.imwrite(output_addr, np.float32(final_output) ) + # import pdb; pdb.set_trace() \ No newline at end of file diff --git a/linearize-ricoh-z10-wb-3500.json b/linearize-ricoh-z10-wb-3500.json index 6ff6171b1..112314c91 100644 --- a/linearize-ricoh-z10-wb-3500.json +++ b/linearize-ricoh-z10-wb-3500.json @@ -1 +1 @@ -{"B": [54.09578323364258, 0.2747989296913147, 0.0019784104079008102], "G": [48.520992279052734, 0.05255260691046715, 0.0029351452831178904], "R": [47.87550735473633, 0.19775021076202393, 0.0023990024346858263], "type": "deg2"} \ No newline at end of file +{"b": [2.6464749792362516], "g": [3.2232453214278767], "r": [2.9613987706773113]} \ No newline at end of file diff --git a/linearize-ricoh-z10-wb-5000.json b/linearize-ricoh-z10-wb-5000.json deleted file mode 100644 index 680d3f147..000000000 --- a/linearize-ricoh-z10-wb-5000.json +++ /dev/null @@ -1 +0,0 @@ -{"B": [46.82997512817383, 0.11160952597856522, 0.002294188365340233], "G": [41.67820739746094, 0.09581712633371353, 0.0028952741995453835], "R": [37.4744873046875, 0.22809027135372162, 0.002323441905900836], "type": "deg2"} \ No newline at end of file diff --git a/video_rendering_code/hdr_blender.py b/video_rendering_code/hdr_blender.py new file mode 100644 index 000000000..d6e0da3a7 --- /dev/null +++ b/video_rendering_code/hdr_blender.py @@ -0,0 +1,58 @@ +""" +Script to render IBL renders +Usage: blender -P hdr_blender.py -b +""" +import os +import random +from math import radians +from os import listdir +from os.path import isfile, join + +import bpy +import cycles +from mathutils import Euler, Matrix, Vector + + +import sys +argv = sys.argv +argv = argv[argv.index("--") + 1:] # get all args after "--" + +inp_dirs = [argv[0]] +out_dirs = [argv[1]] + +pairs = list(zip(inp_dirs,out_dirs)) +scn = bpy.context.scene +scn.render.engine = 'CYCLES' +scn.render.image_settings.file_format = 'OPEN_EXR' +scn.world.use_nodes = True +# Enabf_le GPU rendering. +# bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA' +bpy.context.preferences.addons['cycles'].preferences.get_devices() +for device in bpy.context.preferences.addons['cycles'].preferences.devices: + if "TITAN RX" == device.name: + device.use = True + else: + device.use = False +# select world node tree +wd = scn.world +nt = bpy.data.worlds[wd.name].node_tree + +for pair in pairs: + inp, out = pair + inp = inp + os.makedirs(out,exist_ok=True) + S = bpy.context.scene + envFiles = [f for f in listdir(inp) if isfile(join(inp, f)) and f.endswith('.exr')] + i = 0 + for env_map in envFiles: + env_file = join(inp, env_map) + bpy.ops.image.open(filepath=env_file, directory=inp, files=[ + {"name": env_map, "name": env_map}], show_multiview=False) + output_name = env_map.split('.')[0] +'.exr' + + backNode = nt.nodes['Environment Texture'] + backNode.image = bpy.data.images.load(env_file) + + bpy.context.scene.render.filepath = os.path.join(out, output_name) + print("Output",os.path.join(out, output_name)) + bpy.ops.render.render(write_still=True) diff --git a/video_rendering_code/render_video.sh b/video_rendering_code/render_video.sh new file mode 100644 index 000000000..507a8ebc9 --- /dev/null +++ b/video_rendering_code/render_video.sh @@ -0,0 +1,160 @@ + +export scene_name=my_apartment_ns +export output_dir=renders/${scene_name}/lantern-nerfacto/video_images +export root_dir=data/${scene_name} +export model_path=outputs/${scene_name}/lantern-nerfacto/2024-05-16_212421/config.yml + +# camera paths +# video ground: data/Henry_Gagnon_blue_et_rouge_ns/camera_paths/2024-05-16_015020.json +# still bench: 2024-05-16_015020_render +# pespective cam: 2024-05-16_015020_render3 + +# meeting room +# export camera_path_name=2024-05-16_201826_persp +# export camera_path_name=2024-05-16_201826_envmap + +# my_apartment +# export camera_path_name=2024-05-16_212421_persp +# export camera_path_name=2024-05-16_212421_env1 +# export camera_path_name=2024-05-16_212421_env2 +# export camera_path_name=2024-05-16_212421_env3 +# export camera_path_name=2024-05-16_212421_env4 +export camera_path_name=2024-05-16_212421_env5 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +# blender --background video_rendering_code/scene_new.blend -P video_rendering_code/hdr_blender.py -- $output_dir/${camera_path_name}/combined/ $output_dir/${camera_path_name}/render/ + + + + +######### + +export camera_path_name=2024-05-16_212421_env1 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + +export camera_path_name=2024-05-16_212421_env2 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + + + +export camera_path_name=2024-05-16_212421_env3 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + + + +export camera_path_name=2024-05-16_212421_env4 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir +export camera_path_name=2024-05-16_212421_env5 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + +#### + +# meeting room teaser + + +export scene_name=meeting_room_ns +export output_dir=renders/${scene_name}/lantern-nerfacto/video_images +export root_dir=data/${scene_name} +export model_path=outputs/${scene_name}/lantern-nerfacto/2024-05-16_201826/config.yml + +# meeting room +# export camera_path_name=2024-05-16_201826_persp +# export camera_path_name=2024-05-16_201826_persp2 + +#export camera_path_name=2024-05-16_201826_env1 +# export camera_path_name=2024-05-16_201826_env2 +# export camera_path_name=2024-05-16_201826_env3 +# export camera_path_name=2024-05-16_201826_env4 +# export camera_path_name=2024-05-16_201826_env5 +# export camera_path_name=2024-05-16_201826_env6 +# export camera_path_name=2024-05-16_201826_env7 +export camera_path_name=2024-05-16_201826_env8 + +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +# Fig. 5 - ABO objects + +export scene_name=LVSN_LAB_OpenSFM +export output_dir=renders/${scene_name}/lantern-nerfacto/video_images +export root_dir=data/${scene_name} +export model_path=outputs/${scene_name}/lantern-nerfacto/step_2/config.yml + +# export camera_path_name=persp1 +export camera_path_name=persp2 +# export camera_path_name=env1 +# export camera_path_name=env2 +# export camera_path_name=env3 + +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +######### Render videos ############ + +# LVSN lab upstairs + + +export scene_name=LVSN_LAB_OpenSFM +export model_type=hdr-nerfacto +# export model_path=outputs/${scene_name}/lantern-nerfacto/step_2/config.yml +export output_dir=renders/${scene_name}/${model_type}/video_images +export model_path=outputs/${scene_name}/${model_type}/2024-05-18_143132/config.yml +export camera_path_name=video3 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +# for Yannick +zip -rq combined_${scene_name}_${model_type}.zip $output_dir/${camera_path_name}/combined + +blender --background video_rendering_code/scene_new.blend -P video_rendering_code/hdr_blender.py -- $output_dir/${camera_path_name}/combined/ $output_dir/${camera_path_name}/render/ + +mkdir $output_dir/${camera_path_name}/render_tonemapped +python lantern_scripts/tonemap.py --data_dir $output_dir/${camera_path_name}/render/ --out_dir $output_dir/${camera_path_name}/render_tonemapped +python lantern_scripts/mp4_maker.py --input_Addr $output_dir/${camera_path_name}/render_tonemapped --out_file $output_dir/${camera_path_name}/video_render.mp4 + +## + +export scene_name=meeting_room_ns + +export model_path=outputs/${scene_name}/lantern-nerfacto/2024-05-16_201826/config.yml +