From 631b3c6dae6a882319e03ee26b0a158494bedd26 Mon Sep 17 00:00:00 2001 From: MrKarimiD Date: Fri, 10 May 2024 15:59:39 -0400 Subject: [PATCH] PanoHDR-Nerfacto model --- nerfstudio/configs/method_configs.py | 6 +- .../data/dataparsers/nerfstudio_dataparser.py | 7 +- nerfstudio/lantern/config.py | 37 +++ nerfstudio/lantern/field_pano_hdr_nerfacto.py | 103 ++++++ nerfstudio/lantern/model.py | 8 +- nerfstudio/lantern/model_pano_hdr_nerfacto.py | 294 ++++++++++++++++++ 6 files changed, 447 insertions(+), 8 deletions(-) create mode 100644 nerfstudio/lantern/field_pano_hdr_nerfacto.py create mode 100644 nerfstudio/lantern/model_pano_hdr_nerfacto.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 50357be3e..940c6037b 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -66,13 +66,14 @@ from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig from nerfstudio.plugins.registry import discover_methods -from nerfstudio.lantern.config import Get_lantern_config +from nerfstudio.lantern.config import Get_lantern_config, Get_PanoHDR_Nerfacto_config method_configs: Dict[str, TrainerConfig] = {} descriptions = { "nerfacto": "Recommended real-time model tuned for real captures. This model will be continually updated.", "nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.", - "lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", + "lantern-nerfacto": "Nerfacto that train on two exposures.", + "panoHDR-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", "depth-nerfacto": "Nerfacto with depth supervision.", "volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", @@ -122,6 +123,7 @@ method_configs["nerfacto"] = TrainerConfig( ) method_configs["lantern-nerfacto"] = Get_lantern_config() +method_configs["panoHDR-nerfacto"] = Get_PanoHDR_Nerfacto_config() method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 5137e74af..3c7f2f1f7 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -294,8 +294,11 @@ class Nerfstudio(DataParser): else: distortion_params = torch.stack(distort, dim=0)[idx_tensor] - exposure_tensor = torch.tensor([exposures], dtype=torch.float32).T - + if len(exposures) != 0: + exposure_tensor = torch.tensor([exposures], dtype=torch.float32).T + else: + exposure_tensor = [] + cameras = Cameras( fx=fx, fy=fy, diff --git a/nerfstudio/lantern/config.py b/nerfstudio/lantern/config.py index 4e5f72c9c..14e384fbc 100644 --- a/nerfstudio/lantern/config.py +++ b/nerfstudio/lantern/config.py @@ -14,6 +14,7 @@ from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.lantern.datamanager import HDRVanillaDataManagerConfig from nerfstudio.lantern.model import LanternModelConfig +from nerfstudio.lantern.model_pano_hdr_nerfacto import PanoHDRNeRFModelConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig @@ -50,4 +51,40 @@ def Get_lantern_config(): }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", + ) + + +def Get_PanoHDR_Nerfacto_config(): + return TrainerConfig( + method_name="panoHDR-nerfacto", + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + datamanager=HDRVanillaDataManagerConfig( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=8192, + eval_num_rays_per_batch=8192, + # camera_optimizer=CameraOptimizerConfig( + # mode="SO3xR3", + # optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), + # scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + # ), + camera_optimizer=None, + ), + model=PanoHDRNeRFModelConfig(eval_num_rays_per_chunk=1 << 15), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", ) \ No newline at end of file diff --git a/nerfstudio/lantern/field_pano_hdr_nerfacto.py b/nerfstudio/lantern/field_pano_hdr_nerfacto.py new file mode 100644 index 000000000..c7e3a636e --- /dev/null +++ b/nerfstudio/lantern/field_pano_hdr_nerfacto.py @@ -0,0 +1,103 @@ +""" +HDR Nerfstudio Field + +Currently this subclasses the NerfactoField. Consider subclassing the base Field. +""" + +from typing import Dict, Literal, Optional, Tuple + +import torch +from torch import Tensor, nn + +from nerfstudio.cameras.rays import RaySamples +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.field_components.activations import trunc_exp +from nerfstudio.field_components.embedding import Embedding +from nerfstudio.field_components.encodings import HashEncoding, NeRFEncoding, SHEncoding +from nerfstudio.field_components.encodings import Encoding, Identity +from nerfstudio.field_components.field_heads import ( + FieldHeadNames, + PredNormalsFieldHead, + SemanticFieldHead, + TransientDensityFieldHead, + TransientRGBFieldHead, + UncertaintyFieldHead, +) +from nerfstudio.field_components.mlp import MLP +from nerfstudio.field_components.spatial_distortions import SpatialDistortion +from nerfstudio.fields.base_field import Field, get_normalized_directions +from nerfstudio.fields.nerfacto_field import \ + NerfactoField # for subclassing NerfactoField + + +class PanoHDR_NerfactoField(NerfactoField): + """Template Field + + Args: + aabb: parameters of scene aabb bounds + num_images: number of images in the dataset + """ + + aabb: Tensor + + def __init__( + self, + aabb: Tensor, + num_images: int, + num_layers: int = 2, + hidden_dim: int = 64, + geo_feat_dim: int = 15, + num_levels: int = 16, + base_res: int = 16, + max_res: int = 2048, + log2_hashmap_size: int = 19, + num_layers_color: int = 3, + num_layers_transient: int = 2, + features_per_level: int = 2, + hidden_dim_color: int = 64, + hidden_dim_transient: int = 64, + appearance_embedding_dim: int = 32, + transient_embedding_dim: int = 16, + use_transient_embedding: bool = True, + use_semantics: bool = False, + num_semantic_classes: int = 100, + pass_semantic_gradients: bool = False, + use_pred_normals: bool = False, + use_average_appearance_embedding: bool = False, + spatial_distortion: Optional[SpatialDistortion] = None, + implementation: Literal["tcnn", "torch"] = "tcnn", + use_appearance_embedding: bool = True, + ) -> None: + super().__init__(aabb, num_images, num_layers, hidden_dim, + geo_feat_dim, num_levels, base_res, + max_res, log2_hashmap_size, num_layers_color, + num_layers_transient, features_per_level, hidden_dim_color, + hidden_dim_transient, appearance_embedding_dim, transient_embedding_dim, + use_transient_embedding, use_semantics, num_semantic_classes, + pass_semantic_gradients, use_pred_normals, use_average_appearance_embedding, + spatial_distortion, implementation, use_appearance_embedding) + + if self.use_appearance_embedding: + self.mlp_head = MLP( + in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim, + num_layers=num_layers_color, + layer_width=hidden_dim_color, + out_dim= 3, + activation=nn.ReLU(), + out_activation=nn.ReLU(), + # out_activation=nn.Sigmoid(), + implementation=implementation, + ) + + # Define the last part MLP for RGB outputs and Masks + else: + self.mlp_head = MLP( + in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim, + num_layers=num_layers_color, + layer_width=hidden_dim_color, + out_dim= 3, + activation=nn.ReLU(), + out_activation=nn.ReLU(), + # out_activation=nn.Sigmoid(), + implementation=implementation, + ) \ No newline at end of file diff --git a/nerfstudio/lantern/model.py b/nerfstudio/lantern/model.py index 1fe6e82e1..516ef3ee8 100644 --- a/nerfstudio/lantern/model.py +++ b/nerfstudio/lantern/model.py @@ -188,10 +188,10 @@ class LanternModel(NerfactoModel): validity = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.VALIDITY], weights=weights) rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) - if self.config.apply_mu_law: - u = 5000. - rgb_fast = torch.exp(rgb_fast * torch.log(torch.tensor(u+1.))) - 1. - rgb_fast /= u + # if self.config.apply_mu_law: + # u = 5000. + # rgb_fast = torch.exp(rgb_fast * torch.log(torch.tensor(u+1.))) - 1. + # rgb_fast /= u outputs = { "rgb": rgb, diff --git a/nerfstudio/lantern/model_pano_hdr_nerfacto.py b/nerfstudio/lantern/model_pano_hdr_nerfacto.py new file mode 100644 index 000000000..438554aa6 --- /dev/null +++ b/nerfstudio/lantern/model_pano_hdr_nerfacto.py @@ -0,0 +1,294 @@ +""" +Lantern Model File + +Currently this subclasses the Nerfacto model. Consider subclassing from the base Model. +""" +from dataclasses import dataclass, field +from typing import Dict, Tuple, Type + +import numpy as np +import torch +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image import PeakSignalNoiseRatio +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity + +from nerfstudio.cameras.rays import RayBundle, RaySamples +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.lantern.field_pano_hdr_nerfacto import PanoHDR_NerfactoField +from nerfstudio.lantern.renderer import RGBRenderer_HDR +from nerfstudio.model_components.losses import ( + MSELoss, + interlevel_loss, + orientation_loss, + pred_normal_loss, + scale_gradients_by_distance_squared, +) +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + SemanticRenderer, +) +from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader +from nerfstudio.models.nerfacto import ( # for subclassing Nerfacto model + NerfactoModel, + NerfactoModelConfig, +) +from nerfstudio.utils import colormaps + + +@dataclass +class PanoHDRNeRFModelConfig(NerfactoModelConfig): + """Template Model Configuration. + + Add your custom model config parameters here. + """ + + _target: Type = field(default_factory=lambda: PanoHDRNeRFModel) + + +class PanoHDRNeRFModel(NerfactoModel): + """Template Model.""" + + config: PanoHDRNeRFModelConfig + + def populate_modules(self): + super().populate_modules() + + if self.config.disable_scene_contraction: + scene_contraction = None + else: + scene_contraction = SceneContraction(order=float("inf")) + + # Fields + self.field = PanoHDR_NerfactoField( + self.scene_box.aabb, + hidden_dim=self.config.hidden_dim, + num_levels=self.config.num_levels, + max_res=self.config.max_res, + base_res=self.config.base_res, + features_per_level=self.config.features_per_level, + log2_hashmap_size=self.config.log2_hashmap_size, + hidden_dim_color=self.config.hidden_dim_color, + hidden_dim_transient=self.config.hidden_dim_transient, + spatial_distortion=scene_contraction, + num_images=self.num_train_data, + use_pred_normals=self.config.predict_normals, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, + appearance_embedding_dim=self.config.appearance_embed_dim, + implementation=self.config.implementation, + ) + + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + if self.config.use_same_proposal_network: + assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed." + prop_net_args = self.config.proposal_net_args_list[0] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)]) + else: + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + # Samplers + def update_schedule(step): + return np.clip( + np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]), + 1, + self.config.proposal_update_every, + ) + + # Change proposal network initial sampler if uniform + initial_sampler = None # None is for piecewise as default (see ProposalNetworkSampler) + if self.config.proposal_initial_sampler == "uniform": + initial_sampler = UniformSampler(single_jitter=self.config.use_single_jitter) + + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + initial_sampler=initial_sampler, + ) + + # Collider + self.collider = NearFarCollider(near_plane=self.config.near_plane, far_plane=self.config.far_plane) + + # renderers + self.renderer_rgb = RGBRenderer_HDR(background_color=self.config.background_color) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_normals = NormalsRenderer() + self.renderer_validity = SemanticRenderer() + + # shaders + self.normals_shader = NormalsShader() + + # TODO losses + self.rgb_loss = MSELoss() + + # metrics, psnr + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + + + def get_outputs(self, ray_bundle: RayBundle): + ray_samples: RaySamples + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + + field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals) + if self.config.use_gradient_scaling: + field_outputs = scale_gradients_by_distance_squared(field_outputs, ray_samples) + + weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) + weights_list.append(weights) + ray_samples_list.append(ray_samples) + + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + accumulation = self.renderer_accumulation(weights=weights) + + outputs = { + "rgb": rgb, + "accumulation": accumulation, + "depth": depth, + } + + if self.config.predict_normals: + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + outputs["normals"] = self.normals_shader(normals) + outputs["pred_normals"] = self.normals_shader(pred_normals) + # These use a lot of GPU memory, so we avoid storing them for eval. + if self.training: + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + if self.training and self.config.predict_normals: + outputs["rendered_orientation_loss"] = orientation_loss( + weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions + ) + + outputs["rendered_pred_normal_loss"] = pred_normal_loss( + weights.detach(), + field_outputs[FieldHeadNames.NORMALS].detach(), + field_outputs[FieldHeadNames.PRED_NORMALS], + ) + + for i in range(self.config.num_proposal_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + + return outputs + + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + + gt_rgb = batch["image"].to(self.device) + predicted_rgb = outputs["rgb"] # Blended with background (black if random background) + + gt_rgb = self.renderer_rgb.blend_background(gt_rgb) + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + gt_rgb = batch["mask"] * gt_rgb + predicted_rgb = batch["mask"] * predicted_rgb + + combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + combined_acc = torch.cat([acc], dim=1) + combined_depth = torch.cat([depth], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + # gt_rgb_tonemapped = torch.clamp(colormaps.apply_colormap(gt_rgb), 0, 1) + gt_rgb_tonemapped = torch.clamp(gt_rgb, 0, 1) + # predicted_rgb_tonemapped = torch.clamp(colormaps.apply_colormap(predicted_rgb), 0, 1) + predicted_rgb_tonemapped = torch.clamp(predicted_rgb, 0, 1) + predicted_rgb_tonemapped = torch.nan_to_num(predicted_rgb_tonemapped, nan=0.0) + gt_rgb_tonemapped = torch.moveaxis(gt_rgb_tonemapped, -1, 0)[None, ...] + predicted_rgb_tonemapped = torch.moveaxis(predicted_rgb_tonemapped, -1, 0)[None, ...] + gt_rgb = torch.moveaxis(gt_rgb, -1, 0)[None, ...] + predicted_rgb = torch.moveaxis(predicted_rgb, -1, 0)[None, ...] + + psnr = self.psnr(gt_rgb, predicted_rgb) + ssim = self.ssim(gt_rgb, predicted_rgb) + # tonemapping before computing lpips + lpips = self.lpips(gt_rgb_tonemapped, predicted_rgb_tonemapped) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + + for i in range(self.config.num_proposal_iterations): + key = f"prop_depth_{i}" + prop_depth_i = colormaps.apply_depth_colormap( + outputs[key], + accumulation=outputs["accumulation"], + ) + images_dict[key] = prop_depth_i + + return metrics_dict, images_dict + + + def get_loss_dict(self, outputs, batch, metrics_dict=None): + loss_dict = {} + weights_for_RGB_loss = None + + image = batch["image"].to(self.device) + + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) + + gt_rgb = gt_rgb ** (1 / 2.2) + loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + + if self.training: + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + assert metrics_dict is not None and "distortion" in metrics_dict + loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] + # loss_dict["fg"] = (torch.pow(1 - torch.sum( outputs["weights_list"][-1], dim=0), 2)).mean() + if self.config.predict_normals: + # orientation loss for computed normals + loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( + outputs["rendered_orientation_loss"] + ) + + # ground truth supervision for normals + loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( + outputs["rendered_pred_normal_loss"] + ) + return loss_dict \ No newline at end of file