# Copyright 2022 The Plenoptix Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Compound model that integrates multiple NeRF Advancements Features: NGP Hash encoding for fast training NGP ray sampler for fast training Spatial contraction for training on large scenes Appearance conditioning TODO: Pose optimization """ from __future__ import annotations from dataclasses import dataclass, field from typing import Dict, List, Literal, Optional, Tuple, Type import nerfacc import torch from torch.nn import Parameter from torchmetrics import PeakSignalNoiseRatio from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfactory.cameras.rays import RayBundle from nerfactory.fields.compound_field import field_implementation_to_class from nerfactory.fields.modules.field_heads import FieldHeadNames from nerfactory.models.base import Model, ModelConfig from nerfactory.models.modules.ray_sampler import VolumetricSampler from nerfactory.optimizers.loss import MSELoss from nerfactory.renderers.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) from nerfactory.utils import colors, visualization from nerfactory.utils.callbacks import ( TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation, ) @dataclass class CompoundModelConfig(ModelConfig): """Compound Model Config""" _target: Type = field( default_factory=lambda: CompoundModel ) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet """target class to instantiate""" enable_collider: bool = False """Whether to create a scene collider to filter rays.""" collider_params: Optional[Dict[str, float]] = None """Instant NGP doesn't use a collider.""" field_implementation: Literal["torch", "tcnn"] = "tcnn" """one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'""" num_samples: int = 1024 """Number of max samples per ray""" grid_resolution: int = 128 """Resolution of the grid used for the field.""" cone_angle: float = 0.0 """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" near_plane: float = 0.05 """How far along ray to start sampling.""" randomize_background: bool = False """Whether to randomize the background color.""" class CompoundModel(Model): """Compound model Args: config: Compound model configuration to instantiate model """ config: CompoundModelConfig def __init__(self, config: CompoundModelConfig, **kwargs) -> None: assert config.field_implementation in field_implementation_to_class self.field = None super().__init__(config=config, **kwargs) def populate_modules(self): """Set the fields and modules.""" super().populate_modules() # torch or tiny-cuda-nn version self.field = field_implementation_to_class[self.config.field_implementation]( aabb=self.scene_bounds.aabb, num_images=self.num_train_data ) self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False) # Sampler self.sampler = VolumetricSampler( aabb=self.scene_aabb, density_fn=self.field.density_fn, grid_resolution=self.config.grid_resolution, num_samples=self.config.num_samples, ) # renderers background_color = None if self.config.randomize_background else colors.WHITE self.renderer_rgb = RGBRenderer(background_color=background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer() # losses self.rgb_loss = MSELoss() # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure self.lpips = LearnedPerceptualImagePatchSimilarity() # no colliders default def get_training_callbacks( self, training_callback_attributes: TrainingCallbackAttributes ) -> List[TrainingCallback]: return [ TrainingCallback( where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], update_every_num_iters=1, func=self.sampler.occ_field.every_n_step, ) ] def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} if self.field is None: raise ValueError("populate_fields() must be called before get_param_groups") param_groups["fields"] = list(self.field.parameters()) return param_groups def get_outputs(self, ray_bundle: RayBundle): assert self.field is not None num_rays = len(ray_bundle) with torch.no_grad(): ray_samples, packed_info, ray_indices = self.sampler( ray_bundle=ray_bundle, near_plane=self.config.near_plane, ) field_outputs = self.field.forward(ray_samples) # accumulation weights = nerfacc.volumetric_rendering_weights( packed_info=packed_info, sigmas=field_outputs[FieldHeadNames.DENSITY], frustum_starts=ray_samples.frustums.starts, frustum_ends=ray_samples.frustums.ends, ) rgb = self.renderer_rgb( rgb=field_outputs[FieldHeadNames.RGB], weights=weights, ray_indices=ray_indices, num_rays=num_rays, ) depth = self.renderer_depth( weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays ) accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays) alive_ray_mask = accumulation.squeeze(-1) > 0 outputs = { "rgb": rgb, "accumulation": accumulation, "depth": depth, "alive_ray_mask": alive_ray_mask, # the rays we kept from sampler "num_samples_per_ray": packed_info[:, 1], } return outputs def get_metrics_dict(self, outputs, batch): image = batch["image"].to(self.device) metrics_dict = {} metrics_dict["psnr"] = self.psnr(outputs["rgb"], image) metrics_dict["num_samples_per_batch"] = outputs["num_samples_per_ray"].sum() return metrics_dict def get_loss_dict(self, outputs, batch, metrics_dict=None): image = batch["image"].to(self.device) mask = outputs["alive_ray_mask"] rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) loss_dict = {"rgb_loss": rgb_loss} return loss_dict def get_image_metrics_and_images( self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: image = batch["image"].to(self.device) rgb = outputs["rgb"] acc = visualization.apply_colormap(outputs["accumulation"]) depth = visualization.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], ) alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"]) combined_rgb = torch.cat([image, rgb], dim=1) combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1) # Switch images from [H, W, C] to [1, C, H, W] for metrics computations image = torch.moveaxis(image, -1, 0)[None, ...] rgb = torch.moveaxis(rgb, -1, 0)[None, ...] psnr = self.psnr(image, rgb) ssim = self.ssim(image, rgb) lpips = self.lpips(image, rgb) # all of these metrics will be logged as scalars metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore # TODO(ethan): return an image dictionary images_dict = { "img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth, "alive_ray_mask": combined_alive_ray_mask, } return metrics_dict, images_dict