diff --git a/nerfactory/configs/base.py b/nerfactory/configs/base.py index ce6f394c2..439bfbaf9 100644 --- a/nerfactory/configs/base.py +++ b/nerfactory/configs/base.py @@ -38,7 +38,6 @@ from nerfactory.datamanagers.dataparsers.record3d_parser import Record3D # model instances from nerfactory.models.base import Model -from nerfactory.models.compound import CompoundModel from nerfactory.models.nerfw import NerfWModel from nerfactory.models.tensorf import TensoRFModel from nerfactory.optimizers.schedulers import ExponentialDecaySchedule @@ -398,22 +397,6 @@ class VanillaModelConfig(InstantiateConfig): """Vanilla Model Config""" -@dataclass -class CompoundModelConfig(ModelConfig): - """Compound Model Config""" - - _target: Type = CompoundModel - enable_density_field: bool = True - enable_collider: bool = False - field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ... - loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss": 1.0}) - num_samples: int = 1024 # instead of course/fine samples - cone_angle: float = 0.0 - """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" - near_plane: float = 0.05 - """How far along ray to start sampling.""" - - @dataclass class NerfWModelConfig(ModelConfig): """NerfW model config""" diff --git a/nerfactory/configs/base_configs.py b/nerfactory/configs/base_configs.py index 255ae504e..a430f8d12 100644 --- a/nerfactory/configs/base_configs.py +++ b/nerfactory/configs/base_configs.py @@ -24,7 +24,6 @@ from typeguard import typeguard_ignore from nerfactory.configs.base import ( BlenderDataParserConfig, - CompoundModelConfig, Config, FriendsDataManagerConfig, MipNerf360DataParserConfig, @@ -37,6 +36,7 @@ from nerfactory.configs.base import ( TrainerConfig, VanillaDataManagerConfig, ) +from nerfactory.models.compound import CompoundModelConfig from nerfactory.models.instant_ngp import InstantNGPModelConfig from nerfactory.models.mipnerf import MipNerfModel from nerfactory.models.mipnerf_360 import MipNerf360Model diff --git a/nerfactory/fields/compound_field.py b/nerfactory/fields/compound_field.py index 56ebd44c2..9cb26a8ff 100644 --- a/nerfactory/fields/compound_field.py +++ b/nerfactory/fields/compound_field.py @@ -49,24 +49,39 @@ except ImportError: pass -def get_normalized_directions(directions): - """SH encoding must be in the range [0, 1]""" +def get_normalized_directions(directions: TensorType["bs":..., 3]): + """SH encoding must be in the range [0, 1] + + Args: + directions: batch of directions + """ return (directions + 1.0) / 2.0 class TCNNCompoundField(Field): - """Compound Field that uses TCNN""" + """Compound Field that uses TCNN + + Args: + aabb: parameters of scene aabb bounds + num_layers: number of hidden layers + hidden_dim: dimension of hidden layers + geo_feat_dim: output geo feat dimensions + num_layers_color: number of hidden layers for color network + hidden_dim_color: dimension of hidden layers for color network + appearance_embedding_dim: dimension of appearance embedding + spatial_distortion: spatial distortion to apply to the scene + """ def __init__( self, aabb, num_images: int, - num_layers=2, - hidden_dim=64, - geo_feat_dim=15, - num_layers_color=3, - hidden_dim_color=64, - appearance_embedding_dim: int = 40, + num_layers: int = 2, + hidden_dim: int = 64, + geo_feat_dim: int = 15, + num_layers_color: int = 3, + hidden_dim_color: int = 64, + appearance_embedding_dim: int = 32, spatial_distortion: SpatialDistortion = SceneContraction(), ) -> None: super().__init__() diff --git a/nerfactory/models/compound.py b/nerfactory/models/compound.py index 8740528f6..25f632cd3 100644 --- a/nerfactory/models/compound.py +++ b/nerfactory/models/compound.py @@ -28,23 +28,29 @@ TODO: from __future__ import annotations -from typing import Dict, List, Tuple +from dataclasses import dataclass, field +from typing import Dict, List, Literal, Optional, Tuple, Type +import nerfacc import torch from torch.nn import Parameter from torchmetrics import PeakSignalNoiseRatio from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity -import nerfactory.cuda as nerfactory_cuda from nerfactory.cameras.rays import RayBundle from nerfactory.configs import base as cfg from nerfactory.fields.compound_field import field_implementation_to_class from nerfactory.fields.modules.field_heads import FieldHeadNames from nerfactory.models.base import Model -from nerfactory.models.modules.ray_sampler import NGPSpacedSampler +from nerfactory.models.modules.ray_sampler import VolumetricSampler from nerfactory.optimizers.loss import MSELoss -from nerfactory.utils import colors, misc, visualization +from nerfactory.renderers.renderers import ( + AccumulationRenderer, + DepthRenderer, + RGBRenderer, +) +from nerfactory.utils import colors, visualization from nerfactory.utils.callbacks import ( TrainingCallback, TrainingCallbackAttributes, @@ -52,44 +58,69 @@ from nerfactory.utils.callbacks import ( ) +@dataclass +class CompoundModelConfig(cfg.ModelConfig): + """Compound Model Config""" + + _target: Type = field( + default_factory=lambda: CompoundModel + ) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet + """target class to instantiate""" + enable_collider: bool = False + """Whether to create a scene collider to filter rays.""" + collider_params: Optional[Dict[str, float]] = None + """Instant NGP doesn't use a collider.""" + field_implementation: Literal["torch", "tcnn"] = "tcnn" + """one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'""" + num_samples: int = 1024 + """Number of max samples per ray""" + grid_resolution: int = 128 + """Resolution of the grid used for the field.""" + cone_angle: float = 0.0 + """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" + near_plane: float = 0.05 + """How far along ray to start sampling.""" + randomize_background: bool = False + """Whether to randomize the background color.""" + + class CompoundModel(Model): """Compound model Args: - field_implementation (str): one of "torch" or "tcnn", or other fields in 'field_implementation_to_class' - kwargs: additional params to pass up to the parent class model + config: Compound model configuration to instantiate model """ - config: cfg.CompoundModelConfig + config: CompoundModelConfig - def __init__(self, config: cfg.CompoundModelConfig, **kwargs) -> None: + def __init__(self, config: CompoundModelConfig, **kwargs) -> None: assert config.field_implementation in field_implementation_to_class self.field = None super().__init__(config=config, **kwargs) - def get_training_callbacks( - self, training_callback_attributes: TrainingCallbackAttributes - ) -> List[TrainingCallback]: - assert self.density_field is not None - return [ - TrainingCallback( - where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], - update_every_num_iters=self.density_field.update_every_num_iters, - func=self.density_field.update_density_grid, - kwargs={"density_eval_func": self.field.density_fn}, # type: ignore - ) - ] - def populate_modules(self): """Set the fields and modules.""" super().populate_modules() # torch or tiny-cuda-nn version self.field = field_implementation_to_class[self.config.field_implementation]( - self.scene_bounds.aabb, self.num_train_data + aabb=self.scene_bounds.aabb, num_images=self.num_train_data ) - # samplers - self.sampler = NGPSpacedSampler(num_samples=self.config.num_samples, density_field=self.density_field) + self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False) + + # Sampler + self.sampler = VolumetricSampler( + aabb=self.scene_aabb, + density_fn=self.field.density_fn, + grid_resolution=self.config.grid_resolution, + num_samples=self.config.num_samples, + ) + + # renderers + background_color = None if self.config.randomize_background else colors.WHITE + self.renderer_rgb = RGBRenderer(background_color=background_color) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() # losses self.rgb_loss = MSELoss() @@ -101,6 +132,17 @@ class CompoundModel(Model): # no colliders default + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + return [ + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.sampler.occ_field.every_n_step, + ) + ] + def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} if self.field is None: @@ -109,75 +151,69 @@ class CompoundModel(Model): return param_groups def get_outputs(self, ray_bundle: RayBundle): - # TODO(ruilongli) - # - train test difference - # - visualize "depth_density_grid" + assert self.field is not None num_rays = len(ray_bundle) - device = ray_bundle.origins.device - if self.field is None: - raise ValueError("populate_fields() must be called before get_outputs") - ray_samples, packed_info, t_min, t_max = self.sampler( - ray_bundle, self.field.aabb, cone_angle=self.config.cone_angle, near_plane=self.config.near_plane - ) + with torch.no_grad(): + ray_samples, packed_info, ray_indices = self.sampler( + ray_bundle=ray_bundle, + near_plane=self.config.near_plane, + ) field_outputs = self.field.forward(ray_samples) - rgbs = field_outputs[FieldHeadNames.RGB] - sigmas = field_outputs[FieldHeadNames.DENSITY] - - # accumulate all the rays start from zero opacity - opacities = torch.zeros((num_rays, 1), device=device) - ( - accumulated_weight, - accumulated_depth, - accumulated_color, - alive_ray_mask, - ) = nerfactory_cuda.VolumeRenderer.apply( - packed_info, - ray_samples.frustums.starts, - ray_samples.frustums.ends, - sigmas.contiguous(), - rgbs.contiguous(), - opacities, + # accumulation + weights = nerfacc.volumetric_rendering_weights( + packed_info=packed_info, + sigmas=field_outputs[FieldHeadNames.DENSITY], + frustum_starts=ray_samples.frustums.starts, + frustum_ends=ray_samples.frustums.ends, ) - accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None]) - accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight) + + rgb = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.RGB], + weights=weights, + ray_indices=ray_indices, + num_rays=num_rays, + ) + depth = self.renderer_depth( + weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays + ) + accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays) + alive_ray_mask = accumulation.squeeze(-1) > 0 outputs = { - "rgb": accumulated_color, - "accumulation": accumulated_weight, - "depth": accumulated_depth, + "rgb": rgb, + "accumulation": accumulation, + "depth": depth, "alive_ray_mask": alive_ray_mask, # the rays we kept from sampler } return outputs def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients): - device = self.device - image = batch["image"].to(device) - if "alive_ray_mask" in outputs: - mask = outputs["alive_ray_mask"] - rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) - else: - rgb_loss = self.rgb_loss(image, outputs["rgb"]) + image = batch["image"].to(self.device) + mask = outputs["alive_ray_mask"] + rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) loss_dict = {"rgb_loss": rgb_loss} - loss_dict = misc.scale_dict(loss_dict, loss_coefficients) return loss_dict def get_image_metrics_and_images( self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: - device = self.device - image = batch["image"].to(device) + + image = batch["image"].to(self.device) rgb = outputs["rgb"] acc = visualization.apply_colormap(outputs["accumulation"]) depth = visualization.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], ) + alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"]) + combined_rgb = torch.cat([image, rgb], dim=1) combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) + combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1) # Switch images from [H, W, C] to [1, C, H, W] for metrics computations image = torch.moveaxis(image, -1, 0)[None, ...] @@ -187,10 +223,15 @@ class CompoundModel(Model): ssim = self.ssim(image, rgb) lpips = self.lpips(image, rgb) - metrics_dict = { - "psnr": float(psnr.item()), - "ssim": float(ssim.item()), - "lpips": float(lpips.item()), + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore + # TODO(ethan): return an image dictionary + + images_dict = { + "img": combined_rgb, + "accumulation": combined_acc, + "depth": combined_depth, + "alive_ray_mask": combined_alive_ray_mask, } - images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + return metrics_dict, images_dict diff --git a/nerfactory/models/modules/ray_sampler.py b/nerfactory/models/modules/ray_sampler.py index 300cc8b29..7c3a772fb 100644 --- a/nerfactory/models/modules/ray_sampler.py +++ b/nerfactory/models/modules/ray_sampler.py @@ -25,7 +25,6 @@ import torch from torch import nn from torchtyping import TensorType -import nerfactory.cuda as nerfactory_cuda from nerfactory.cameras.rays import Frustums, RayBundle, RaySamples from nerfactory.fields.density_fields.density_grid import DensityGrid @@ -344,108 +343,6 @@ class PDFSampler(Sampler): return ray_samples -class NGPSpacedSampler(Sampler): - """Sampler that matches Instant-NGP paper. - - This sampler does ray-box AABB test, ray samples generation and density check, all together. - - TODO(ruilongli): check whether fuse AABB test together with `raymarching` can speed things up. - Otherwise we can seperate it out and use collision detecoter that already exists in the repo. - """ - - def generate_ray_samples(self) -> RaySamples: - raise RuntimeError("For NGP we fused ray samples and density check together. Please call forward() directly.") - - # pylint: disable=arguments-differ - def forward( - self, - ray_bundle: RayBundle, - aabb: TensorType[2, 3], - num_samples: Optional[int] = None, - cone_angle: float = 0.0, - marching_steps: Optional[int] = 128, - near_plane: float = 0.0, - ) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]: - """Generate ray samples in a bounding box. - - TODO(ruilongli): write a Packed[Ray_samples] class to ray_samples with packed_info. - TODO(ruilongli): maybe move aabb test to the collision detector? - - Args: - ray_bundle: Rays to generate samples for - aabb: Bounding box of the scene. - num_samples: Number of samples per ray - cone_angle: 0.0 for nerf-synthetic and 1.0/256 for real scenes - marching_steps: Number of marching steps (Note: not currently used) - near_plane: Near plane for raymarching - - Returns: - First return is ray samples in a packed way where only the valid samples are kept. - Second return contains all the information to recover packed samples into unpacked mode for rendering. - The last two returns are t_min and t_max from ray-aabb test. - """ - - if self.density_field is None: - raise ValueError("density_field must be set to use NGPSpacedSampler") - - num_samples = num_samples or self.num_samples - - aabb = aabb.flatten().contiguous() - rays_o = ray_bundle.origins.contiguous() - rays_d = ray_bundle.directions.contiguous() - scene_scale = (aabb[3:6] - aabb[0:3]).max() - - t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb) - t_min = torch.max(t_min, torch.ones_like(t_min) * near_plane) - - marching_steps = -1 # disable marching mode for now - # TODO(ruilongli): * 16 is for original impl for training. Need to run - # some profiling test with this choice. - max_samples_per_batch = len(rays_o) * num_samples - - # marching - packed_info, origins, dirs, starts, ends = nerfactory_cuda.raymarching( - # rays - rays_o, - rays_d, - t_min, - t_max, - # density grid - self.density_field.center, - self.density_field.base_scale, - self.density_field.num_cascades, - self.density_field.resolution, - self.density_field.density_bitfield, - # sampling args - marching_steps, - max_samples_per_batch, - num_samples, - cone_angle, - scene_scale, - ) - - # squeeze valid samples - total_samples = max(packed_info[:, -1].sum(), 1) - origins = origins[:total_samples] - dirs = dirs[:total_samples] - starts = starts[:total_samples] - ends = ends[:total_samples] - - assert ray_bundle.camera_indices is not None - indices = torch.zeros_like(origins[:, :1], dtype=torch.long) - - ray_bundle.camera_indices = ray_bundle.camera_indices.to(torch.long).to(packed_info.device) - - indices = nerfactory_cuda.get_camera_indices(packed_info, ray_bundle.camera_indices, indices) - - zeros = torch.zeros_like(origins[:, :1]) - ray_samples = RaySamples( - frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros), - camera_indices=indices, - ) - return ray_samples, packed_info, t_min, t_max - - class VolumetricSampler(Sampler): """Sampler inspired by the one proposed in the Instant-NGP paper.