diff --git a/nerfactory/configs/base_configs.py b/nerfactory/configs/base_configs.py index a5a5ba9c4..824b827ac 100644 --- a/nerfactory/configs/base_configs.py +++ b/nerfactory/configs/base_configs.py @@ -37,7 +37,6 @@ from nerfactory.datamanagers.dataparsers.mipnerf_parser import ( MipNerf360DataParserConfig, ) from nerfactory.models.base import VanillaModelConfig -from nerfactory.models.compound import CompoundModelConfig from nerfactory.models.instant_ngp import InstantNGPModelConfig from nerfactory.models.mipnerf import MipNerfModel from nerfactory.models.mipnerf_360 import MipNerf360Model @@ -50,37 +49,6 @@ from nerfactory.pipelines.base import VanillaPipelineConfig from nerfactory.pipelines.dynamic_batch import DynamicBatchPipelineConfig base_configs: Dict[str, Config] = {} -base_configs["compound"] = Config( - method_name="compound", - trainer=TrainerConfig(mixed_precision=True), - pipeline=DynamicBatchPipelineConfig( - datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), - model=CompoundModelConfig(eval_num_rays_per_chunk=8192), - ), - optimizers={ - "fields": { - "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), - "scheduler": None, - } - }, - vis=["viewer"], -) - -base_configs["instant-ngp"] = Config( - method_name="instant-ngp", - trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True), - pipeline=DynamicBatchPipelineConfig( - datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), - model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192), - ), - optimizers={ - "fields": { - "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), - "scheduler": None, - } - }, - vis=["viewer"], -) base_configs["proposal"] = Config( method_name="proposal", @@ -105,6 +73,23 @@ base_configs["proposal"] = Config( vis=["viewer"], ) +base_configs["instant-ngp"] = Config( + method_name="instant-ngp", + trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True), + pipeline=DynamicBatchPipelineConfig( + datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), + model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192), + ), + optimizers={ + "fields": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": None, + } + }, + viewer=ViewerConfig(num_rays_per_chunk=64000), + vis=["viewer"], +) + base_configs["mipnerf-360"] = Config( method_name="mipnerf-360", trainer=TrainerConfig(steps_per_eval_batch=200), diff --git a/nerfactory/datamanagers/dataparsers/nerfactory_parser.py b/nerfactory/datamanagers/dataparsers/nerfactory_parser.py index 8c35c7093..1272396da 100644 --- a/nerfactory/datamanagers/dataparsers/nerfactory_parser.py +++ b/nerfactory/datamanagers/dataparsers/nerfactory_parser.py @@ -43,8 +43,8 @@ class NerfactoryDataParserConfig(DataParserConfig): """How much to scale the camera origins by.""" downscale_factor: int = 1 """How much to downscale images.""" - scene_scale: float = 4.0 - """How much to scale the scene.""" + scene_scale: float = 1.0 + """How much to scale the region of interest by.""" orientation_method: Literal["pca", "up"] = "up" """The method to use for orientation.""" train_split_percentage: float = 0.9 diff --git a/nerfactory/fields/instant_ngp_field.py b/nerfactory/fields/instant_ngp_field.py index e0782f256..057264c5e 100644 --- a/nerfactory/fields/instant_ngp_field.py +++ b/nerfactory/fields/instant_ngp_field.py @@ -17,19 +17,18 @@ Instant-NGP field implementations using tiny-cuda-nn, torch, .... """ -from typing import Optional, Tuple +from typing import Optional import torch +from nerfacc import ContractionType, contract from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfactory.cameras.rays import RaySamples from nerfactory.datamanagers.structs import SceneBounds from nerfactory.fields.base import Field -from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding +from nerfactory.fields.modules.embedding import Embedding from nerfactory.fields.modules.field_heads import FieldHeadNames -from nerfactory.fields.modules.spatial_distortions import SpatialDistortion -from nerfactory.fields.nerf_field import NeRFField from nerfactory.utils.activations import trunc_exp try: @@ -58,6 +57,10 @@ class TCNNInstantNGPField(Field): geo_feat_dim: output geo feat dimensions num_layers_color: number of hidden layers for color network hidden_dim_color: dimension of hidden layers for color network + use_appearance_embedding: whether to use appearance embedding + num_images: number of images, requried if use_appearance_embedding is True + appearance_embedding_dim: dimension of appearance embedding + contraction_type: type of contraction """ def __init__( @@ -68,13 +71,22 @@ class TCNNInstantNGPField(Field): geo_feat_dim: int = 15, num_layers_color: int = 3, hidden_dim_color: int = 64, - spatial_distortion: Optional[SpatialDistortion] = None, + use_appearance_embedding: bool = False, + num_images: Optional[int] = None, + appearance_embedding_dim: int = 32, + contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE, ) -> None: super().__init__() self.aabb = Parameter(aabb, requires_grad=False) self.geo_feat_dim = geo_feat_dim - self.spatial_distortion = spatial_distortion + self.contraction_type = contraction_type + + self.use_appearance_embedding = use_appearance_embedding + if use_appearance_embedding: + assert num_images is not None + self.appearance_embedding_dim = appearance_embedding_dim + self.appearance_embedding = Embedding(num_images, appearance_embedding_dim) # TODO: set this properly based on the aabb per_level_scale = 1.4472692012786865 @@ -107,8 +119,11 @@ class TCNNInstantNGPField(Field): }, ) + in_dim = self.direction_encoding.n_output_dims + self.geo_feat_dim + if self.use_appearance_embedding: + in_dim += self.appearance_embedding_dim self.mlp_head = tcnn.Network( - n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim, + n_input_dims=in_dim, n_output_dims=3, network_config={ "otype": "FullyFusedMLP", @@ -120,18 +135,10 @@ class TCNNInstantNGPField(Field): ) def get_density(self, ray_samples: RaySamples): - if self.spatial_distortion is not None: - positions = ray_samples.frustums.get_positions() - positions = self.spatial_distortion(positions) - positions = (positions + 2.0) / 4.0 - else: - positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + positions = ray_samples.frustums.get_positions() positions_flat = positions.view(-1, 3) - # assert all positions are in the range [0, 1] - # otherwise print min and max values - # assert torch.all(positions >= 0.0) and torch.all( - # positions <= 1.0 - # ), f"positions: {positions.min()} {positions.max()}" + positions_flat = contract(x=positions_flat, roi=self.aabb, type=self.contraction_type) + h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) @@ -142,49 +149,53 @@ class TCNNInstantNGPField(Field): return density, base_mlp_out def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None): - # TODO: add valid_mask masking! directions = get_normalized_directions(ray_samples.frustums.directions) directions_flat = directions.view(-1, 3) - # assert all directions are in the range [0, 1] - # assert torch.all(directions >= 0.0) and torch.all( - # directions <= 1.0 - # ), f"directions: {directions.min()} {directions.max()}" + d = self.direction_encoding(directions_flat) if density_embedding is None: positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) h = torch.cat([d, positions.view(-1, 3)], dim=-1) else: h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) + + if self.use_appearance_embedding: + if ray_samples.camera_indices is None: + raise AttributeError("Camera indices are not provided.") + camera_indices = ray_samples.camera_indices.squeeze() + if self.training: + embedded_appearance = self.appearance_embedding(camera_indices) + else: + embedded_appearance = torch.zeros( + (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + ) + h = torch.cat([h, embedded_appearance.view(-1, self.appearance_embedding_dim)], dim=-1) + rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions) return {FieldHeadNames.RGB: rgb} + def get_opacity(self, positions: TensorType["bs":..., 3], step_size) -> TensorType["bs":..., 1]: + """Returns the opacity for a position. Used primarily by the occupancy grid. -class TorchInstantNGPField(NeRFField): - """ - PyTorch implementation of the Instant-NGP field. - """ + Args: + positions: the positions to evaluate the opacity at. + step_size: the step size to use for the opacity evaluation. + """ + density = self.density_fn(positions) + ## TODO: We should scale step size based on the distortion. Currently it uses too much memory. + # aabb_min, aabb_max = self.aabb[0], self.aabb[1] + # if self.contraction_type is not ContractionType.AABB: + # x = (positions - aabb_min) / (aabb_max - aabb_min) + # x = x * 2 - 1 # aabb is at [-1, 1] + # mag = x.norm(dim=-1, keepdim=True) + # mask = mag.squeeze(-1) > 1 - def __init__( - self, - aabb, - position_encoding: Encoding = HashEncoding(), - direction_encoding: Encoding = SHEncoding(), - base_mlp_num_layers: int = 3, - base_mlp_layer_width: int = 64, - head_mlp_num_layers: int = 2, - head_mlp_layer_width: int = 32, - skip_connections: Tuple = (4,), - ) -> None: - super().__init__( - position_encoding, - direction_encoding, - base_mlp_num_layers, - base_mlp_layer_width, - head_mlp_num_layers, - head_mlp_layer_width, - skip_connections, - ) - self.aabb = Parameter(aabb, requires_grad=False) + # dev = (2 * mag - 1) / mag**2 + 2 * x**2 * (1 / mag**3 - (2 * mag - 1) / mag**4) + # dev[~mask] = 1.0 + # dev = torch.clamp(dev, min=1e-6) + # step_size = step_size / dev.norm(dim=-1, keepdim=True) + # else: + # step_size = step_size * (aabb_max - aabb_min) - -field_implementation_to_class = {"tcnn": TCNNInstantNGPField, "torch": TorchInstantNGPField} + opacity = density * step_size + return opacity diff --git a/nerfactory/models/compound.py b/nerfactory/models/compound.py deleted file mode 100644 index 398d361b6..000000000 --- a/nerfactory/models/compound.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Compound model that integrates multiple NeRF Advancements - -Features: - - NGP Hash encoding for fast training - NGP ray sampler for fast training - Spatial contraction for training on large scenes - Appearance conditioning - -TODO: - Pose optimization - -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Dict, List, Literal, Optional, Tuple, Type - -import nerfacc -import torch -from torch.nn import Parameter -from torchmetrics import PeakSignalNoiseRatio -from torchmetrics.functional import structural_similarity_index_measure -from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity - -from nerfactory.cameras.rays import RayBundle -from nerfactory.fields.compound_field import field_implementation_to_class -from nerfactory.fields.modules.field_heads import FieldHeadNames -from nerfactory.models.base import Model, ModelConfig -from nerfactory.models.modules.ray_sampler import VolumetricSampler -from nerfactory.optimizers.loss import MSELoss -from nerfactory.renderers.renderers import ( - AccumulationRenderer, - DepthRenderer, - RGBRenderer, -) -from nerfactory.utils import colors, visualization -from nerfactory.utils.callbacks import ( - TrainingCallback, - TrainingCallbackAttributes, - TrainingCallbackLocation, -) - - -@dataclass -class CompoundModelConfig(ModelConfig): - """Compound Model Config""" - - _target: Type = field( - default_factory=lambda: CompoundModel - ) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet - """target class to instantiate""" - enable_collider: bool = False - """Whether to create a scene collider to filter rays.""" - collider_params: Optional[Dict[str, float]] = None - """Instant NGP doesn't use a collider.""" - field_implementation: Literal["torch", "tcnn"] = "tcnn" - """one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'""" - num_samples: int = 1024 - """Number of max samples per ray""" - grid_resolution: int = 128 - """Resolution of the grid used for the field.""" - cone_angle: float = 0.0 - """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" - near_plane: float = 0.05 - """How far along ray to start sampling.""" - randomize_background: bool = False - """Whether to randomize the background color.""" - - -class CompoundModel(Model): - """Compound model - - Args: - config: Compound model configuration to instantiate model - """ - - config: CompoundModelConfig - - def __init__(self, config: CompoundModelConfig, **kwargs) -> None: - assert config.field_implementation in field_implementation_to_class - self.field = None - super().__init__(config=config, **kwargs) - - def populate_modules(self): - """Set the fields and modules.""" - super().populate_modules() - # torch or tiny-cuda-nn version - self.field = field_implementation_to_class[self.config.field_implementation]( - aabb=self.scene_bounds.aabb, num_images=self.num_train_data - ) - - self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False) - - # Sampler - self.sampler = VolumetricSampler( - aabb=self.scene_aabb, - density_fn=self.field.density_fn, - grid_resolution=self.config.grid_resolution, - num_samples=self.config.num_samples, - ) - - # renderers - background_color = None if self.config.randomize_background else colors.WHITE - self.renderer_rgb = RGBRenderer(background_color=background_color) - self.renderer_accumulation = AccumulationRenderer() - self.renderer_depth = DepthRenderer() - - # losses - self.rgb_loss = MSELoss() - - # metrics - self.psnr = PeakSignalNoiseRatio(data_range=1.0) - self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() - - # no colliders default - - def get_training_callbacks( - self, training_callback_attributes: TrainingCallbackAttributes - ) -> List[TrainingCallback]: - return [ - TrainingCallback( - where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], - update_every_num_iters=1, - func=self.sampler.occ_field.every_n_step, - ) - ] - - def get_param_groups(self) -> Dict[str, List[Parameter]]: - param_groups = {} - if self.field is None: - raise ValueError("populate_fields() must be called before get_param_groups") - param_groups["fields"] = list(self.field.parameters()) - return param_groups - - def get_outputs(self, ray_bundle: RayBundle): - assert self.field is not None - num_rays = len(ray_bundle) - - with torch.no_grad(): - ray_samples, packed_info, ray_indices = self.sampler( - ray_bundle=ray_bundle, - near_plane=self.config.near_plane, - ) - - field_outputs = self.field.forward(ray_samples) - - # accumulation - weights = nerfacc.volumetric_rendering_weights( - packed_info=packed_info, - sigmas=field_outputs[FieldHeadNames.DENSITY], - frustum_starts=ray_samples.frustums.starts, - frustum_ends=ray_samples.frustums.ends, - ) - - rgb = self.renderer_rgb( - rgb=field_outputs[FieldHeadNames.RGB], - weights=weights, - ray_indices=ray_indices, - num_rays=num_rays, - ) - depth = self.renderer_depth( - weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays - ) - accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays) - alive_ray_mask = accumulation.squeeze(-1) > 0 - - outputs = { - "rgb": rgb, - "accumulation": accumulation, - "depth": depth, - "alive_ray_mask": alive_ray_mask, # the rays we kept from sampler - "num_samples_per_ray": packed_info[:, 1], - } - return outputs - - def get_metrics_dict(self, outputs, batch): - image = batch["image"].to(self.device) - metrics_dict = {} - metrics_dict["psnr"] = self.psnr(outputs["rgb"], image) - metrics_dict["num_samples_per_batch"] = outputs["num_samples_per_ray"].sum() - return metrics_dict - - def get_loss_dict(self, outputs, batch, metrics_dict=None): - image = batch["image"].to(self.device) - mask = outputs["alive_ray_mask"] - rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) - loss_dict = {"rgb_loss": rgb_loss} - return loss_dict - - def get_image_metrics_and_images( - self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] - ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: - - image = batch["image"].to(self.device) - rgb = outputs["rgb"] - acc = visualization.apply_colormap(outputs["accumulation"]) - depth = visualization.apply_depth_colormap( - outputs["depth"], - accumulation=outputs["accumulation"], - ) - alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"]) - - combined_rgb = torch.cat([image, rgb], dim=1) - combined_acc = torch.cat([acc], dim=1) - combined_depth = torch.cat([depth], dim=1) - combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1) - - # Switch images from [H, W, C] to [1, C, H, W] for metrics computations - image = torch.moveaxis(image, -1, 0)[None, ...] - rgb = torch.moveaxis(rgb, -1, 0)[None, ...] - - psnr = self.psnr(image, rgb) - ssim = self.ssim(image, rgb) - lpips = self.lpips(image, rgb) - - # all of these metrics will be logged as scalars - metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore - # TODO(ethan): return an image dictionary - - images_dict = { - "img": combined_rgb, - "accumulation": combined_acc, - "depth": combined_depth, - "alive_ray_mask": combined_alive_ray_mask, - } - - return metrics_dict, images_dict diff --git a/nerfactory/models/instant_ngp.py b/nerfactory/models/instant_ngp.py index 2cd92283b..5c326c9b7 100644 --- a/nerfactory/models/instant_ngp.py +++ b/nerfactory/models/instant_ngp.py @@ -19,17 +19,18 @@ Implementation of Instant NGP. from __future__ import annotations from dataclasses import dataclass, field -from typing import Dict, List, Literal, Optional, Tuple, Type +from typing import Dict, List, Optional, Tuple, Type import nerfacc import torch +from nerfacc import ContractionType from torch.nn import Parameter from torchmetrics import PeakSignalNoiseRatio from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfactory.cameras.rays import RayBundle -from nerfactory.fields.instant_ngp_field import field_implementation_to_class +from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField from nerfactory.fields.modules.field_heads import FieldHeadNames from nerfactory.models.base import Model, ModelConfig from nerfactory.models.modules.ray_sampler import VolumetricSampler @@ -59,17 +60,23 @@ class InstantNGPModelConfig(ModelConfig): """Whether to create a scene collider to filter rays.""" collider_params: Optional[Dict[str, float]] = None """Instant NGP doesn't use a collider.""" - field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ... - """one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'""" - max_num_samples_per_ray: int = 1024 + max_num_samples_per_ray: int = 24 """Number of samples in field evaluation.""" grid_resolution: int = 128 """Resolution of the grid used for the field.""" - cone_angle: float = 0.0 + contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE + """Resolution of the grid used for the field.""" + cone_angle: float = 0.004 """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" + render_step_size: float = 0.01 + """Minimum step size for rendering.""" near_plane: float = 0.05 """How far along ray to start sampling.""" - randomize_background: bool = False + far_plane: float = 1e3 + """How far along ray to stop sampling.""" + use_appearance_embedding: bool = False + """Whether to use an appearance embedding.""" + randomize_background: bool = True """Whether to randomize the background color.""" @@ -81,27 +88,37 @@ class NGPModel(Model): """ config: InstantNGPModelConfig + field: TCNNInstantNGPField def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None: - assert config.field_implementation in field_implementation_to_class - self.field = None super().__init__(config=config, **kwargs) def populate_modules(self): """Set the fields and modules.""" super().populate_modules() - # torch or tiny-cuda-nn version - self.field = field_implementation_to_class[self.config.field_implementation](self.scene_bounds.aabb) + self.field = TCNNInstantNGPField( + aabb=self.scene_bounds.aabb, + contraction_type=self.config.contraction_type, + use_appearance_embedding=self.config.use_appearance_embedding, + num_images=self.num_train_data, + ) self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False) + # Occupancy Grid + self.occupancy_grid = nerfacc.OccupancyGrid( + roi_aabb=self.scene_aabb, + resolution=self.config.grid_resolution, + contraction_type=self.config.contraction_type, + ) + # Sampler + vol_sampler_aabb = self.scene_bounds.aabb if self.config.contraction_type == ContractionType.AABB else None self.sampler = VolumetricSampler( - aabb=self.scene_aabb, + scene_aabb=vol_sampler_aabb, + occupancy_grid=self.occupancy_grid, density_fn=self.field.density_fn, - grid_resolution=self.config.grid_resolution, - num_samples=self.config.max_num_samples_per_ray, ) # renderers @@ -121,11 +138,17 @@ class NGPModel(Model): def get_training_callbacks( self, training_callback_attributes: TrainingCallbackAttributes ) -> List[TrainingCallback]: + def update_occupancy_grid(step: int): + self.occupancy_grid.every_n_step( + step=step, + occ_eval_fn=lambda x: self.field.get_opacity(x, self.config.render_step_size), + ) + return [ TrainingCallback( where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], update_every_num_iters=1, - func=self.sampler.occ_field.every_n_step, + func=update_occupancy_grid, ), ] @@ -144,16 +167,19 @@ class NGPModel(Model): ray_samples, packed_info, ray_indices = self.sampler( ray_bundle=ray_bundle, near_plane=self.config.near_plane, + far_plane=self.config.far_plane, + render_step_size=self.config.render_step_size, + cone_angle=self.config.cone_angle, ) field_outputs = self.field(ray_samples) # accumulation - weights = nerfacc.volumetric_rendering_weights( + weights = nerfacc.render_weight_from_density( packed_info=packed_info, sigmas=field_outputs[FieldHeadNames.DENSITY], - frustum_starts=ray_samples.frustums.starts, - frustum_ends=ray_samples.frustums.ends, + t_starts=ray_samples.frustums.starts, + t_ends=ray_samples.frustums.ends, ) rgb = self.renderer_rgb( diff --git a/nerfactory/models/modules/ray_sampler.py b/nerfactory/models/modules/ray_sampler.py index 1c04c2d56..cc1d57eb5 100644 --- a/nerfactory/models/modules/ray_sampler.py +++ b/nerfactory/models/modules/ray_sampler.py @@ -16,12 +16,12 @@ Collection of sampling strategies """ -import math from abc import abstractmethod from typing import Callable, List, Optional, Tuple import nerfacc import torch +from nerfacc import OccupancyGrid from torch import nn from torchtyping import TensorType @@ -398,46 +398,52 @@ class PDFSampler(Sampler): class VolumetricSampler(Sampler): """Sampler inspired by the one proposed in the Instant-NGP paper. + Generates samples along a ray by sampling the occupancy field. + Optionally removes occluded samples if the density_fn is provided. - This sampler does ray-box AABB test, ray samples generation and density check, all together. - - Args: - aabb: Bounding box of the scene. - density_fn: Function that takes a tensor of points and returns a tensor of densities. - grid_resolution: Resolution of the density grid. - num_samples: Number of samples per ray. + Args: + occupancy_grid: Occupancy grid to sample from. + density_fn: Function that evaluates density at a given point. + scene_aabb: Axis-aligned bounding box of the scene, should be set to None if the scene is unbounded. """ def __init__( self, - aabb: TensorType[2, 3], - density_fn: Callable[[TensorType["N", 3]], TensorType["N"]], - grid_resolution: int = 128, - num_samples: int = 1024, + occupancy_grid: Optional[OccupancyGrid] = None, + density_fn: Optional[Callable[[TensorType[..., 3]], TensorType[..., 1]]] = None, + scene_aabb: Optional[TensorType[2, 3]] = None, ) -> None: - super().__init__(num_samples=num_samples) - self.aabb = aabb + super().__init__() + self.scene_aabb = scene_aabb self.density_fn = density_fn + self.occupancy_grid = occupancy_grid + if self.scene_aabb is not None: + self.scene_aabb = self.scene_aabb.to("cuda").flatten() + print(self.scene_aabb) - self.render_step_size = ((self.aabb[3:] - self.aabb[:3]).max() * math.sqrt(3) / num_samples).item() + def get_sigma_fn(self, origins, directions) -> Optional[Callable]: + """Returns a function that returns the density of a point. - # setup occupancy field with eval function - def occ_eval_fn(x: torch.Tensor) -> torch.Tensor: - """Evaluate occupancy given positions. + Args: + origins: Origins of rays + directions: Directions of rays + Returns: + Function that returns the density of a point or None if a density function is not provided. + """ - Args: - x: positions with shape (N, 3). - Returns: - occupancy values with shape (N, 1). - """ - density_after_activation = density_fn(x) - # those two are similar when density is small. - # occupancy = 1.0 - torch.exp(-density_after_activation * render_step_size) - occupancy = density_after_activation * self.render_step_size - return occupancy + if self.density_fn is None or not self.training: + return None - self.occ_field = nerfacc.OccupancyField(occ_eval_fn=occ_eval_fn, aabb=aabb, resolution=grid_resolution) + density_fn = self.density_fn + + def sigma_fn(t_starts, t_ends, ray_indices): + t_origins = origins[ray_indices] + t_dirs = directions[ray_indices] + positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0 + return density_fn(positions) + + return sigma_fn def generate_ray_samples(self) -> RaySamples: raise RuntimeError( @@ -448,17 +454,19 @@ class VolumetricSampler(Sampler): def forward( self, ray_bundle: RayBundle, - num_samples: Optional[int] = None, + render_step_size: float, near_plane: float = 0.0, - remove_occluded_samples: bool = True, + far_plane: Optional[float] = None, + cone_angle: float = 0.0, ) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples", 2]]: """Generate ray samples in a bounding box. Args: ray_bundle: Rays to generate samples for - num_samples: Number of samples per ray + render_step_size: Minimum step size to use for rendering near_plane: Near plane for raymarching - remove_occluded_samples: Whether to remove occluded samples. (requires evaluating the density function) + far_plane: Far plane for raymarching + cone_angle: Cone angle for raymarching, set to 0 for uniform marching. Returns: a tuple of (ray_samples, packed_info, ray_indices) @@ -470,12 +478,6 @@ class VolumetricSampler(Sampler): if self.density_field is not None: raise ValueError("NGPSpacedSampler does not support a density field") - if num_samples is not None: - render_step_size = ((self.aabb[3:] - self.aabb[:3]).max() * math.sqrt(3) / num_samples).item() - else: - num_samples = self.num_samples - render_step_size = self.render_step_size - rays_o = ray_bundle.origins.contiguous() rays_d = ray_bundle.directions.contiguous() if ray_bundle.camera_indices is not None: @@ -483,15 +485,17 @@ class VolumetricSampler(Sampler): else: camera_indices = None - packed_info, starts, ends = nerfacc.volumetric_marching( + packed_info, starts, ends = nerfacc.ray_marching( rays_o=rays_o, rays_d=rays_d, - aabb=self.aabb, - scene_resolution=self.occ_field.resolution, - scene_occ_binary=self.occ_field.occ_grid_binary, + scene_aabb=self.scene_aabb, + grid=self.occupancy_grid, + sigma_fn=self.get_sigma_fn(rays_o, rays_d), render_step_size=render_step_size, near_plane=near_plane, + far_plane=far_plane, stratified=self.training, + cone_angle=cone_angle, ) num_samples = starts.shape[0] @@ -508,20 +512,6 @@ class VolumetricSampler(Sampler): if camera_indices is not None: camera_indices = camera_indices[ray_indices] - if remove_occluded_samples and self.training: - with torch.no_grad(): - positions = origins + dirs * (starts + ends) / 2.0 - densities = self.density_fn(positions) - if camera_indices is not None: - packed_info, starts, ends, origins, dirs, camera_indices = nerfacc.volumetric_rendering_steps( - packed_info, densities, starts, ends, origins, dirs, camera_indices - ) - else: - packed_info, starts, ends, origins, dirs = nerfacc.volumetric_rendering_steps( - packed_info, densities, starts, ends, origins, dirs - ) - ray_indices = nerfacc.unpack_to_ray_indices(packed_info).long() - zeros = torch.zeros_like(origins[:, :1]) ray_samples = RaySamples( frustums=Frustums( diff --git a/nerfactory/models/proposal.py b/nerfactory/models/proposal.py index 7a08faffb..2453552b9 100644 --- a/nerfactory/models/proposal.py +++ b/nerfactory/models/proposal.py @@ -31,7 +31,6 @@ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfactory.cameras.rays import RayBundle from nerfactory.fields.compound_field import TCNNCompoundField from nerfactory.fields.density_field import DensityField -from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField from nerfactory.fields.modules.field_heads import FieldHeadNames from nerfactory.fields.modules.spatial_distortions import SceneContraction from nerfactory.models.base import Model, ModelConfig @@ -106,7 +105,7 @@ class ProposalModel(Model): self.scene_bounds.aabb, spatial_distortion=scene_contraction, num_images=self.num_train_data ) else: - self.field = TCNNInstantNGPField(self.scene_bounds.aabb, spatial_distortion=scene_contraction) + raise NotImplementedError("Only appearance conditioning is supported.") # Build the proposal network(s) self.proposal_networks = torch.nn.ModuleList() diff --git a/nerfactory/renderers/renderers.py b/nerfactory/renderers/renderers.py index 67b6a6e69..538050414 100644 --- a/nerfactory/renderers/renderers.py +++ b/nerfactory/renderers/renderers.py @@ -71,8 +71,9 @@ class RGBRenderer(nn.Module): Outputs rgb values. """ if ray_indices is not None and num_rays is not None: - rgb = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, rgb, num_rays) - accumulated_weight = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, None, num_rays) + # Necessary for packed samples from volumetric ray sampler + rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays) + accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays) else: rgb = torch.sum(weights * rgb, dim=-2) accumulated_weight = torch.sum(weights, dim=-2) @@ -181,7 +182,8 @@ class AccumulationRenderer(nn.Module): """ if ray_indices is not None and num_rays is not None: - accumulation = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, None, num_rays) + # Necessary for packed samples from volumetric ray sampler + accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays) else: accumulation = torch.sum(weights, dim=-2) return accumulation @@ -222,7 +224,8 @@ class DepthRenderer(nn.Module): steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2 if ray_indices is not None and num_rays is not None: - depth = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, steps, num_rays) + # Necessary for packed samples from volumetric ray sampler + depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays) else: depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps) diff --git a/pyproject.toml b/pyproject.toml index a07451b48..7294e4363 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "mediapy==1.1.0", "msgpack==1.0.4", "msgpack_numpy==0.4.8", - "nerfacc==0.0.9", + "nerfacc==0.1.1", "opencv-python==4.6.0.66", "plotly==5.7.0", "protobuf==3.20.0", diff --git a/tests/test_run_train.py b/tests/test_run_train.py index e15c0aa48..b06eb4222 100644 --- a/tests/test_run_train.py +++ b/tests/test_run_train.py @@ -14,7 +14,7 @@ from nerfactory.configs.base_configs import base_configs from nerfactory.datamanagers.dataparsers.blender_parser import BlenderDataParserConfig from nerfactory.engine.trainer import train_loop -BLACKLIST = ["base", "semantic-nerf", "mipnerf-360", "instant-ngp", "compound", "proposal"] +BLACKLIST = ["base", "semantic-nerf", "mipnerf-360", "instant-ngp", "proposal"] def set_reduced_config(config: Config):