From 04ccc32cdf8ef8785d23d89e95f2f34be30ce793 Mon Sep 17 00:00:00 2001 From: Ruilong Li Date: Sun, 9 Apr 2023 09:35:39 +0000 Subject: [PATCH] update to nerfacc 0.5.1. Upgrade instant-ngp model. --- nerfstudio/configs/method_configs.py | 27 +++--- .../data/datamanagers/base_datamanager.py | 11 ++- nerfstudio/data/pixel_samplers.py | 3 - nerfstudio/fields/instant_ngp_field.py | 55 +++++------ nerfstudio/fields/nerfacto_field.py | 2 +- nerfstudio/fields/nerfplayer_ngp_field.py | 34 ++++--- nerfstudio/model_components/ray_generators.py | 1 - nerfstudio/model_components/ray_samplers.py | 49 ++++------ nerfstudio/model_components/renderers.py | 15 +-- nerfstudio/models/instant_ngp.py | 88 ++++++++++-------- nerfstudio/models/nerfplayer_ngp.py | 29 +++--- nerfstudio/pipelines/base_pipeline.py | 1 - nerfstudio/utils/math.py | 4 +- tests/utils/test_aabb_intersection.py | 4 +- train.py.lprof | Bin 2380 -> 0 bytes 15 files changed, 165 insertions(+), 158 deletions(-) delete mode 100644 train.py.lprof diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 0357d0f9b..ce33b52de 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -21,7 +21,6 @@ from __future__ import annotations from typing import Dict import tyro -from nerfacc import ContractionType from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig @@ -232,16 +231,20 @@ method_configs["instant-ngp"] = TrainerConfig( max_num_iterations=30000, mixed_precision=True, pipeline=DynamicBatchPipelineConfig( - datamanager=VanillaDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=8192), + datamanager=VanillaDataManagerConfig( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=512, + eval_num_rays_per_batch=512, + ), model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192), ), optimizers={ "fields": { "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), - "scheduler": None, + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), } }, - viewer=ViewerConfig(num_rays_per_chunk=64000), + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) @@ -256,20 +259,22 @@ method_configs["instant-ngp-bounded"] = TrainerConfig( datamanager=VanillaDataManagerConfig(dataparser=InstantNGPDataParserConfig(), train_num_rays_per_batch=8192), model=InstantNGPModelConfig( eval_num_rays_per_chunk=8192, - contraction_type=ContractionType.AABB, - render_step_size=0.001, - max_num_samples_per_ray=48, + grid_levels=1, + alpha_thre=0., + cone_angle=0., + render_step_size=5e-3, + disable_scene_contraction=True, near_plane=0.01, - background_color="black", + background_color="white", ), ), optimizers={ "fields": { "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), - "scheduler": None, + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), } }, - viewer=ViewerConfig(num_rays_per_chunk=64000), + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) @@ -467,9 +472,7 @@ method_configs["nerfplayer-ngp"] = TrainerConfig( datamanager=DepthDataManagerConfig(dataparser=DycheckDataParserConfig(), train_num_rays_per_batch=8192), model=NerfplayerNGPModelConfig( eval_num_rays_per_chunk=8192, - contraction_type=ContractionType.AABB, render_step_size=0.001, - max_num_samples_per_ray=48, near_plane=0.01, ), ), diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index c4e71e23f..60d335e12 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -518,7 +518,6 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method ) @profiler.time_function - @profile def next_train(self, step: int) -> Tuple[RayBundle, Dict]: """Returns the next batch of data from the train dataloader.""" self.train_count += 1 @@ -547,10 +546,16 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method raise ValueError("No more eval images") def get_train_rays_per_batch(self) -> int: - return self.config.train_num_rays_per_batch + if self.train_pixel_sampler is None: + return self.config.train_num_rays_per_batch + else: + return self.train_pixel_sampler.num_rays_per_batch def get_eval_rays_per_batch(self) -> int: - return self.config.eval_num_rays_per_batch + if self.eval_pixel_sampler is None: + return self.config.eval_num_rays_per_batch + else: + return self.eval_pixel_sampler.num_rays_per_batch def get_datapath(self) -> Path: return self.config.dataparser.data diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 261a3795e..1d5a9697e 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -74,7 +74,6 @@ class PixelSampler: # pylint: disable=too-few-public-methods return indices - @profile def collate_image_dataset_batch(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): """ Operates on a batch of images and samples pixels to use for generating rays. @@ -113,7 +112,6 @@ class PixelSampler: # pylint: disable=too-few-public-methods return collated_batch - @profile def collate_image_dataset_batch_list(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): """ Does the same as collate_image_dataset_batch, except it will operate over a list of images / masks inside @@ -186,7 +184,6 @@ class PixelSampler: # pylint: disable=too-few-public-methods return collated_batch @profiler.time_function - @profile def sample(self, image_batch: Dict): """Sample an image batch and return a pixel batch. diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index d1b3e2fa7..8d6bed021 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -21,8 +21,6 @@ from typing import Dict, Optional, Tuple import numpy as np import torch -from nerfacc import ContractionType, contract -from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples @@ -30,6 +28,10 @@ from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp from nerfstudio.field_components.embedding import Embedding from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import ( + SceneContraction, + SpatialDistortion, +) from nerfstudio.fields.base_field import Field, shift_directions_for_tcnn try: @@ -69,16 +71,16 @@ class TCNNInstantNGPField(Field): use_appearance_embedding: Optional[bool] = False, num_images: Optional[int] = None, appearance_embedding_dim: int = 32, - contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE, num_levels: int = 16, log2_hashmap_size: int = 19, max_res: int = 2048, + spatial_distortion: Optional[SpatialDistortion] = SceneContraction(), ) -> None: super().__init__() - self.aabb = Parameter(aabb, requires_grad=False) + self.register_buffer("aabb", aabb) self.geo_feat_dim = geo_feat_dim - self.contraction_type = contraction_type + self.spatial_distortion = spatial_distortion self.use_appearance_embedding = use_appearance_embedding if use_appearance_embedding: @@ -88,7 +90,8 @@ class TCNNInstantNGPField(Field): # TODO: set this properly based on the aabb base_res: int = 16 - per_level_scale = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) + features_per_level: int = 2 + growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) self.direction_encoding = tcnn.Encoding( n_input_dims=3, @@ -104,10 +107,10 @@ class TCNNInstantNGPField(Field): encoding_config={ "otype": "HashGrid", "n_levels": num_levels, - "n_features_per_level": 2, + "n_features_per_level": features_per_level, "log2_hashmap_size": log2_hashmap_size, "base_resolution": base_res, - "per_level_scale": per_level_scale, + "per_level_scale": growth_factor, }, network_config={ "otype": "FullyFusedMLP", @@ -134,9 +137,16 @@ class TCNNInstantNGPField(Field): ) def get_density(self, ray_samples: RaySamples) -> Tuple[TensorType, TensorType]: - positions = ray_samples.frustums.get_positions() + if self.spatial_distortion is not None: + positions = ray_samples.frustums.get_positions() + positions = self.spatial_distortion(positions) + positions = (positions + 2.0) / 4.0 + else: + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + # Make sure the tcnn gets inputs between 0 and 1. + selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) + positions = positions * selector[..., None] positions_flat = positions.view(-1, 3) - positions_flat = contract(x=positions_flat, roi=self.aabb, type=self.contraction_type) h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) @@ -145,20 +155,18 @@ class TCNNInstantNGPField(Field): # softplus, because it enables high post-activation (float32) density outputs # from smaller internal (float16) parameters. density = trunc_exp(density_before_activation.to(positions)) + density = density * selector[..., None] return density, base_mlp_out def get_outputs( self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None ) -> Dict[FieldHeadNames, TensorType]: + assert density_embedding is not None directions = shift_directions_for_tcnn(ray_samples.frustums.directions) directions_flat = directions.view(-1, 3) - d = self.direction_encoding(directions_flat) - if density_embedding is None: - positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) - h = torch.cat([d, positions.view(-1, 3)], dim=-1) - else: - h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) + + h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) if self.use_appearance_embedding: if ray_samples.camera_indices is None: @@ -183,20 +191,5 @@ class TCNNInstantNGPField(Field): step_size: the step size to use for the opacity evaluation. """ density = self.density_fn(positions) - ## TODO: We should scale step size based on the distortion. Currently it uses too much memory. - # aabb_min, aabb_max = self.aabb[0], self.aabb[1] - # if self.contraction_type is not ContractionType.AABB: - # x = (positions - aabb_min) / (aabb_max - aabb_min) - # x = x * 2 - 1 # aabb is at [-1, 1] - # mag = x.norm(dim=-1, keepdim=True) - # mask = mag.squeeze(-1) > 1 - - # dev = (2 * mag - 1) / mag**2 + 2 * x**2 * (1 / mag**3 - (2 * mag - 1) / mag**4) - # dev[~mask] = 1.0 - # dev = torch.clamp(dev, min=1e-6) - # step_size = step_size / dev.norm(dim=-1, keepdim=True) - # else: - # step_size = step_size * (aabb_max - aabb_min) - opacity = density * step_size return opacity diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index de0fd7c7d..acc868648 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -103,7 +103,7 @@ class TCNNNerfactoField(Field): pass_semantic_gradients: bool = False, use_pred_normals: bool = False, use_average_appearance_embedding: bool = False, - spatial_distortion: SpatialDistortion = None, + spatial_distortion: Optional[SpatialDistortion] = None, ) -> None: super().__init__() diff --git a/nerfstudio/fields/nerfplayer_ngp_field.py b/nerfstudio/fields/nerfplayer_ngp_field.py index 2952a4369..bbf7b72e6 100644 --- a/nerfstudio/fields/nerfplayer_ngp_field.py +++ b/nerfstudio/fields/nerfplayer_ngp_field.py @@ -20,7 +20,6 @@ NeRFPlayer (https://arxiv.org/abs/2210.15947) field implementations with Instant from typing import Dict, Optional, Tuple import torch -from nerfacc import ContractionType, contract from torch.nn.parameter import Parameter from torchtyping import TensorType @@ -29,6 +28,10 @@ from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp from nerfstudio.field_components.embedding import Embedding from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import ( + SceneContraction, + SpatialDistortion, +) from nerfstudio.field_components.temporal_grid import TemporalGridEncoder from nerfstudio.fields.base_field import Field, shift_directions_for_tcnn @@ -82,13 +85,13 @@ class NerfplayerNGPField(Field): disable_viewing_dependent: bool = False, num_images: Optional[int] = None, appearance_embedding_dim: int = 32, - contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE, + spatial_distortion: Optional[SpatialDistortion] = SceneContraction(), ) -> None: super().__init__() self.aabb = Parameter(aabb, requires_grad=False) self.geo_feat_dim = geo_feat_dim - self.contraction_type = contraction_type + self.spatial_distortion = spatial_distortion self.use_appearance_embedding = use_appearance_embedding if use_appearance_embedding: @@ -144,9 +147,16 @@ class NerfplayerNGPField(Field): ) def get_density(self, ray_samples: RaySamples) -> Tuple[TensorType, TensorType]: - positions = ray_samples.frustums.get_positions() + if self.spatial_distortion is not None: + positions = ray_samples.frustums.get_positions() + positions = self.spatial_distortion(positions) + positions = (positions + 2.0) / 4.0 + else: + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + # Make sure the tcnn gets inputs between 0 and 1. + selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) + positions = positions * selector[..., None] positions_flat = positions.view(-1, 3) - positions_flat = contract(x=positions_flat, roi=self.aabb, type=self.contraction_type) assert ray_samples.times is not None, "Time should be included in the input for NeRFPlayer" times_flat = ray_samples.times.view(-1, 1) @@ -158,28 +168,22 @@ class NerfplayerNGPField(Field): # softplus, because it enables high post-activation (float32) density outputs # from smaller internal (float16) parameters. density = trunc_exp(density_before_activation.to(positions)) + density = density * selector[..., None] return density, base_mlp_out def get_outputs( self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None ) -> Dict[FieldHeadNames, TensorType]: + assert density_embedding is not None directions = shift_directions_for_tcnn(ray_samples.frustums.directions) directions_flat = directions.view(-1, 3) if self.direction_encoding is not None: d = self.direction_encoding(directions_flat) - if density_embedding is None: - positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) - h = torch.cat([d, positions.view(-1, 3)], dim=-1) - else: - h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) + h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) else: # viewing direction is disabled - if density_embedding is None: - positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) - h = positions.view(-1, 3) - else: - h = density_embedding.view(-1, self.geo_feat_dim) + h = density_embedding.view(-1, self.geo_feat_dim) if self.use_appearance_embedding: if ray_samples.camera_indices is None: diff --git a/nerfstudio/model_components/ray_generators.py b/nerfstudio/model_components/ray_generators.py index 8d7170f45..71d3e5cfd 100644 --- a/nerfstudio/model_components/ray_generators.py +++ b/nerfstudio/model_components/ray_generators.py @@ -47,7 +47,6 @@ class RayGenerator(nn.Module): self.register_buffer("image_coords", cameras.get_image_coords(), persistent=False) @profiler.time_function - @profile def forward(self, ray_indices: TensorType["num_rays", 3]) -> RayBundle: """Index into the cameras to generate the rays. diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index f5a96255b..968b315c0 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -19,9 +19,8 @@ Collection of sampling strategies from abc import abstractmethod from typing import Callable, List, Optional, Tuple, Union -import nerfacc import torch -from nerfacc import OccupancyGrid +from nerfacc import OccGridEstimator from torch import nn from torchtyping import TensorType @@ -382,17 +381,14 @@ class VolumetricSampler(Sampler): def __init__( self, - occupancy_grid: Optional[OccupancyGrid] = None, + occupancy_grid: OccGridEstimator, density_fn: Optional[Callable[[TensorType[..., 3]], TensorType[..., 1]]] = None, - scene_aabb: Optional[TensorType[2, 3]] = None, ) -> None: super().__init__() - self.scene_aabb = scene_aabb + assert occupancy_grid is not None self.density_fn = density_fn self.occupancy_grid = occupancy_grid - if self.scene_aabb is not None: - self.scene_aabb = self.scene_aabb.to("cuda").flatten() def get_sigma_fn(self, origins, directions) -> Optional[Callable]: """Returns a function that returns the density of a point. @@ -412,8 +408,8 @@ class VolumetricSampler(Sampler): def sigma_fn(t_starts, t_ends, ray_indices): t_origins = origins[ray_indices] t_dirs = directions[ray_indices] - positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0 - return density_fn(positions) + positions = t_origins + t_dirs * (t_starts + t_ends)[:, None] / 2.0 + return density_fn(positions).squeeze(-1) return sigma_fn @@ -429,6 +425,7 @@ class VolumetricSampler(Sampler): render_step_size: float, near_plane: float = 0.0, far_plane: Optional[float] = None, + alpha_thre: float = 0.01, cone_angle: float = 0.0, ) -> Tuple[RaySamples, TensorType["total_samples",]]: """Generate ray samples in a bounding box. @@ -438,6 +435,7 @@ class VolumetricSampler(Sampler): render_step_size: Minimum step size to use for rendering near_plane: Near plane for raymarching far_plane: Far plane for raymarching + alpha_thre: Opacity threshold skipping samples. cone_angle: Cone angle for raymarching, set to 0 for uniform marching. Returns: @@ -446,45 +444,34 @@ class VolumetricSampler(Sampler): The ray_indices contains the indices of the rays that each sample belongs to. """ - rays_o = ray_bundle.origins.contiguous() - rays_d = ray_bundle.directions.contiguous() + rays_o = ray_bundle.origins + rays_d = ray_bundle.directions - if ray_bundle.nears is not None and ray_bundle.fars is not None: - t_min = ray_bundle.nears.contiguous().reshape(-1) - t_max = ray_bundle.fars.contiguous().reshape(-1) - - else: - t_min = None - t_max = None + if far_plane is None: + far_plane = 1e10 if ray_bundle.camera_indices is not None: camera_indices = ray_bundle.camera_indices.contiguous() else: camera_indices = None - - ray_indices, starts, ends = nerfacc.ray_marching( + ray_indices, starts, ends = self.occupancy_grid.sampling( rays_o=rays_o, rays_d=rays_d, - t_min=t_min, - t_max=t_max, - scene_aabb=self.scene_aabb, - grid=self.occupancy_grid, - # this is a workaround - using density causes crash and damage quality. should be fixed - sigma_fn=None, # self.get_sigma_fn(rays_o, rays_d), + sigma_fn=self.get_sigma_fn(rays_o, rays_d), render_step_size=render_step_size, near_plane=near_plane, far_plane=far_plane, stratified=self.training, cone_angle=cone_angle, - alpha_thre=1e-2, + alpha_thre=alpha_thre, ) num_samples = starts.shape[0] if num_samples == 0: # create a single fake sample and update packed_info accordingly # this says the last ray in packed_info has 1 sample, which starts and ends at 1 ray_indices = torch.zeros((1,), dtype=torch.long, device=rays_o.device) - starts = torch.ones((1, 1), dtype=starts.dtype, device=rays_o.device) - ends = torch.ones((1, 1), dtype=ends.dtype, device=rays_o.device) + starts = torch.ones((1,), dtype=starts.dtype, device=rays_o.device) + ends = torch.ones((1,), dtype=ends.dtype, device=rays_o.device) origins = rays_o[ray_indices] dirs = rays_d[ray_indices] @@ -496,8 +483,8 @@ class VolumetricSampler(Sampler): frustums=Frustums( origins=origins, directions=dirs, - starts=starts, - ends=ends, + starts=starts[..., None], + ends=ends[..., None], pixel_area=zeros, ), camera_indices=camera_indices, diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 43f81404c..cd2c18f10 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -91,8 +91,8 @@ class RGBRenderer(nn.Module): # Necessary for packed samples from volumetric ray sampler if background_color == "last_sample": raise NotImplementedError("Background color 'last_sample' not implemented for packed samples.") - comp_rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays) - accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays) + comp_rgb = nerfacc.accumulate_along_rays(weights.squeeze(-1), rgb, ray_indices, num_rays) + accumulated_weight = nerfacc.accumulate_along_rays(weights.squeeze(-1), None, ray_indices, num_rays) else: comp_rgb = torch.sum(weights * rgb, dim=-2) accumulated_weight = torch.sum(weights, dim=-2) @@ -102,12 +102,13 @@ class RGBRenderer(nn.Module): if background_color == "last_sample": background_color = rgb[..., -1, :] if background_color == "random": - background_color = torch.rand_like(comp_rgb).to(rgb.device) + background_color = torch.rand_like(comp_rgb) if isinstance(background_color, str) and background_color in colors.COLORS_DICT: + # TODO(ruilongli): Avoid to(device) by moving it to the constructor. background_color = colors.COLORS_DICT[background_color].to(rgb.device) assert isinstance(background_color, torch.Tensor) - comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight) + comp_rgb = comp_rgb + background_color * (1.0 - accumulated_weight) return comp_rgb @@ -217,7 +218,7 @@ class AccumulationRenderer(nn.Module): if ray_indices is not None and num_rays is not None: # Necessary for packed samples from volumetric ray sampler - accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays) + accumulation = nerfacc.accumulate_along_rays(weights.squeeze(-1), None, ray_indices, num_rays) else: accumulation = torch.sum(weights, dim=-2) return accumulation @@ -274,8 +275,8 @@ class DepthRenderer(nn.Module): if ray_indices is not None and num_rays is not None: # Necessary for packed samples from volumetric ray sampler - depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays) - accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays) + depth = nerfacc.accumulate_along_rays(weights.squeeze(-1), steps, ray_indices, num_rays) + accumulation = nerfacc.accumulate_along_rays(weights.squeeze(-1), None, ray_indices, num_rays) depth = depth / (accumulation + eps) else: depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps) diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index 2bdf85b6a..b0d9c1e4d 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -23,8 +23,9 @@ from typing import Dict, List, Optional, Tuple, Type import nerfacc import torch -from nerfacc import ContractionType from torch.nn import Parameter + +# from torch_efficient_distloss import flatten_eff_distloss from torchmetrics import PeakSignalNoiseRatio from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity @@ -37,7 +38,8 @@ from nerfstudio.engine.callbacks import ( TrainingCallbackLocation, ) from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.instant_ngp_field import TCNNInstantNGPField +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.nerfacto_field import TCNNNerfactoField from nerfstudio.model_components.losses import MSELoss from nerfstudio.model_components.ray_samplers import VolumetricSampler from nerfstudio.model_components.renderers import ( @@ -57,32 +59,30 @@ class InstantNGPModelConfig(ModelConfig): default_factory=lambda: NGPModel ) # We can't write `NGPModel` directly, because `NGPModel` doesn't exist yet """target class to instantiate""" - enable_collider: bool = False - """Whether to create a scene collider to filter rays.""" - collider_params: Optional[Dict[str, float]] = None - """Instant NGP doesn't use a collider.""" - max_num_samples_per_ray: int = 24 - """Number of samples in field evaluation.""" grid_resolution: int = 128 """Resolution of the grid used for the field.""" + grid_levels: int = 4 + """Levels of the grid used for the field.""" max_res: int = 2048 """Maximum resolution of the hashmap for the base mlp.""" log2_hashmap_size: int = 19 """Size of the hashmap for the base mlp""" - contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE - """Contraction type used for spatial deformation of the field.""" + alpha_thre: float = 0.01 + """Threshold for opacity skipping.""" cone_angle: float = 0.004 """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" - render_step_size: float = 0.01 + render_step_size: Optional[float] = None """Minimum step size for rendering.""" near_plane: float = 0.05 """How far along ray to start sampling.""" far_plane: float = 1e3 """How far along ray to stop sampling.""" - use_appearance_embedding: bool = False - """Whether to use an appearance embedding.""" background_color: Literal["random", "black", "white"] = "random" """The color that is given to untrained areas.""" + disable_scene_contraction: bool = False + """Whether to disable scene contraction or not.""" + # distortion_loss_mult: float = 0.002 + # """Distortion loss multiplier.""" class NGPModel(Model): @@ -93,7 +93,7 @@ class NGPModel(Model): """ config: InstantNGPModelConfig - field: TCNNInstantNGPField + field: TCNNNerfactoField def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None: super().__init__(config=config, **kwargs) @@ -102,28 +102,35 @@ class NGPModel(Model): """Set the fields and modules.""" super().populate_modules() - self.field = TCNNInstantNGPField( + # scene aabb + self.scene_aabb = Parameter(self.scene_box.aabb.flatten(), requires_grad=False) + + if self.config.disable_scene_contraction: + scene_contraction = None + else: + scene_contraction = SceneContraction(order=float("inf")) + + self.field = TCNNNerfactoField( aabb=self.scene_box.aabb, - contraction_type=self.config.contraction_type, - use_appearance_embedding=self.config.use_appearance_embedding, num_images=self.num_train_data, log2_hashmap_size=self.config.log2_hashmap_size, max_res=self.config.max_res, + spatial_distortion=scene_contraction, ) - self.scene_aabb = Parameter(self.scene_box.aabb.flatten(), requires_grad=False) + if self.config.render_step_size is None: + # auto step size: ~1000 samples in the base level grid + self.config.render_step_size = ((self.scene_aabb[3:] - self.scene_aabb[:3]) ** 2).sum().sqrt().item() / 1000 - # Occupancy Grid - self.occupancy_grid = nerfacc.OccupancyGrid( + # Occupancy Grid. + self.occupancy_grid = nerfacc.OccGridEstimator( roi_aabb=self.scene_aabb, resolution=self.config.grid_resolution, - contraction_type=self.config.contraction_type, + levels=self.config.grid_levels, ) # Sampler - vol_sampler_aabb = self.scene_box.aabb if self.config.contraction_type == ContractionType.AABB else None self.sampler = VolumetricSampler( - scene_aabb=vol_sampler_aabb, occupancy_grid=self.occupancy_grid, density_fn=self.field.density_fn, ) @@ -145,11 +152,9 @@ class NGPModel(Model): self, training_callback_attributes: TrainingCallbackAttributes ) -> List[TrainingCallback]: def update_occupancy_grid(step: int): - # TODO: needs to get access to the sampler, on how the step size is determinated at each x. See - # https://github.com/KAIR-BAIR/nerfacc/blob/127223b11401125a9fce5ce269bb0546ee4de6e8/examples/train_ngp_nerf.py#L190-L213 - self.occupancy_grid.every_n_step( + self.occupancy_grid.update_every_n_steps( step=step, - occ_eval_fn=lambda x: self.field.get_opacity(x, self.config.render_step_size), + occ_eval_fn=lambda x: self.field.density_fn(x) * self.config.render_step_size, ) return [ @@ -177,6 +182,7 @@ class NGPModel(Model): near_plane=self.config.near_plane, far_plane=self.config.far_plane, render_step_size=self.config.render_step_size, + alpha_thre=self.config.alpha_thre, cone_angle=self.config.cone_angle, ) @@ -186,10 +192,11 @@ class NGPModel(Model): packed_info = nerfacc.pack_info(ray_indices, num_rays) weights = nerfacc.render_weight_from_density( packed_info=packed_info, - sigmas=field_outputs[FieldHeadNames.DENSITY], - t_starts=ray_samples.frustums.starts, - t_ends=ray_samples.frustums.ends, - ) + sigmas=field_outputs[FieldHeadNames.DENSITY].squeeze(-1), + t_starts=ray_samples.frustums.starts.squeeze(-1), + t_ends=ray_samples.frustums.ends.squeeze(-1), + )[0] + weights = weights.unsqueeze(-1) rgb = self.renderer_rgb( rgb=field_outputs[FieldHeadNames.RGB], @@ -201,15 +208,20 @@ class NGPModel(Model): weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays ) accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays) - alive_ray_mask = accumulation.squeeze(-1) > 0 outputs = { "rgb": rgb, "accumulation": accumulation, "depth": depth, - "alive_ray_mask": alive_ray_mask, # the rays we kept from sampler "num_samples_per_ray": packed_info[:, 1], } + # if self.training: + # steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2 + # intervals = ray_samples.frustums.ends - ray_samples.frustums.starts + # outputs["weights"] = weights.flatten() + # outputs["steps"] = steps.flatten() + # outputs["intervals"] = intervals.flatten() + # outputs["ray_indices"] = ray_indices.flatten() return outputs def get_metrics_dict(self, outputs, batch): @@ -221,9 +233,12 @@ class NGPModel(Model): def get_loss_dict(self, outputs, batch, metrics_dict=None): image = batch["image"].to(self.device) - mask = outputs["alive_ray_mask"] - rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) + rgb_loss = self.rgb_loss(image, outputs["rgb"]) loss_dict = {"rgb_loss": rgb_loss} + # if self.training: + # loss_dict["distortion_loss"] = self.config.distortion_loss_mult * flatten_eff_distloss( + # outputs["weights"], outputs["steps"], outputs["intervals"], outputs["ray_indices"] + # ) return loss_dict def get_image_metrics_and_images( @@ -237,12 +252,10 @@ class NGPModel(Model): outputs["depth"], accumulation=outputs["accumulation"], ) - alive_ray_mask = colormaps.apply_colormap(outputs["alive_ray_mask"]) combined_rgb = torch.cat([image, rgb], dim=1) combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) - combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1) # Switch images from [H, W, C] to [1, C, H, W] for metrics computations image = torch.moveaxis(image, -1, 0)[None, ...] @@ -260,7 +273,6 @@ class NGPModel(Model): "img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth, - "alive_ray_mask": combined_alive_ray_mask, } return metrics_dict, images_dict diff --git a/nerfstudio/models/nerfplayer_ngp.py b/nerfstudio/models/nerfplayer_ngp.py index 09f68d67f..59a0743bc 100644 --- a/nerfstudio/models/nerfplayer_ngp.py +++ b/nerfstudio/models/nerfplayer_ngp.py @@ -23,7 +23,6 @@ from typing import Type import nerfacc import torch -from nerfacc import ContractionType from torch.nn import Parameter from torchmetrics import PeakSignalNoiseRatio from torchmetrics.functional import structural_similarity_index_measure @@ -32,6 +31,7 @@ from typing_extensions import Literal from nerfstudio.cameras.rays import RayBundle from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.nerfplayer_ngp_field import NerfplayerNGPField from nerfstudio.model_components.losses import MSELoss from nerfstudio.model_components.ray_samplers import VolumetricSampler @@ -80,6 +80,8 @@ class NerfplayerNGPModelConfig(InstantNGPModelConfig): """The training background color that is given to untrained areas.""" disable_viewing_dependent: bool = True """Disable viewing dependent effects.""" + disable_scene_contraction: bool = False + """Whether to disable scene contraction or not.""" class NerfplayerNGPModel(NGPModel): @@ -96,9 +98,13 @@ class NerfplayerNGPModel(NGPModel): """Set the fields and modules.""" Model.populate_modules(self) + if self.config.disable_scene_contraction: + scene_contraction = None + else: + scene_contraction = SceneContraction(order=float("inf")) + self.field = NerfplayerNGPField( aabb=self.scene_box.aabb, - contraction_type=self.config.contraction_type, use_appearance_embedding=self.config.use_appearance_embedding, num_images=self.num_train_data, temporal_dim=self.config.temporal_dim, @@ -107,21 +113,21 @@ class NerfplayerNGPModel(NGPModel): log2_hashmap_size=self.config.log2_hashmap_size, base_resolution=self.config.base_resolution, disable_viewing_dependent=self.config.disable_viewing_dependent, + spatial_distortion=scene_contraction, ) self.scene_aabb = Parameter(self.scene_box.aabb.flatten(), requires_grad=False) # Occupancy Grid - self.occupancy_grid = nerfacc.OccupancyGrid( - roi_aabb=self.scene_aabb, + grid_scale = 1 << (self.config.grid_levels - 1) + self.occupancy_grid = nerfacc.OccGridEstimator( + roi_aabb=nerfacc.grid._enlarge_aabb(self.scene_aabb, 1.0 / grid_scale), resolution=self.config.grid_resolution, - contraction_type=self.config.contraction_type, + levels=self.config.grid_levels, ) # Sampler - vol_sampler_aabb = self.scene_box.aabb if self.config.contraction_type == ContractionType.AABB else None self.sampler = VolumetricSampler( - scene_aabb=vol_sampler_aabb, occupancy_grid=self.occupancy_grid, density_fn=self.field.density_fn, ) # need to update the density_fn later during forward (for input time) @@ -161,8 +167,8 @@ class NerfplayerNGPModel(NGPModel): weights = nerfacc.render_weight_from_density( packed_info=packed_info, sigmas=field_outputs[FieldHeadNames.DENSITY], - t_starts=ray_samples.frustums.starts, - t_ends=ray_samples.frustums.ends, + t_starts=ray_samples.frustums.starts.squeeze(-1), + t_ends=ray_samples.frustums.ends.squeeze(-1), ) # update bgcolor in the renderer; usually random color for training and fixed color for inference @@ -180,13 +186,11 @@ class NerfplayerNGPModel(NGPModel): weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays ) accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays) - alive_ray_mask = accumulation.squeeze(-1) > 0 outputs = { "rgb": rgb, "accumulation": accumulation, "depth": depth, - "alive_ray_mask": alive_ray_mask, # the rays we kept from sampler "num_samples_per_ray": packed_info[:, 1], } # adding them to outputs for calculating losses @@ -199,8 +203,7 @@ class NerfplayerNGPModel(NGPModel): def get_loss_dict(self, outputs, batch, metrics_dict=None): image = batch["image"].to(self.device) - mask = outputs["alive_ray_mask"] - rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) + rgb_loss = self.rgb_loss(image, outputs["rgb"]) loss_dict = {"rgb_loss": rgb_loss} if "depth_image" in batch.keys() and self.config.depth_weight > 0: mask = batch["depth_image"] != 0 diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index c1dccc77f..c572bebeb 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -266,7 +266,6 @@ class VanillaPipeline(Pipeline): return self.model.device @profiler.time_function - @profile def get_train_loss_dict(self, step: int): """This function gets your training loss dict. This will be responsible for getting the next batch of data from the DataManager and interfacing with the diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index f54b6509a..9786382b4 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -262,7 +262,9 @@ def intersect_aabb( try: import nerfacc # pylint: disable=import-outside-toplevel - t_min, t_max = nerfacc.ray_aabb_intersect(origins, directions, aabb) + t_min, t_max, _ = nerfacc.ray_aabb_intersect( + origins, directions, aabb[None, :], near_plane=0, far_plane=1e10, miss_value=1e10) + t_min, t_max = t_min.squeeze(-1), t_max.squeeze(-1) except: # pylint: disable=bare-except t_min, t_max = _intersect_aabb(origins, directions, aabb, max_bound=1e10, invalid_value=1e10) _USE_NERFACC = False diff --git a/tests/utils/test_aabb_intersection.py b/tests/utils/test_aabb_intersection.py index 1d6ba34b9..a2aa29d43 100644 --- a/tests/utils/test_aabb_intersection.py +++ b/tests/utils/test_aabb_intersection.py @@ -168,7 +168,9 @@ def test_equall_nerfacc(): # time2 = time.time() # time3 = time.time() - t_min_nerfacc, t_max_nerfacc = nerfacc.ray_aabb_intersect(origins, directions, aabb) + t_min_nerfacc, t_max_nerfacc, _ = nerfacc.ray_aabb_intersect(origins, directions, aabb[None, :]) + t_max_nerfacc = t_max_nerfacc.squeeze(-1) + t_min_nerfacc = t_min_nerfacc.squeeze(-1) # time4 = time.time() # print("pytorch ", time2-time1) diff --git a/train.py.lprof b/train.py.lprof deleted file mode 100644 index 8fbdf9351162b03a3213724fa0ba25018812537f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2380 zcmb8xdrVVT90zd9tfjmad>|?T9CeCxqEkUYWfP)pRYY(OwYk#U_G;T(@4XeuV%vbr}sSW z;dj`qZCRiX!IwJTC3twNR}$@liSR0(I>J-5(cpNg2 zZ*+=o-YEG5m*{c01Y?CLZIHcOIdAmvl3iAOHbH#0Sght@V6HKQG*x-z(WG8Ua z9!+QkZtM>u><8wx^(HeM1un)%Qt}lL`dNa8;XELZz|$@OzPs3iZXmaCGJWQ2;KNQ# z^b#SUJ-A}`3@ytpBl|ZS!(Iin1yS^=?|~08k^Bh+b_`<)@6=iT)~MsG&%Ft&T@u~a za#56QPv1&TD7}@`=59TETrNC7cc|$70 z7T`hhz-Weg;MG+)&kcb6NEXdb1v=7>k7C#k%)E4$k{!U-wjz36O+fQLZ8XD9U|H)o zG@)i-OaJkFhWCJdNrm$mb^(9%?WJxlgQ!fR2OI=u+LTC!c3{?{bo%o;2CV8c^)RA^QO^UrrBa$#pgUVsblW#jPb@)x}<+@g(lvFhXYiXPsh$VPTKJgrZnB-7<#nxg9L zqEx|2w!vG&lu6~O?4LtyNYe~zcimh@?U~emY&KE*HEQ=PsbVj{G@G>p+O`9C^4RGx zWkH&{`(LD^66PRnUTmIq5$*i;ul?vSMc)mAH>iio)RmF)TnU|erN$! zmnh3tn^3L{EoC&_L;-DqSYrue^B6P8NnGpubPmT3a?!^jYiR}JaBY=YGHKn3K5t U9zh8f=8edR=uf-R;#-&TKdp>sJ^%m!