NGP with contraction (#580)

This commit is contained in:
Matthew Tancik authored and GitHub committed 2022-09-29 13:18:49 -07:00
1 parent f0adbba91f
commit 5bc011e5fa
10 files changed
+181 -411

No files matched your search

+17 -32
View File
@@ -37,7 +37,6 @@ from nerfactory.datamanagers.dataparsers.mipnerf_parser import (
MipNerf360DataParserConfig,
)
from nerfactory.models.base import VanillaModelConfig
from nerfactory.models.compound import CompoundModelConfig
from nerfactory.models.instant_ngp import InstantNGPModelConfig
from nerfactory.models.mipnerf import MipNerfModel
from nerfactory.models.mipnerf_360 import MipNerf360Model
@@ -50,37 +49,6 @@ from nerfactory.pipelines.base import VanillaPipelineConfig
from nerfactory.pipelines.dynamic_batch import DynamicBatchPipelineConfig
base_configs: Dict[str, Config] = {}
base_configs["compound"] = Config(
method_name="compound",
trainer=TrainerConfig(mixed_precision=True),
pipeline=DynamicBatchPipelineConfig(
datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192),
model=CompoundModelConfig(eval_num_rays_per_chunk=8192),
),
optimizers={
"fields": {
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": None,
}
},
vis=["viewer"],
)
base_configs["instant-ngp"] = Config(
method_name="instant-ngp",
trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True),
pipeline=DynamicBatchPipelineConfig(
datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192),
model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192),
),
optimizers={
"fields": {
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": None,
}
},
vis=["viewer"],
)
base_configs["proposal"] = Config(
method_name="proposal",
@@ -105,6 +73,23 @@ base_configs["proposal"] = Config(
vis=["viewer"],
)
base_configs["instant-ngp"] = Config(
method_name="instant-ngp",
trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True),
pipeline=DynamicBatchPipelineConfig(
datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192),
model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192),
),
optimizers={
"fields": {
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": None,
}
},
viewer=ViewerConfig(num_rays_per_chunk=64000),
vis=["viewer"],
)
base_configs["mipnerf-360"] = Config(
method_name="mipnerf-360",
trainer=TrainerConfig(steps_per_eval_batch=200),
@@ -43,8 +43,8 @@ class NerfactoryDataParserConfig(DataParserConfig):
"""How much to scale the camera origins by."""
downscale_factor: int = 1
"""How much to downscale images."""
scene_scale: float = 4.0
"""How much to scale the scene."""
scene_scale: float = 1.0
"""How much to scale the region of interest by."""
orientation_method: Literal["pca", "up"] = "up"
"""The method to use for orientation."""
train_split_percentage: float = 0.9
+61 -50
View File
@@ -17,19 +17,18 @@ Instant-NGP field implementations using tiny-cuda-nn, torch, ....
"""
from typing import Optional, Tuple
from typing import Optional
import torch
from nerfacc import ContractionType, contract
from torch.nn.parameter import Parameter
from torchtyping import TensorType
from nerfactory.cameras.rays import RaySamples
from nerfactory.datamanagers.structs import SceneBounds
from nerfactory.fields.base import Field
from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
from nerfactory.fields.modules.embedding import Embedding
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.fields.modules.spatial_distortions import SpatialDistortion
from nerfactory.fields.nerf_field import NeRFField
from nerfactory.utils.activations import trunc_exp
try:
@@ -58,6 +57,10 @@ class TCNNInstantNGPField(Field):
geo_feat_dim: output geo feat dimensions
num_layers_color: number of hidden layers for color network
hidden_dim_color: dimension of hidden layers for color network
use_appearance_embedding: whether to use appearance embedding
num_images: number of images, requried if use_appearance_embedding is True
appearance_embedding_dim: dimension of appearance embedding
contraction_type: type of contraction
"""
def __init__(
@@ -68,13 +71,22 @@ class TCNNInstantNGPField(Field):
geo_feat_dim: int = 15,
num_layers_color: int = 3,
hidden_dim_color: int = 64,
spatial_distortion: Optional[SpatialDistortion] = None,
use_appearance_embedding: bool = False,
num_images: Optional[int] = None,
appearance_embedding_dim: int = 32,
contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE,
) -> None:
super().__init__()
self.aabb = Parameter(aabb, requires_grad=False)
self.geo_feat_dim = geo_feat_dim
self.spatial_distortion = spatial_distortion
self.contraction_type = contraction_type
self.use_appearance_embedding = use_appearance_embedding
if use_appearance_embedding:
assert num_images is not None
self.appearance_embedding_dim = appearance_embedding_dim
self.appearance_embedding = Embedding(num_images, appearance_embedding_dim)
# TODO: set this properly based on the aabb
per_level_scale = 1.4472692012786865
@@ -107,8 +119,11 @@ class TCNNInstantNGPField(Field):
},
)
in_dim = self.direction_encoding.n_output_dims + self.geo_feat_dim
if self.use_appearance_embedding:
in_dim += self.appearance_embedding_dim
self.mlp_head = tcnn.Network(
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim,
n_input_dims=in_dim,
n_output_dims=3,
network_config={
"otype": "FullyFusedMLP",
@@ -120,18 +135,10 @@ class TCNNInstantNGPField(Field):
)
def get_density(self, ray_samples: RaySamples):
if self.spatial_distortion is not None:
positions = ray_samples.frustums.get_positions()
positions = self.spatial_distortion(positions)
positions = (positions + 2.0) / 4.0
else:
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
positions = ray_samples.frustums.get_positions()
positions_flat = positions.view(-1, 3)
# assert all positions are in the range [0, 1]
# otherwise print min and max values
# assert torch.all(positions >= 0.0) and torch.all(
# positions <= 1.0
# ), f"positions: {positions.min()} {positions.max()}"
positions_flat = contract(x=positions_flat, roi=self.aabb, type=self.contraction_type)
h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1)
density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1)
@@ -142,49 +149,53 @@ class TCNNInstantNGPField(Field):
return density, base_mlp_out
def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None):
# TODO: add valid_mask masking!
directions = get_normalized_directions(ray_samples.frustums.directions)
directions_flat = directions.view(-1, 3)
# assert all directions are in the range [0, 1]
# assert torch.all(directions >= 0.0) and torch.all(
# directions <= 1.0
# ), f"directions: {directions.min()} {directions.max()}"
d = self.direction_encoding(directions_flat)
if density_embedding is None:
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
h = torch.cat([d, positions.view(-1, 3)], dim=-1)
else:
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
if self.use_appearance_embedding:
if ray_samples.camera_indices is None:
raise AttributeError("Camera indices are not provided.")
camera_indices = ray_samples.camera_indices.squeeze()
if self.training:
embedded_appearance = self.appearance_embedding(camera_indices)
else:
embedded_appearance = torch.zeros(
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
)
h = torch.cat([h, embedded_appearance.view(-1, self.appearance_embedding_dim)], dim=-1)
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
return {FieldHeadNames.RGB: rgb}
def get_opacity(self, positions: TensorType["bs":..., 3], step_size) -> TensorType["bs":..., 1]:
"""Returns the opacity for a position. Used primarily by the occupancy grid.
class TorchInstantNGPField(NeRFField):
"""
PyTorch implementation of the Instant-NGP field.
"""
Args:
positions: the positions to evaluate the opacity at.
step_size: the step size to use for the opacity evaluation.
"""
density = self.density_fn(positions)
## TODO: We should scale step size based on the distortion. Currently it uses too much memory.
# aabb_min, aabb_max = self.aabb[0], self.aabb[1]
# if self.contraction_type is not ContractionType.AABB:
# x = (positions - aabb_min) / (aabb_max - aabb_min)
# x = x * 2 - 1 # aabb is at [-1, 1]
# mag = x.norm(dim=-1, keepdim=True)
# mask = mag.squeeze(-1) > 1
def __init__(
self,
aabb,
position_encoding: Encoding = HashEncoding(),
direction_encoding: Encoding = SHEncoding(),
base_mlp_num_layers: int = 3,
base_mlp_layer_width: int = 64,
head_mlp_num_layers: int = 2,
head_mlp_layer_width: int = 32,
skip_connections: Tuple = (4,),
) -> None:
super().__init__(
position_encoding,
direction_encoding,
base_mlp_num_layers,
base_mlp_layer_width,
head_mlp_num_layers,
head_mlp_layer_width,
skip_connections,
)
self.aabb = Parameter(aabb, requires_grad=False)
# dev = (2 * mag - 1) / mag**2 + 2 * x**2 * (1 / mag**3 - (2 * mag - 1) / mag**4)
# dev[~mask] = 1.0
# dev = torch.clamp(dev, min=1e-6)
# step_size = step_size / dev.norm(dim=-1, keepdim=True)
# else:
# step_size = step_size * (aabb_max - aabb_min)
field_implementation_to_class = {"tcnn": TCNNInstantNGPField, "torch": TorchInstantNGPField}
opacity = density * step_size
return opacity
-244
View File
@@ -1,244 +0,0 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Compound model that integrates multiple NeRF Advancements
Features:
NGP Hash encoding for fast training
NGP ray sampler for fast training
Spatial contraction for training on large scenes
Appearance conditioning
TODO:
Pose optimization
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, List, Literal, Optional, Tuple, Type
import nerfacc
import torch
from torch.nn import Parameter
from torchmetrics import PeakSignalNoiseRatio
from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from nerfactory.cameras.rays import RayBundle
from nerfactory.fields.compound_field import field_implementation_to_class
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.models.base import Model, ModelConfig
from nerfactory.models.modules.ray_sampler import VolumetricSampler
from nerfactory.optimizers.loss import MSELoss
from nerfactory.renderers.renderers import (
AccumulationRenderer,
DepthRenderer,
RGBRenderer,
)
from nerfactory.utils import colors, visualization
from nerfactory.utils.callbacks import (
TrainingCallback,
TrainingCallbackAttributes,
TrainingCallbackLocation,
)
@dataclass
class CompoundModelConfig(ModelConfig):
"""Compound Model Config"""
_target: Type = field(
default_factory=lambda: CompoundModel
) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet
"""target class to instantiate"""
enable_collider: bool = False
"""Whether to create a scene collider to filter rays."""
collider_params: Optional[Dict[str, float]] = None
"""Instant NGP doesn't use a collider."""
field_implementation: Literal["torch", "tcnn"] = "tcnn"
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
num_samples: int = 1024
"""Number of max samples per ray"""
grid_resolution: int = 128
"""Resolution of the grid used for the field."""
cone_angle: float = 0.0
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
near_plane: float = 0.05
"""How far along ray to start sampling."""
randomize_background: bool = False
"""Whether to randomize the background color."""
class CompoundModel(Model):
"""Compound model
Args:
config: Compound model configuration to instantiate model
"""
config: CompoundModelConfig
def __init__(self, config: CompoundModelConfig, **kwargs) -> None:
assert config.field_implementation in field_implementation_to_class
self.field = None
super().__init__(config=config, **kwargs)
def populate_modules(self):
"""Set the fields and modules."""
super().populate_modules()
# torch or tiny-cuda-nn version
self.field = field_implementation_to_class[self.config.field_implementation](
aabb=self.scene_bounds.aabb, num_images=self.num_train_data
)
self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False)
# Sampler
self.sampler = VolumetricSampler(
aabb=self.scene_aabb,
density_fn=self.field.density_fn,
grid_resolution=self.config.grid_resolution,
num_samples=self.config.num_samples,
)
# renderers
background_color = None if self.config.randomize_background else colors.WHITE
self.renderer_rgb = RGBRenderer(background_color=background_color)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer()
# losses
self.rgb_loss = MSELoss()
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
# no colliders default
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
) -> List[TrainingCallback]:
return [
TrainingCallback(
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
update_every_num_iters=1,
func=self.sampler.occ_field.every_n_step,
)
]
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
if self.field is None:
raise ValueError("populate_fields() must be called before get_param_groups")
param_groups["fields"] = list(self.field.parameters())
return param_groups
def get_outputs(self, ray_bundle: RayBundle):
assert self.field is not None
num_rays = len(ray_bundle)
with torch.no_grad():
ray_samples, packed_info, ray_indices = self.sampler(
ray_bundle=ray_bundle,
near_plane=self.config.near_plane,
)
field_outputs = self.field.forward(ray_samples)
# accumulation
weights = nerfacc.volumetric_rendering_weights(
packed_info=packed_info,
sigmas=field_outputs[FieldHeadNames.DENSITY],
frustum_starts=ray_samples.frustums.starts,
frustum_ends=ray_samples.frustums.ends,
)
rgb = self.renderer_rgb(
rgb=field_outputs[FieldHeadNames.RGB],
weights=weights,
ray_indices=ray_indices,
num_rays=num_rays,
)
depth = self.renderer_depth(
weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays
)
accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays)
alive_ray_mask = accumulation.squeeze(-1) > 0
outputs = {
"rgb": rgb,
"accumulation": accumulation,
"depth": depth,
"alive_ray_mask": alive_ray_mask, # the rays we kept from sampler
"num_samples_per_ray": packed_info[:, 1],
}
return outputs
def get_metrics_dict(self, outputs, batch):
image = batch["image"].to(self.device)
metrics_dict = {}
metrics_dict["psnr"] = self.psnr(outputs["rgb"], image)
metrics_dict["num_samples_per_batch"] = outputs["num_samples_per_ray"].sum()
return metrics_dict
def get_loss_dict(self, outputs, batch, metrics_dict=None):
image = batch["image"].to(self.device)
mask = outputs["alive_ray_mask"]
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
loss_dict = {"rgb_loss": rgb_loss}
return loss_dict
def get_image_metrics_and_images(
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
image = batch["image"].to(self.device)
rgb = outputs["rgb"]
acc = visualization.apply_colormap(outputs["accumulation"])
depth = visualization.apply_depth_colormap(
outputs["depth"],
accumulation=outputs["accumulation"],
)
alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"])
combined_rgb = torch.cat([image, rgb], dim=1)
combined_acc = torch.cat([acc], dim=1)
combined_depth = torch.cat([depth], dim=1)
combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1)
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
image = torch.moveaxis(image, -1, 0)[None, ...]
rgb = torch.moveaxis(rgb, -1, 0)[None, ...]
psnr = self.psnr(image, rgb)
ssim = self.ssim(image, rgb)
lpips = self.lpips(image, rgb)
# all of these metrics will be logged as scalars
metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore
# TODO(ethan): return an image dictionary
images_dict = {
"img": combined_rgb,
"accumulation": combined_acc,
"depth": combined_depth,
"alive_ray_mask": combined_alive_ray_mask,
}
return metrics_dict, images_dict
+44 -18
View File
@@ -19,17 +19,18 @@ Implementation of Instant NGP.
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, List, Literal, Optional, Tuple, Type
from typing import Dict, List, Optional, Tuple, Type
import nerfacc
import torch
from nerfacc import ContractionType
from torch.nn import Parameter
from torchmetrics import PeakSignalNoiseRatio
from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from nerfactory.cameras.rays import RayBundle
from nerfactory.fields.instant_ngp_field import field_implementation_to_class
from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.models.base import Model, ModelConfig
from nerfactory.models.modules.ray_sampler import VolumetricSampler
@@ -59,17 +60,23 @@ class InstantNGPModelConfig(ModelConfig):
"""Whether to create a scene collider to filter rays."""
collider_params: Optional[Dict[str, float]] = None
"""Instant NGP doesn't use a collider."""
field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ...
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
max_num_samples_per_ray: int = 1024
max_num_samples_per_ray: int = 24
"""Number of samples in field evaluation."""
grid_resolution: int = 128
"""Resolution of the grid used for the field."""
cone_angle: float = 0.0
contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE
"""Resolution of the grid used for the field."""
cone_angle: float = 0.004
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
render_step_size: float = 0.01
"""Minimum step size for rendering."""
near_plane: float = 0.05
"""How far along ray to start sampling."""
randomize_background: bool = False
far_plane: float = 1e3
"""How far along ray to stop sampling."""
use_appearance_embedding: bool = False
"""Whether to use an appearance embedding."""
randomize_background: bool = True
"""Whether to randomize the background color."""
@@ -81,27 +88,37 @@ class NGPModel(Model):
"""
config: InstantNGPModelConfig
field: TCNNInstantNGPField
def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None:
assert config.field_implementation in field_implementation_to_class
self.field = None
super().__init__(config=config, **kwargs)
def populate_modules(self):
"""Set the fields and modules."""
super().populate_modules()
# torch or tiny-cuda-nn version
self.field = field_implementation_to_class[self.config.field_implementation](self.scene_bounds.aabb)
self.field = TCNNInstantNGPField(
aabb=self.scene_bounds.aabb,
contraction_type=self.config.contraction_type,
use_appearance_embedding=self.config.use_appearance_embedding,
num_images=self.num_train_data,
)
self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False)
# Occupancy Grid
self.occupancy_grid = nerfacc.OccupancyGrid(
roi_aabb=self.scene_aabb,
resolution=self.config.grid_resolution,
contraction_type=self.config.contraction_type,
)
# Sampler
vol_sampler_aabb = self.scene_bounds.aabb if self.config.contraction_type == ContractionType.AABB else None
self.sampler = VolumetricSampler(
aabb=self.scene_aabb,
scene_aabb=vol_sampler_aabb,
occupancy_grid=self.occupancy_grid,
density_fn=self.field.density_fn,
grid_resolution=self.config.grid_resolution,
num_samples=self.config.max_num_samples_per_ray,
)
# renderers
@@ -121,11 +138,17 @@ class NGPModel(Model):
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
) -> List[TrainingCallback]:
def update_occupancy_grid(step: int):
self.occupancy_grid.every_n_step(
step=step,
occ_eval_fn=lambda x: self.field.get_opacity(x, self.config.render_step_size),
)
return [
TrainingCallback(
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
update_every_num_iters=1,
func=self.sampler.occ_field.every_n_step,
func=update_occupancy_grid,
),
]
@@ -144,16 +167,19 @@ class NGPModel(Model):
ray_samples, packed_info, ray_indices = self.sampler(
ray_bundle=ray_bundle,
near_plane=self.config.near_plane,
far_plane=self.config.far_plane,
render_step_size=self.config.render_step_size,
cone_angle=self.config.cone_angle,
)
field_outputs = self.field(ray_samples)
# accumulation
weights = nerfacc.volumetric_rendering_weights(
weights = nerfacc.render_weight_from_density(
packed_info=packed_info,
sigmas=field_outputs[FieldHeadNames.DENSITY],
frustum_starts=ray_samples.frustums.starts,
frustum_ends=ray_samples.frustums.ends,
t_starts=ray_samples.frustums.starts,
t_ends=ray_samples.frustums.ends,
)
rgb = self.renderer_rgb(
+47 -57
View File
@@ -16,12 +16,12 @@
Collection of sampling strategies
"""
import math
from abc import abstractmethod
from typing import Callable, List, Optional, Tuple
import nerfacc
import torch
from nerfacc import OccupancyGrid
from torch import nn
from torchtyping import TensorType
@@ -398,46 +398,52 @@ class PDFSampler(Sampler):
class VolumetricSampler(Sampler):
"""Sampler inspired by the one proposed in the Instant-NGP paper.
Generates samples along a ray by sampling the occupancy field.
Optionally removes occluded samples if the density_fn is provided.
This sampler does ray-box AABB test, ray samples generation and density check, all together.
Args:
aabb: Bounding box of the scene.
density_fn: Function that takes a tensor of points and returns a tensor of densities.
grid_resolution: Resolution of the density grid.
num_samples: Number of samples per ray.
Args:
occupancy_grid: Occupancy grid to sample from.
density_fn: Function that evaluates density at a given point.
scene_aabb: Axis-aligned bounding box of the scene, should be set to None if the scene is unbounded.
"""
def __init__(
self,
aabb: TensorType[2, 3],
density_fn: Callable[[TensorType["N", 3]], TensorType["N"]],
grid_resolution: int = 128,
num_samples: int = 1024,
occupancy_grid: Optional[OccupancyGrid] = None,
density_fn: Optional[Callable[[TensorType[..., 3]], TensorType[..., 1]]] = None,
scene_aabb: Optional[TensorType[2, 3]] = None,
) -> None:
super().__init__(num_samples=num_samples)
self.aabb = aabb
super().__init__()
self.scene_aabb = scene_aabb
self.density_fn = density_fn
self.occupancy_grid = occupancy_grid
if self.scene_aabb is not None:
self.scene_aabb = self.scene_aabb.to("cuda").flatten()
print(self.scene_aabb)
self.render_step_size = ((self.aabb[3:] - self.aabb[:3]).max() * math.sqrt(3) / num_samples).item()
def get_sigma_fn(self, origins, directions) -> Optional[Callable]:
"""Returns a function that returns the density of a point.
# setup occupancy field with eval function
def occ_eval_fn(x: torch.Tensor) -> torch.Tensor:
"""Evaluate occupancy given positions.
Args:
origins: Origins of rays
directions: Directions of rays
Returns:
Function that returns the density of a point or None if a density function is not provided.
"""
Args:
x: positions with shape (N, 3).
Returns:
occupancy values with shape (N, 1).
"""
density_after_activation = density_fn(x)
# those two are similar when density is small.
# occupancy = 1.0 - torch.exp(-density_after_activation * render_step_size)
occupancy = density_after_activation * self.render_step_size
return occupancy
if self.density_fn is None or not self.training:
return None
self.occ_field = nerfacc.OccupancyField(occ_eval_fn=occ_eval_fn, aabb=aabb, resolution=grid_resolution)
density_fn = self.density_fn
def sigma_fn(t_starts, t_ends, ray_indices):
t_origins = origins[ray_indices]
t_dirs = directions[ray_indices]
positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0
return density_fn(positions)
return sigma_fn
def generate_ray_samples(self) -> RaySamples:
raise RuntimeError(
@@ -448,17 +454,19 @@ class VolumetricSampler(Sampler):
def forward(
self,
ray_bundle: RayBundle,
num_samples: Optional[int] = None,
render_step_size: float,
near_plane: float = 0.0,
remove_occluded_samples: bool = True,
far_plane: Optional[float] = None,
cone_angle: float = 0.0,
) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples", 2]]:
"""Generate ray samples in a bounding box.
Args:
ray_bundle: Rays to generate samples for
num_samples: Number of samples per ray
render_step_size: Minimum step size to use for rendering
near_plane: Near plane for raymarching
remove_occluded_samples: Whether to remove occluded samples. (requires evaluating the density function)
far_plane: Far plane for raymarching
cone_angle: Cone angle for raymarching, set to 0 for uniform marching.
Returns:
a tuple of (ray_samples, packed_info, ray_indices)
@@ -470,12 +478,6 @@ class VolumetricSampler(Sampler):
if self.density_field is not None:
raise ValueError("NGPSpacedSampler does not support a density field")
if num_samples is not None:
render_step_size = ((self.aabb[3:] - self.aabb[:3]).max() * math.sqrt(3) / num_samples).item()
else:
num_samples = self.num_samples
render_step_size = self.render_step_size
rays_o = ray_bundle.origins.contiguous()
rays_d = ray_bundle.directions.contiguous()
if ray_bundle.camera_indices is not None:
@@ -483,15 +485,17 @@ class VolumetricSampler(Sampler):
else:
camera_indices = None
packed_info, starts, ends = nerfacc.volumetric_marching(
packed_info, starts, ends = nerfacc.ray_marching(
rays_o=rays_o,
rays_d=rays_d,
aabb=self.aabb,
scene_resolution=self.occ_field.resolution,
scene_occ_binary=self.occ_field.occ_grid_binary,
scene_aabb=self.scene_aabb,
grid=self.occupancy_grid,
sigma_fn=self.get_sigma_fn(rays_o, rays_d),
render_step_size=render_step_size,
near_plane=near_plane,
far_plane=far_plane,
stratified=self.training,
cone_angle=cone_angle,
)
num_samples = starts.shape[0]
@@ -508,20 +512,6 @@ class VolumetricSampler(Sampler):
if camera_indices is not None:
camera_indices = camera_indices[ray_indices]
if remove_occluded_samples and self.training:
with torch.no_grad():
positions = origins + dirs * (starts + ends) / 2.0
densities = self.density_fn(positions)
if camera_indices is not None:
packed_info, starts, ends, origins, dirs, camera_indices = nerfacc.volumetric_rendering_steps(
packed_info, densities, starts, ends, origins, dirs, camera_indices
)
else:
packed_info, starts, ends, origins, dirs = nerfacc.volumetric_rendering_steps(
packed_info, densities, starts, ends, origins, dirs
)
ray_indices = nerfacc.unpack_to_ray_indices(packed_info).long()
zeros = torch.zeros_like(origins[:, :1])
ray_samples = RaySamples(
frustums=Frustums(
+1 -2
View File
@@ -31,7 +31,6 @@ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from nerfactory.cameras.rays import RayBundle
from nerfactory.fields.compound_field import TCNNCompoundField
from nerfactory.fields.density_field import DensityField
from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.fields.modules.spatial_distortions import SceneContraction
from nerfactory.models.base import Model, ModelConfig
@@ -106,7 +105,7 @@ class ProposalModel(Model):
self.scene_bounds.aabb, spatial_distortion=scene_contraction, num_images=self.num_train_data
)
else:
self.field = TCNNInstantNGPField(self.scene_bounds.aabb, spatial_distortion=scene_contraction)
raise NotImplementedError("Only appearance conditioning is supported.")
# Build the proposal network(s)
self.proposal_networks = torch.nn.ModuleList()
+7 -4
View File
@@ -71,8 +71,9 @@ class RGBRenderer(nn.Module):
Outputs rgb values.
"""
if ray_indices is not None and num_rays is not None:
rgb = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, rgb, num_rays)
accumulated_weight = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, None, num_rays)
# Necessary for packed samples from volumetric ray sampler
rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays)
accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
else:
rgb = torch.sum(weights * rgb, dim=-2)
accumulated_weight = torch.sum(weights, dim=-2)
@@ -181,7 +182,8 @@ class AccumulationRenderer(nn.Module):
"""
if ray_indices is not None and num_rays is not None:
accumulation = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, None, num_rays)
# Necessary for packed samples from volumetric ray sampler
accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
else:
accumulation = torch.sum(weights, dim=-2)
return accumulation
@@ -222,7 +224,8 @@ class DepthRenderer(nn.Module):
steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2
if ray_indices is not None and num_rays is not None:
depth = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, steps, num_rays)
# Necessary for packed samples from volumetric ray sampler
depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)
else:
depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)