mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-22 15:45:39 +08:00
NGP with contraction (#580)
This commit is contained in:
1 parent
f0adbba91f
commit
5bc011e5fa
10 files changed
+181
-411
No files matched your search
@@ -37,7 +37,6 @@ from nerfactory.datamanagers.dataparsers.mipnerf_parser import (
|
||||
MipNerf360DataParserConfig,
|
||||
)
|
||||
from nerfactory.models.base import VanillaModelConfig
|
||||
from nerfactory.models.compound import CompoundModelConfig
|
||||
from nerfactory.models.instant_ngp import InstantNGPModelConfig
|
||||
from nerfactory.models.mipnerf import MipNerfModel
|
||||
from nerfactory.models.mipnerf_360 import MipNerf360Model
|
||||
@@ -50,37 +49,6 @@ from nerfactory.pipelines.base import VanillaPipelineConfig
|
||||
from nerfactory.pipelines.dynamic_batch import DynamicBatchPipelineConfig
|
||||
|
||||
base_configs: Dict[str, Config] = {}
|
||||
base_configs["compound"] = Config(
|
||||
method_name="compound",
|
||||
trainer=TrainerConfig(mixed_precision=True),
|
||||
pipeline=DynamicBatchPipelineConfig(
|
||||
datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192),
|
||||
model=CompoundModelConfig(eval_num_rays_per_chunk=8192),
|
||||
),
|
||||
optimizers={
|
||||
"fields": {
|
||||
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
|
||||
"scheduler": None,
|
||||
}
|
||||
},
|
||||
vis=["viewer"],
|
||||
)
|
||||
|
||||
base_configs["instant-ngp"] = Config(
|
||||
method_name="instant-ngp",
|
||||
trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True),
|
||||
pipeline=DynamicBatchPipelineConfig(
|
||||
datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192),
|
||||
model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192),
|
||||
),
|
||||
optimizers={
|
||||
"fields": {
|
||||
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
|
||||
"scheduler": None,
|
||||
}
|
||||
},
|
||||
vis=["viewer"],
|
||||
)
|
||||
|
||||
base_configs["proposal"] = Config(
|
||||
method_name="proposal",
|
||||
@@ -105,6 +73,23 @@ base_configs["proposal"] = Config(
|
||||
vis=["viewer"],
|
||||
)
|
||||
|
||||
base_configs["instant-ngp"] = Config(
|
||||
method_name="instant-ngp",
|
||||
trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True),
|
||||
pipeline=DynamicBatchPipelineConfig(
|
||||
datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192),
|
||||
model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192),
|
||||
),
|
||||
optimizers={
|
||||
"fields": {
|
||||
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
|
||||
"scheduler": None,
|
||||
}
|
||||
},
|
||||
viewer=ViewerConfig(num_rays_per_chunk=64000),
|
||||
vis=["viewer"],
|
||||
)
|
||||
|
||||
base_configs["mipnerf-360"] = Config(
|
||||
method_name="mipnerf-360",
|
||||
trainer=TrainerConfig(steps_per_eval_batch=200),
|
||||
|
||||
@@ -43,8 +43,8 @@ class NerfactoryDataParserConfig(DataParserConfig):
|
||||
"""How much to scale the camera origins by."""
|
||||
downscale_factor: int = 1
|
||||
"""How much to downscale images."""
|
||||
scene_scale: float = 4.0
|
||||
"""How much to scale the scene."""
|
||||
scene_scale: float = 1.0
|
||||
"""How much to scale the region of interest by."""
|
||||
orientation_method: Literal["pca", "up"] = "up"
|
||||
"""The method to use for orientation."""
|
||||
train_split_percentage: float = 0.9
|
||||
|
||||
@@ -17,19 +17,18 @@ Instant-NGP field implementations using tiny-cuda-nn, torch, ....
|
||||
"""
|
||||
|
||||
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from nerfacc import ContractionType, contract
|
||||
from torch.nn.parameter import Parameter
|
||||
from torchtyping import TensorType
|
||||
|
||||
from nerfactory.cameras.rays import RaySamples
|
||||
from nerfactory.datamanagers.structs import SceneBounds
|
||||
from nerfactory.fields.base import Field
|
||||
from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
|
||||
from nerfactory.fields.modules.embedding import Embedding
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.fields.modules.spatial_distortions import SpatialDistortion
|
||||
from nerfactory.fields.nerf_field import NeRFField
|
||||
from nerfactory.utils.activations import trunc_exp
|
||||
|
||||
try:
|
||||
@@ -58,6 +57,10 @@ class TCNNInstantNGPField(Field):
|
||||
geo_feat_dim: output geo feat dimensions
|
||||
num_layers_color: number of hidden layers for color network
|
||||
hidden_dim_color: dimension of hidden layers for color network
|
||||
use_appearance_embedding: whether to use appearance embedding
|
||||
num_images: number of images, requried if use_appearance_embedding is True
|
||||
appearance_embedding_dim: dimension of appearance embedding
|
||||
contraction_type: type of contraction
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -68,13 +71,22 @@ class TCNNInstantNGPField(Field):
|
||||
geo_feat_dim: int = 15,
|
||||
num_layers_color: int = 3,
|
||||
hidden_dim_color: int = 64,
|
||||
spatial_distortion: Optional[SpatialDistortion] = None,
|
||||
use_appearance_embedding: bool = False,
|
||||
num_images: Optional[int] = None,
|
||||
appearance_embedding_dim: int = 32,
|
||||
contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
self.geo_feat_dim = geo_feat_dim
|
||||
self.spatial_distortion = spatial_distortion
|
||||
self.contraction_type = contraction_type
|
||||
|
||||
self.use_appearance_embedding = use_appearance_embedding
|
||||
if use_appearance_embedding:
|
||||
assert num_images is not None
|
||||
self.appearance_embedding_dim = appearance_embedding_dim
|
||||
self.appearance_embedding = Embedding(num_images, appearance_embedding_dim)
|
||||
|
||||
# TODO: set this properly based on the aabb
|
||||
per_level_scale = 1.4472692012786865
|
||||
@@ -107,8 +119,11 @@ class TCNNInstantNGPField(Field):
|
||||
},
|
||||
)
|
||||
|
||||
in_dim = self.direction_encoding.n_output_dims + self.geo_feat_dim
|
||||
if self.use_appearance_embedding:
|
||||
in_dim += self.appearance_embedding_dim
|
||||
self.mlp_head = tcnn.Network(
|
||||
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim,
|
||||
n_input_dims=in_dim,
|
||||
n_output_dims=3,
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
@@ -120,18 +135,10 @@ class TCNNInstantNGPField(Field):
|
||||
)
|
||||
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
if self.spatial_distortion is not None:
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
positions = self.spatial_distortion(positions)
|
||||
positions = (positions + 2.0) / 4.0
|
||||
else:
|
||||
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
positions_flat = positions.view(-1, 3)
|
||||
# assert all positions are in the range [0, 1]
|
||||
# otherwise print min and max values
|
||||
# assert torch.all(positions >= 0.0) and torch.all(
|
||||
# positions <= 1.0
|
||||
# ), f"positions: {positions.min()} {positions.max()}"
|
||||
positions_flat = contract(x=positions_flat, roi=self.aabb, type=self.contraction_type)
|
||||
|
||||
h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1)
|
||||
density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1)
|
||||
|
||||
@@ -142,49 +149,53 @@ class TCNNInstantNGPField(Field):
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None):
|
||||
# TODO: add valid_mask masking!
|
||||
directions = get_normalized_directions(ray_samples.frustums.directions)
|
||||
directions_flat = directions.view(-1, 3)
|
||||
# assert all directions are in the range [0, 1]
|
||||
# assert torch.all(directions >= 0.0) and torch.all(
|
||||
# directions <= 1.0
|
||||
# ), f"directions: {directions.min()} {directions.max()}"
|
||||
|
||||
d = self.direction_encoding(directions_flat)
|
||||
if density_embedding is None:
|
||||
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
h = torch.cat([d, positions.view(-1, 3)], dim=-1)
|
||||
else:
|
||||
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
|
||||
|
||||
if self.use_appearance_embedding:
|
||||
if ray_samples.camera_indices is None:
|
||||
raise AttributeError("Camera indices are not provided.")
|
||||
camera_indices = ray_samples.camera_indices.squeeze()
|
||||
if self.training:
|
||||
embedded_appearance = self.appearance_embedding(camera_indices)
|
||||
else:
|
||||
embedded_appearance = torch.zeros(
|
||||
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
|
||||
)
|
||||
h = torch.cat([h, embedded_appearance.view(-1, self.appearance_embedding_dim)], dim=-1)
|
||||
|
||||
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
|
||||
return {FieldHeadNames.RGB: rgb}
|
||||
|
||||
def get_opacity(self, positions: TensorType["bs":..., 3], step_size) -> TensorType["bs":..., 1]:
|
||||
"""Returns the opacity for a position. Used primarily by the occupancy grid.
|
||||
|
||||
class TorchInstantNGPField(NeRFField):
|
||||
"""
|
||||
PyTorch implementation of the Instant-NGP field.
|
||||
"""
|
||||
Args:
|
||||
positions: the positions to evaluate the opacity at.
|
||||
step_size: the step size to use for the opacity evaluation.
|
||||
"""
|
||||
density = self.density_fn(positions)
|
||||
## TODO: We should scale step size based on the distortion. Currently it uses too much memory.
|
||||
# aabb_min, aabb_max = self.aabb[0], self.aabb[1]
|
||||
# if self.contraction_type is not ContractionType.AABB:
|
||||
# x = (positions - aabb_min) / (aabb_max - aabb_min)
|
||||
# x = x * 2 - 1 # aabb is at [-1, 1]
|
||||
# mag = x.norm(dim=-1, keepdim=True)
|
||||
# mask = mag.squeeze(-1) > 1
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aabb,
|
||||
position_encoding: Encoding = HashEncoding(),
|
||||
direction_encoding: Encoding = SHEncoding(),
|
||||
base_mlp_num_layers: int = 3,
|
||||
base_mlp_layer_width: int = 64,
|
||||
head_mlp_num_layers: int = 2,
|
||||
head_mlp_layer_width: int = 32,
|
||||
skip_connections: Tuple = (4,),
|
||||
) -> None:
|
||||
super().__init__(
|
||||
position_encoding,
|
||||
direction_encoding,
|
||||
base_mlp_num_layers,
|
||||
base_mlp_layer_width,
|
||||
head_mlp_num_layers,
|
||||
head_mlp_layer_width,
|
||||
skip_connections,
|
||||
)
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
# dev = (2 * mag - 1) / mag**2 + 2 * x**2 * (1 / mag**3 - (2 * mag - 1) / mag**4)
|
||||
# dev[~mask] = 1.0
|
||||
# dev = torch.clamp(dev, min=1e-6)
|
||||
# step_size = step_size / dev.norm(dim=-1, keepdim=True)
|
||||
# else:
|
||||
# step_size = step_size * (aabb_max - aabb_min)
|
||||
|
||||
|
||||
field_implementation_to_class = {"tcnn": TCNNInstantNGPField, "torch": TorchInstantNGPField}
|
||||
opacity = density * step_size
|
||||
return opacity
|
||||
@@ -1,244 +0,0 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
""" Compound model that integrates multiple NeRF Advancements
|
||||
|
||||
Features:
|
||||
|
||||
NGP Hash encoding for fast training
|
||||
NGP ray sampler for fast training
|
||||
Spatial contraction for training on large scenes
|
||||
Appearance conditioning
|
||||
|
||||
TODO:
|
||||
Pose optimization
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Literal, Optional, Tuple, Type
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
from torchmetrics import PeakSignalNoiseRatio
|
||||
from torchmetrics.functional import structural_similarity_index_measure
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
|
||||
from nerfactory.cameras.rays import RayBundle
|
||||
from nerfactory.fields.compound_field import field_implementation_to_class
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.models.base import Model, ModelConfig
|
||||
from nerfactory.models.modules.ray_sampler import VolumetricSampler
|
||||
from nerfactory.optimizers.loss import MSELoss
|
||||
from nerfactory.renderers.renderers import (
|
||||
AccumulationRenderer,
|
||||
DepthRenderer,
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfactory.utils import colors, visualization
|
||||
from nerfactory.utils.callbacks import (
|
||||
TrainingCallback,
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompoundModelConfig(ModelConfig):
|
||||
"""Compound Model Config"""
|
||||
|
||||
_target: Type = field(
|
||||
default_factory=lambda: CompoundModel
|
||||
) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet
|
||||
"""target class to instantiate"""
|
||||
enable_collider: bool = False
|
||||
"""Whether to create a scene collider to filter rays."""
|
||||
collider_params: Optional[Dict[str, float]] = None
|
||||
"""Instant NGP doesn't use a collider."""
|
||||
field_implementation: Literal["torch", "tcnn"] = "tcnn"
|
||||
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
|
||||
num_samples: int = 1024
|
||||
"""Number of max samples per ray"""
|
||||
grid_resolution: int = 128
|
||||
"""Resolution of the grid used for the field."""
|
||||
cone_angle: float = 0.0
|
||||
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
|
||||
near_plane: float = 0.05
|
||||
"""How far along ray to start sampling."""
|
||||
randomize_background: bool = False
|
||||
"""Whether to randomize the background color."""
|
||||
|
||||
|
||||
class CompoundModel(Model):
|
||||
"""Compound model
|
||||
|
||||
Args:
|
||||
config: Compound model configuration to instantiate model
|
||||
"""
|
||||
|
||||
config: CompoundModelConfig
|
||||
|
||||
def __init__(self, config: CompoundModelConfig, **kwargs) -> None:
|
||||
assert config.field_implementation in field_implementation_to_class
|
||||
self.field = None
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules."""
|
||||
super().populate_modules()
|
||||
# torch or tiny-cuda-nn version
|
||||
self.field = field_implementation_to_class[self.config.field_implementation](
|
||||
aabb=self.scene_bounds.aabb, num_images=self.num_train_data
|
||||
)
|
||||
|
||||
self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False)
|
||||
|
||||
# Sampler
|
||||
self.sampler = VolumetricSampler(
|
||||
aabb=self.scene_aabb,
|
||||
density_fn=self.field.density_fn,
|
||||
grid_resolution=self.config.grid_resolution,
|
||||
num_samples=self.config.num_samples,
|
||||
)
|
||||
|
||||
# renderers
|
||||
background_color = None if self.config.randomize_background else colors.WHITE
|
||||
self.renderer_rgb = RGBRenderer(background_color=background_color)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer()
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
|
||||
# no colliders default
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
return [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=self.sampler.occ_field.every_n_step,
|
||||
)
|
||||
]
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
if self.field is None:
|
||||
raise ValueError("populate_fields() must be called before get_param_groups")
|
||||
param_groups["fields"] = list(self.field.parameters())
|
||||
return param_groups
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
assert self.field is not None
|
||||
num_rays = len(ray_bundle)
|
||||
|
||||
with torch.no_grad():
|
||||
ray_samples, packed_info, ray_indices = self.sampler(
|
||||
ray_bundle=ray_bundle,
|
||||
near_plane=self.config.near_plane,
|
||||
)
|
||||
|
||||
field_outputs = self.field.forward(ray_samples)
|
||||
|
||||
# accumulation
|
||||
weights = nerfacc.volumetric_rendering_weights(
|
||||
packed_info=packed_info,
|
||||
sigmas=field_outputs[FieldHeadNames.DENSITY],
|
||||
frustum_starts=ray_samples.frustums.starts,
|
||||
frustum_ends=ray_samples.frustums.ends,
|
||||
)
|
||||
|
||||
rgb = self.renderer_rgb(
|
||||
rgb=field_outputs[FieldHeadNames.RGB],
|
||||
weights=weights,
|
||||
ray_indices=ray_indices,
|
||||
num_rays=num_rays,
|
||||
)
|
||||
depth = self.renderer_depth(
|
||||
weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays
|
||||
)
|
||||
accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays)
|
||||
alive_ray_mask = accumulation.squeeze(-1) > 0
|
||||
|
||||
outputs = {
|
||||
"rgb": rgb,
|
||||
"accumulation": accumulation,
|
||||
"depth": depth,
|
||||
"alive_ray_mask": alive_ray_mask, # the rays we kept from sampler
|
||||
"num_samples_per_ray": packed_info[:, 1],
|
||||
}
|
||||
return outputs
|
||||
|
||||
def get_metrics_dict(self, outputs, batch):
|
||||
image = batch["image"].to(self.device)
|
||||
metrics_dict = {}
|
||||
metrics_dict["psnr"] = self.psnr(outputs["rgb"], image)
|
||||
metrics_dict["num_samples_per_batch"] = outputs["num_samples_per_ray"].sum()
|
||||
return metrics_dict
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None):
|
||||
image = batch["image"].to(self.device)
|
||||
mask = outputs["alive_ray_mask"]
|
||||
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
|
||||
loss_dict = {"rgb_loss": rgb_loss}
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
|
||||
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
|
||||
|
||||
image = batch["image"].to(self.device)
|
||||
rgb = outputs["rgb"]
|
||||
acc = visualization.apply_colormap(outputs["accumulation"])
|
||||
depth = visualization.apply_depth_colormap(
|
||||
outputs["depth"],
|
||||
accumulation=outputs["accumulation"],
|
||||
)
|
||||
alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"])
|
||||
|
||||
combined_rgb = torch.cat([image, rgb], dim=1)
|
||||
combined_acc = torch.cat([acc], dim=1)
|
||||
combined_depth = torch.cat([depth], dim=1)
|
||||
combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1)
|
||||
|
||||
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
|
||||
image = torch.moveaxis(image, -1, 0)[None, ...]
|
||||
rgb = torch.moveaxis(rgb, -1, 0)[None, ...]
|
||||
|
||||
psnr = self.psnr(image, rgb)
|
||||
ssim = self.ssim(image, rgb)
|
||||
lpips = self.lpips(image, rgb)
|
||||
|
||||
# all of these metrics will be logged as scalars
|
||||
metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore
|
||||
# TODO(ethan): return an image dictionary
|
||||
|
||||
images_dict = {
|
||||
"img": combined_rgb,
|
||||
"accumulation": combined_acc,
|
||||
"depth": combined_depth,
|
||||
"alive_ray_mask": combined_alive_ray_mask,
|
||||
}
|
||||
|
||||
return metrics_dict, images_dict
|
||||
@@ -19,17 +19,18 @@ Implementation of Instant NGP.
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Literal, Optional, Tuple, Type
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
from nerfacc import ContractionType
|
||||
from torch.nn import Parameter
|
||||
from torchmetrics import PeakSignalNoiseRatio
|
||||
from torchmetrics.functional import structural_similarity_index_measure
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
|
||||
from nerfactory.cameras.rays import RayBundle
|
||||
from nerfactory.fields.instant_ngp_field import field_implementation_to_class
|
||||
from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.models.base import Model, ModelConfig
|
||||
from nerfactory.models.modules.ray_sampler import VolumetricSampler
|
||||
@@ -59,17 +60,23 @@ class InstantNGPModelConfig(ModelConfig):
|
||||
"""Whether to create a scene collider to filter rays."""
|
||||
collider_params: Optional[Dict[str, float]] = None
|
||||
"""Instant NGP doesn't use a collider."""
|
||||
field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ...
|
||||
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
|
||||
max_num_samples_per_ray: int = 1024
|
||||
max_num_samples_per_ray: int = 24
|
||||
"""Number of samples in field evaluation."""
|
||||
grid_resolution: int = 128
|
||||
"""Resolution of the grid used for the field."""
|
||||
cone_angle: float = 0.0
|
||||
contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE
|
||||
"""Resolution of the grid used for the field."""
|
||||
cone_angle: float = 0.004
|
||||
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
|
||||
render_step_size: float = 0.01
|
||||
"""Minimum step size for rendering."""
|
||||
near_plane: float = 0.05
|
||||
"""How far along ray to start sampling."""
|
||||
randomize_background: bool = False
|
||||
far_plane: float = 1e3
|
||||
"""How far along ray to stop sampling."""
|
||||
use_appearance_embedding: bool = False
|
||||
"""Whether to use an appearance embedding."""
|
||||
randomize_background: bool = True
|
||||
"""Whether to randomize the background color."""
|
||||
|
||||
|
||||
@@ -81,27 +88,37 @@ class NGPModel(Model):
|
||||
"""
|
||||
|
||||
config: InstantNGPModelConfig
|
||||
field: TCNNInstantNGPField
|
||||
|
||||
def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None:
|
||||
assert config.field_implementation in field_implementation_to_class
|
||||
self.field = None
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules."""
|
||||
super().populate_modules()
|
||||
|
||||
# torch or tiny-cuda-nn version
|
||||
self.field = field_implementation_to_class[self.config.field_implementation](self.scene_bounds.aabb)
|
||||
self.field = TCNNInstantNGPField(
|
||||
aabb=self.scene_bounds.aabb,
|
||||
contraction_type=self.config.contraction_type,
|
||||
use_appearance_embedding=self.config.use_appearance_embedding,
|
||||
num_images=self.num_train_data,
|
||||
)
|
||||
|
||||
self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False)
|
||||
|
||||
# Occupancy Grid
|
||||
self.occupancy_grid = nerfacc.OccupancyGrid(
|
||||
roi_aabb=self.scene_aabb,
|
||||
resolution=self.config.grid_resolution,
|
||||
contraction_type=self.config.contraction_type,
|
||||
)
|
||||
|
||||
# Sampler
|
||||
vol_sampler_aabb = self.scene_bounds.aabb if self.config.contraction_type == ContractionType.AABB else None
|
||||
self.sampler = VolumetricSampler(
|
||||
aabb=self.scene_aabb,
|
||||
scene_aabb=vol_sampler_aabb,
|
||||
occupancy_grid=self.occupancy_grid,
|
||||
density_fn=self.field.density_fn,
|
||||
grid_resolution=self.config.grid_resolution,
|
||||
num_samples=self.config.max_num_samples_per_ray,
|
||||
)
|
||||
|
||||
# renderers
|
||||
@@ -121,11 +138,17 @@ class NGPModel(Model):
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
def update_occupancy_grid(step: int):
|
||||
self.occupancy_grid.every_n_step(
|
||||
step=step,
|
||||
occ_eval_fn=lambda x: self.field.get_opacity(x, self.config.render_step_size),
|
||||
)
|
||||
|
||||
return [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=self.sampler.occ_field.every_n_step,
|
||||
func=update_occupancy_grid,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -144,16 +167,19 @@ class NGPModel(Model):
|
||||
ray_samples, packed_info, ray_indices = self.sampler(
|
||||
ray_bundle=ray_bundle,
|
||||
near_plane=self.config.near_plane,
|
||||
far_plane=self.config.far_plane,
|
||||
render_step_size=self.config.render_step_size,
|
||||
cone_angle=self.config.cone_angle,
|
||||
)
|
||||
|
||||
field_outputs = self.field(ray_samples)
|
||||
|
||||
# accumulation
|
||||
weights = nerfacc.volumetric_rendering_weights(
|
||||
weights = nerfacc.render_weight_from_density(
|
||||
packed_info=packed_info,
|
||||
sigmas=field_outputs[FieldHeadNames.DENSITY],
|
||||
frustum_starts=ray_samples.frustums.starts,
|
||||
frustum_ends=ray_samples.frustums.ends,
|
||||
t_starts=ray_samples.frustums.starts,
|
||||
t_ends=ray_samples.frustums.ends,
|
||||
)
|
||||
|
||||
rgb = self.renderer_rgb(
|
||||
|
||||
@@ -16,12 +16,12 @@
|
||||
Collection of sampling strategies
|
||||
"""
|
||||
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from typing import Callable, List, Optional, Tuple
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
from nerfacc import OccupancyGrid
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
@@ -398,46 +398,52 @@ class PDFSampler(Sampler):
|
||||
|
||||
class VolumetricSampler(Sampler):
|
||||
"""Sampler inspired by the one proposed in the Instant-NGP paper.
|
||||
Generates samples along a ray by sampling the occupancy field.
|
||||
Optionally removes occluded samples if the density_fn is provided.
|
||||
|
||||
This sampler does ray-box AABB test, ray samples generation and density check, all together.
|
||||
|
||||
Args:
|
||||
aabb: Bounding box of the scene.
|
||||
density_fn: Function that takes a tensor of points and returns a tensor of densities.
|
||||
grid_resolution: Resolution of the density grid.
|
||||
num_samples: Number of samples per ray.
|
||||
Args:
|
||||
occupancy_grid: Occupancy grid to sample from.
|
||||
density_fn: Function that evaluates density at a given point.
|
||||
scene_aabb: Axis-aligned bounding box of the scene, should be set to None if the scene is unbounded.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aabb: TensorType[2, 3],
|
||||
density_fn: Callable[[TensorType["N", 3]], TensorType["N"]],
|
||||
grid_resolution: int = 128,
|
||||
num_samples: int = 1024,
|
||||
occupancy_grid: Optional[OccupancyGrid] = None,
|
||||
density_fn: Optional[Callable[[TensorType[..., 3]], TensorType[..., 1]]] = None,
|
||||
scene_aabb: Optional[TensorType[2, 3]] = None,
|
||||
) -> None:
|
||||
|
||||
super().__init__(num_samples=num_samples)
|
||||
self.aabb = aabb
|
||||
super().__init__()
|
||||
self.scene_aabb = scene_aabb
|
||||
self.density_fn = density_fn
|
||||
self.occupancy_grid = occupancy_grid
|
||||
if self.scene_aabb is not None:
|
||||
self.scene_aabb = self.scene_aabb.to("cuda").flatten()
|
||||
print(self.scene_aabb)
|
||||
|
||||
self.render_step_size = ((self.aabb[3:] - self.aabb[:3]).max() * math.sqrt(3) / num_samples).item()
|
||||
def get_sigma_fn(self, origins, directions) -> Optional[Callable]:
|
||||
"""Returns a function that returns the density of a point.
|
||||
|
||||
# setup occupancy field with eval function
|
||||
def occ_eval_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Evaluate occupancy given positions.
|
||||
Args:
|
||||
origins: Origins of rays
|
||||
directions: Directions of rays
|
||||
Returns:
|
||||
Function that returns the density of a point or None if a density function is not provided.
|
||||
"""
|
||||
|
||||
Args:
|
||||
x: positions with shape (N, 3).
|
||||
Returns:
|
||||
occupancy values with shape (N, 1).
|
||||
"""
|
||||
density_after_activation = density_fn(x)
|
||||
# those two are similar when density is small.
|
||||
# occupancy = 1.0 - torch.exp(-density_after_activation * render_step_size)
|
||||
occupancy = density_after_activation * self.render_step_size
|
||||
return occupancy
|
||||
if self.density_fn is None or not self.training:
|
||||
return None
|
||||
|
||||
self.occ_field = nerfacc.OccupancyField(occ_eval_fn=occ_eval_fn, aabb=aabb, resolution=grid_resolution)
|
||||
density_fn = self.density_fn
|
||||
|
||||
def sigma_fn(t_starts, t_ends, ray_indices):
|
||||
t_origins = origins[ray_indices]
|
||||
t_dirs = directions[ray_indices]
|
||||
positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0
|
||||
return density_fn(positions)
|
||||
|
||||
return sigma_fn
|
||||
|
||||
def generate_ray_samples(self) -> RaySamples:
|
||||
raise RuntimeError(
|
||||
@@ -448,17 +454,19 @@ class VolumetricSampler(Sampler):
|
||||
def forward(
|
||||
self,
|
||||
ray_bundle: RayBundle,
|
||||
num_samples: Optional[int] = None,
|
||||
render_step_size: float,
|
||||
near_plane: float = 0.0,
|
||||
remove_occluded_samples: bool = True,
|
||||
far_plane: Optional[float] = None,
|
||||
cone_angle: float = 0.0,
|
||||
) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples", 2]]:
|
||||
"""Generate ray samples in a bounding box.
|
||||
|
||||
Args:
|
||||
ray_bundle: Rays to generate samples for
|
||||
num_samples: Number of samples per ray
|
||||
render_step_size: Minimum step size to use for rendering
|
||||
near_plane: Near plane for raymarching
|
||||
remove_occluded_samples: Whether to remove occluded samples. (requires evaluating the density function)
|
||||
far_plane: Far plane for raymarching
|
||||
cone_angle: Cone angle for raymarching, set to 0 for uniform marching.
|
||||
|
||||
Returns:
|
||||
a tuple of (ray_samples, packed_info, ray_indices)
|
||||
@@ -470,12 +478,6 @@ class VolumetricSampler(Sampler):
|
||||
if self.density_field is not None:
|
||||
raise ValueError("NGPSpacedSampler does not support a density field")
|
||||
|
||||
if num_samples is not None:
|
||||
render_step_size = ((self.aabb[3:] - self.aabb[:3]).max() * math.sqrt(3) / num_samples).item()
|
||||
else:
|
||||
num_samples = self.num_samples
|
||||
render_step_size = self.render_step_size
|
||||
|
||||
rays_o = ray_bundle.origins.contiguous()
|
||||
rays_d = ray_bundle.directions.contiguous()
|
||||
if ray_bundle.camera_indices is not None:
|
||||
@@ -483,15 +485,17 @@ class VolumetricSampler(Sampler):
|
||||
else:
|
||||
camera_indices = None
|
||||
|
||||
packed_info, starts, ends = nerfacc.volumetric_marching(
|
||||
packed_info, starts, ends = nerfacc.ray_marching(
|
||||
rays_o=rays_o,
|
||||
rays_d=rays_d,
|
||||
aabb=self.aabb,
|
||||
scene_resolution=self.occ_field.resolution,
|
||||
scene_occ_binary=self.occ_field.occ_grid_binary,
|
||||
scene_aabb=self.scene_aabb,
|
||||
grid=self.occupancy_grid,
|
||||
sigma_fn=self.get_sigma_fn(rays_o, rays_d),
|
||||
render_step_size=render_step_size,
|
||||
near_plane=near_plane,
|
||||
far_plane=far_plane,
|
||||
stratified=self.training,
|
||||
cone_angle=cone_angle,
|
||||
)
|
||||
|
||||
num_samples = starts.shape[0]
|
||||
@@ -508,20 +512,6 @@ class VolumetricSampler(Sampler):
|
||||
if camera_indices is not None:
|
||||
camera_indices = camera_indices[ray_indices]
|
||||
|
||||
if remove_occluded_samples and self.training:
|
||||
with torch.no_grad():
|
||||
positions = origins + dirs * (starts + ends) / 2.0
|
||||
densities = self.density_fn(positions)
|
||||
if camera_indices is not None:
|
||||
packed_info, starts, ends, origins, dirs, camera_indices = nerfacc.volumetric_rendering_steps(
|
||||
packed_info, densities, starts, ends, origins, dirs, camera_indices
|
||||
)
|
||||
else:
|
||||
packed_info, starts, ends, origins, dirs = nerfacc.volumetric_rendering_steps(
|
||||
packed_info, densities, starts, ends, origins, dirs
|
||||
)
|
||||
ray_indices = nerfacc.unpack_to_ray_indices(packed_info).long()
|
||||
|
||||
zeros = torch.zeros_like(origins[:, :1])
|
||||
ray_samples = RaySamples(
|
||||
frustums=Frustums(
|
||||
|
||||
@@ -31,7 +31,6 @@ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
from nerfactory.cameras.rays import RayBundle
|
||||
from nerfactory.fields.compound_field import TCNNCompoundField
|
||||
from nerfactory.fields.density_field import DensityField
|
||||
from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.fields.modules.spatial_distortions import SceneContraction
|
||||
from nerfactory.models.base import Model, ModelConfig
|
||||
@@ -106,7 +105,7 @@ class ProposalModel(Model):
|
||||
self.scene_bounds.aabb, spatial_distortion=scene_contraction, num_images=self.num_train_data
|
||||
)
|
||||
else:
|
||||
self.field = TCNNInstantNGPField(self.scene_bounds.aabb, spatial_distortion=scene_contraction)
|
||||
raise NotImplementedError("Only appearance conditioning is supported.")
|
||||
|
||||
# Build the proposal network(s)
|
||||
self.proposal_networks = torch.nn.ModuleList()
|
||||
|
||||
@@ -71,8 +71,9 @@ class RGBRenderer(nn.Module):
|
||||
Outputs rgb values.
|
||||
"""
|
||||
if ray_indices is not None and num_rays is not None:
|
||||
rgb = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, rgb, num_rays)
|
||||
accumulated_weight = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, None, num_rays)
|
||||
# Necessary for packed samples from volumetric ray sampler
|
||||
rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays)
|
||||
accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
|
||||
else:
|
||||
rgb = torch.sum(weights * rgb, dim=-2)
|
||||
accumulated_weight = torch.sum(weights, dim=-2)
|
||||
@@ -181,7 +182,8 @@ class AccumulationRenderer(nn.Module):
|
||||
"""
|
||||
|
||||
if ray_indices is not None and num_rays is not None:
|
||||
accumulation = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, None, num_rays)
|
||||
# Necessary for packed samples from volumetric ray sampler
|
||||
accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
|
||||
else:
|
||||
accumulation = torch.sum(weights, dim=-2)
|
||||
return accumulation
|
||||
@@ -222,7 +224,8 @@ class DepthRenderer(nn.Module):
|
||||
steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2
|
||||
|
||||
if ray_indices is not None and num_rays is not None:
|
||||
depth = nerfacc.volumetric_rendering_accumulate(weights, ray_indices, steps, num_rays)
|
||||
# Necessary for packed samples from volumetric ray sampler
|
||||
depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)
|
||||
else:
|
||||
depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)
|
||||
|
||||
|
||||
Reference in new issue
Block a user