Files
nerfstudio/nerfactory/fields/instant_ngp_field.py
T

202 lines
7.8 KiB
Python

# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Instant-NGP field implementations using tiny-cuda-nn, torch, ....
"""
from typing import Optional
import torch
from nerfacc import ContractionType, contract
from torch.nn.parameter import Parameter
from torchtyping import TensorType
from nerfactory.cameras.rays import RaySamples
from nerfactory.datamanagers.structs import SceneBounds
from nerfactory.fields.base import Field
from nerfactory.fields.modules.embedding import Embedding
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.utils.activations import trunc_exp
try:
import tinycudann as tcnn
except ImportError:
# tinycudann module doesn't exist
pass
def get_normalized_directions(directions: TensorType["bs":..., 3]):
"""SH encoding must be in the range [0, 1]
Args:
directions: batch of directions
"""
return (directions + 1.0) / 2.0
class TCNNInstantNGPField(Field):
"""TCNN implementation of the Instant-NGP field.
Args:
aabb: parameters of scene aabb bounds
num_layers: number of hidden layers
hidden_dim: dimension of hidden layers
geo_feat_dim: output geo feat dimensions
num_layers_color: number of hidden layers for color network
hidden_dim_color: dimension of hidden layers for color network
use_appearance_embedding: whether to use appearance embedding
num_images: number of images, requried if use_appearance_embedding is True
appearance_embedding_dim: dimension of appearance embedding
contraction_type: type of contraction
"""
def __init__(
self,
aabb,
num_layers: int = 2,
hidden_dim: int = 64,
geo_feat_dim: int = 15,
num_layers_color: int = 3,
hidden_dim_color: int = 64,
use_appearance_embedding: bool = False,
num_images: Optional[int] = None,
appearance_embedding_dim: int = 32,
contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE,
) -> None:
super().__init__()
self.aabb = Parameter(aabb, requires_grad=False)
self.geo_feat_dim = geo_feat_dim
self.contraction_type = contraction_type
self.use_appearance_embedding = use_appearance_embedding
if use_appearance_embedding:
assert num_images is not None
self.appearance_embedding_dim = appearance_embedding_dim
self.appearance_embedding = Embedding(num_images, appearance_embedding_dim)
# TODO: set this properly based on the aabb
per_level_scale = 1.4472692012786865
self.direction_encoding = tcnn.Encoding(
n_input_dims=3,
encoding_config={
"otype": "SphericalHarmonics",
"degree": 4,
},
)
self.mlp_base = tcnn.NetworkWithInputEncoding(
n_input_dims=3,
n_output_dims=1 + self.geo_feat_dim,
encoding_config={
"otype": "HashGrid",
"n_levels": 16,
"n_features_per_level": 2,
"log2_hashmap_size": 19,
"base_resolution": 16,
"per_level_scale": per_level_scale,
},
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "None",
"n_neurons": hidden_dim,
"n_hidden_layers": num_layers - 1,
},
)
in_dim = self.direction_encoding.n_output_dims + self.geo_feat_dim
if self.use_appearance_embedding:
in_dim += self.appearance_embedding_dim
self.mlp_head = tcnn.Network(
n_input_dims=in_dim,
n_output_dims=3,
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "Sigmoid",
"n_neurons": hidden_dim_color,
"n_hidden_layers": num_layers_color - 1,
},
)
def get_density(self, ray_samples: RaySamples):
positions = ray_samples.frustums.get_positions()
positions_flat = positions.view(-1, 3)
positions_flat = contract(x=positions_flat, roi=self.aabb, type=self.contraction_type)
h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1)
density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1)
# Rectifying the density with an exponential is much more stable than a ReLU or
# softplus, because it enables high post-activation (float32) density outputs
# from smaller internal (float16) parameters.
density = trunc_exp(density_before_activation.to(positions))
return density, base_mlp_out
def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None):
directions = get_normalized_directions(ray_samples.frustums.directions)
directions_flat = directions.view(-1, 3)
d = self.direction_encoding(directions_flat)
if density_embedding is None:
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
h = torch.cat([d, positions.view(-1, 3)], dim=-1)
else:
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
if self.use_appearance_embedding:
if ray_samples.camera_indices is None:
raise AttributeError("Camera indices are not provided.")
camera_indices = ray_samples.camera_indices.squeeze()
if self.training:
embedded_appearance = self.appearance_embedding(camera_indices)
else:
embedded_appearance = torch.zeros(
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
)
h = torch.cat([h, embedded_appearance.view(-1, self.appearance_embedding_dim)], dim=-1)
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
return {FieldHeadNames.RGB: rgb}
def get_opacity(self, positions: TensorType["bs":..., 3], step_size) -> TensorType["bs":..., 1]:
"""Returns the opacity for a position. Used primarily by the occupancy grid.
Args:
positions: the positions to evaluate the opacity at.
step_size: the step size to use for the opacity evaluation.
"""
density = self.density_fn(positions)
## TODO: We should scale step size based on the distortion. Currently it uses too much memory.
# aabb_min, aabb_max = self.aabb[0], self.aabb[1]
# if self.contraction_type is not ContractionType.AABB:
# x = (positions - aabb_min) / (aabb_max - aabb_min)
# x = x * 2 - 1 # aabb is at [-1, 1]
# mag = x.norm(dim=-1, keepdim=True)
# mask = mag.squeeze(-1) > 1
# dev = (2 * mag - 1) / mag**2 + 2 * x**2 * (1 / mag**3 - (2 * mag - 1) / mag**4)
# dev[~mask] = 1.0
# dev = torch.clamp(dev, min=1e-6)
# step_size = step_size / dev.norm(dim=-1, keepdim=True)
# else:
# step_size = step_size * (aabb_max - aabb_min)
opacity = density * step_size
return opacity