Cleanup mipnerf implementation

This commit is contained in:
Matt Tancik authored and Matthew Tancik committed 2022-06-09 17:26:58 -07:00
1 parent 6deb452825
commit cc79708df7
7 files changed
+377 -16

No files matched your search

+25 -6
View File
@@ -126,7 +126,7 @@ class NeRFEncoding(Encoding):
if covs is None:
encoded_inputs = torch.sin(torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1))
else:
input_var = torch.diagonal(covs, dim1=-2, dim2=-1)[..., None, :] * freqs[:, None] ** 2
input_var = torch.diagonal(covs, dim1=-2, dim2=-1)[..., :, None] * freqs[None, :] ** 2
input_var = input_var.reshape((*input_var.shape[:-2], -1))
encoded_inputs = expected_sin(
torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1), torch.cat(2 * [input_var], dim=-1)
@@ -138,15 +138,16 @@ class NeRFEncoding(Encoding):
class RFFEncoding(Encoding):
"""Random Fourier Feature encoding"""
"""Random Fourier Feature encoding. Supports integrated encodings."""
def __init__(self, in_dim: int, num_frequencies: int, scale: float) -> None:
def __init__(self, in_dim: int, num_frequencies: int, scale: float, include_input: bool = False) -> None:
"""
Args:
in_dim (int): Input dimension of tensor
num_frequencies (int): Number of encoding frequencies
scale (float): Std of Gaussian to sample frequencies. Must be greater than zero
include_input (float): Append the input coordinate to the encoding
"""
super().__init__(in_dim)
@@ -156,20 +157,38 @@ class RFFEncoding(Encoding):
self.scale = scale
b_matrix = torch.normal(mean=0, std=self.scale, size=(self.in_dim, self.num_frequencies))
self.register_buffer(name="b_matrix", tensor=b_matrix)
self.include_input = include_input
def get_out_dim(self) -> int:
return self.num_frequencies * 2
def forward(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
"""
def forward(
self, in_tensor: TensorType[..., "input_dim"], covs: Optional[TensorType[..., "input_dim", "input_dim"]] = None
) -> TensorType[..., "output_dim"]:
"""Calculates RFF encoding. If covariances are provided the encodings will be integrated as proposed
in mip-NeRF.
Args:
in_tensor (TensorType[..., "input_dim"]): For best performance, the input tensor should be between 0 and 1.
covs (TensorType[..., "input_dim", "input_dim"], optional): Covariances of input points. Defaults to None.
Returns:
TensorType[..., "output_dim"]: Output values will be between -1 and 1
"""
in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
scaled_inputs = in_tensor @ self.b_matrix # [..., "num_frequencies"]
encoded_inputs = torch.cat([torch.sin(scaled_inputs), torch.cos(scaled_inputs)], axis=-1)
if covs is None:
encoded_inputs = torch.sin(torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1))
else:
input_var = torch.sum(((covs @ self.b_matrix)) * self.b_matrix, -2)
encoded_inputs = expected_sin(
torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1), torch.cat(2 * [input_var], dim=-1)
)
if self.include_input:
encoded_inputs = torch.cat([encoded_inputs, in_tensor], axis=-1)
return encoded_inputs
+8 -1
View File
@@ -41,6 +41,7 @@ class NeRFField(Field):
head_mlp_layer_width: int = 128,
skip_connections: Tuple[int] = (4,),
field_heads: Tuple[FieldHead] = (RGBFieldHead(),),
use_integrated_encoding: bool = False,
) -> None:
"""Create boilerplate NeRF field.
@@ -52,10 +53,12 @@ class NeRFField(Field):
head_mlp_num_layers (int, optional): Number of layer for ourput head MLP. Defaults to 2.
head_mlp_layer_width (int, optional): Width of output head MLP layers. Defaults to 128.
skip_connections (Tuple, optional): Where to add skip connection in base MLP. Defaults to (4,).
use_integrated_encoding (bool, optional): Used integrated samples as encoding input, Defaults to False.
"""
super().__init__()
self.position_encoding = position_encoding
self.direction_encoding = direction_encoding
self.use_integrated_encoding = use_integrated_encoding
self.mlp_base = MLP(
in_dim=self.position_encoding.get_out_dim(),
@@ -77,7 +80,11 @@ class NeRFField(Field):
def get_density(self, point_samples: PointSamples):
"""Computes and returns the densities."""
encoded_xyz = self.position_encoding(point_samples.frustums.get_positions())
if self.use_integrated_encoding:
gaussian_samples = point_samples.frustums.get_gaussian_blob()
encoded_xyz = self.position_encoding(gaussian_samples.mean, covs=gaussian_samples.cov)
else:
encoded_xyz = self.position_encoding(point_samples.frustums.get_positions())
base_mlp_out = self.mlp_base(encoded_xyz)
density = self.field_output_density(base_mlp_out)
return density, base_mlp_out