mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-12 19:04:03 +08:00
300 lines
12 KiB
Python
300 lines
12 KiB
Python
"""
|
|
Encoding functions
|
|
"""
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from torch import nn
|
|
from torchtyping import TensorType
|
|
|
|
from mattport.nerf.field_modules.base import FieldModule
|
|
|
|
|
|
class Encoding(FieldModule):
|
|
"""Encode an input tensor. Intended to be subclassed"""
|
|
|
|
def __init__(self, in_dim: int) -> None:
|
|
"""
|
|
Args:
|
|
in_dim (int): Input dimension of tensor
|
|
"""
|
|
super().__init__()
|
|
if in_dim <= 0:
|
|
raise ValueError("Input dimension should be greater than zero")
|
|
self.in_dim = in_dim
|
|
|
|
def encode(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
"""Encodes an input tensor.
|
|
|
|
Args:
|
|
x (TensorType[..., "input_dim"]): Input tensor to be encoded
|
|
|
|
Returns:
|
|
TensorType[..., "output_dim"]: A encoded input tensor
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def forward(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
"""Call forward"""
|
|
return self.encode(in_tensor)
|
|
|
|
|
|
class ScalingAndOffset(Encoding):
|
|
"""Simple scaling and offet to input"""
|
|
|
|
def __init__(self, in_dim: int, scaling: float = 1.0, offset: float = 0.0) -> None:
|
|
"""Each input is scaled then offset
|
|
|
|
Args:
|
|
in_dim (int): Input dimension of tensor
|
|
scaling (float, optional): Scaling applied to tensor. Defaults to 1.0.
|
|
offset (float, optional): Offset applied to tensor. Defaults to 0.0.
|
|
"""
|
|
super().__init__(in_dim)
|
|
|
|
self.scaling = scaling
|
|
self.offset = offset
|
|
|
|
def get_out_dim(self) -> int:
|
|
return self.in_dim
|
|
|
|
def encode(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
return self.scaling * in_tensor + self.offset
|
|
|
|
|
|
class NeRFEncoding(Encoding):
|
|
"""Multi-scale sinousoidal encoding proposed in the original NeRF paper"""
|
|
|
|
def __init__(
|
|
self, in_dim: int, num_frequencies: int, min_freq_exp: float, max_freq_exp: float, include_input: bool = False
|
|
) -> None:
|
|
"""Each axis is encoded with frequencies ranging from 2^min_freq_exp to 2^max_freq_exp.
|
|
|
|
Args:
|
|
in_dim (int): Input dimension of tensor
|
|
num_frequencies (int): Number of encoded frequencies per axis
|
|
min_freq_exp (float): Minimum frequency exponent
|
|
max_freq_exp (float): Maximum frequency exponent
|
|
include_input (float): Append the input coordinate to the encoding
|
|
"""
|
|
super().__init__(in_dim)
|
|
|
|
self.num_frequencies = num_frequencies
|
|
self.min_freq = min_freq_exp
|
|
self.max_freq = max_freq_exp
|
|
self.include_input = include_input
|
|
|
|
def get_out_dim(self) -> int:
|
|
out_dim = self.in_dim * self.num_frequencies * 2
|
|
if self.include_input:
|
|
out_dim += self.in_dim
|
|
return out_dim
|
|
|
|
def encode(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
"""
|
|
Args:
|
|
in_tensor (TensorType[..., "input_dim"]): For best performance, the input tensor should be between 0 and 1.
|
|
|
|
Returns:
|
|
TensorType[..., "output_dim"]: Output values will be between -1 and 1
|
|
"""
|
|
freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies).to(in_tensor.device)
|
|
scaled_inputs = 2 * torch.pi * in_tensor[..., None] * freqs # [..., "input_dim", "num_scales"]
|
|
scaled_inputs = scaled_inputs.view(*scaled_inputs.shape[:-2], -1) # [..., "input_dim" * "num_scales"]
|
|
encoded_inputs = torch.cat([torch.sin(scaled_inputs), torch.cos(scaled_inputs)], axis=-1)
|
|
if self.include_input:
|
|
encoded_inputs = torch.cat([encoded_inputs, in_tensor], axis=-1)
|
|
return encoded_inputs
|
|
|
|
|
|
class RFFEncoding(Encoding):
|
|
"""Random Fourier Feature encoding"""
|
|
|
|
def __init__(self, in_dim: int, num_frequencies: int, scale: float) -> None:
|
|
"""
|
|
|
|
Args:
|
|
in_dim (int): Input dimension of tensor
|
|
num_frequencies (int): Number of encoding frequencies
|
|
scale (float): Std of Gaussian to sample frequencies. Must be greater than zero
|
|
"""
|
|
super().__init__(in_dim)
|
|
|
|
self.num_frequencies = num_frequencies
|
|
if not scale > 0:
|
|
raise ValueError("RFF encoding scale should be greater than zero")
|
|
self.scale = scale
|
|
self.b_matrix = torch.normal(mean=0, std=self.scale, size=(self.in_dim, self.num_frequencies))
|
|
|
|
def get_out_dim(self) -> int:
|
|
return self.num_frequencies * 2
|
|
|
|
def encode(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
"""
|
|
Args:
|
|
in_tensor (TensorType[..., "input_dim"]): For best performance, the input tensor should be between 0 and 1.
|
|
|
|
Returns:
|
|
TensorType[..., "output_dim"]: Output values will be between -1 and 1
|
|
"""
|
|
scaled_inputs = in_tensor @ self.b_matrix # [..., "num_frequencies"]
|
|
encoded_inputs = torch.cat([torch.sin(scaled_inputs), torch.cos(scaled_inputs)], axis=-1)
|
|
return encoded_inputs
|
|
|
|
|
|
class HashEncoding(Encoding):
|
|
"""Hash encoding"""
|
|
|
|
def __init__(
|
|
self,
|
|
num_levels: int = 16,
|
|
min_res: int = 16,
|
|
max_res: int = 1024,
|
|
hash_table_size: int = 2**19,
|
|
features_per_level: int = 2,
|
|
hash_init_scale: float = 0.001,
|
|
) -> None:
|
|
"""_summary_
|
|
|
|
Args:
|
|
num_levels (int, optional): Number of feature grids. Defaults to 16.
|
|
min_res (int, optional): Resolution of smallest feature grid. Defaults to 16.
|
|
max_res (int, optional): Resolution of largest feature grid. Defaults to 1024.
|
|
hash_table_size (int, optional): Size of hash table. Defaults to 2**19.
|
|
features_per_level (int, optional): Number of features per level. Defaults to 2.
|
|
hash_init_scale (float, optional): Value to initialize hash grid. Defaults to 0.001.
|
|
"""
|
|
super().__init__(in_dim=3)
|
|
self.num_levels = num_levels
|
|
self.features_per_level = features_per_level
|
|
self.hash_table_size = hash_table_size
|
|
|
|
levels = torch.arange(num_levels)
|
|
growth_factor = np.exp((np.log(max_res) - np.log(min_res)) / (num_levels - 1))
|
|
self.scalings = torch.floor(min_res * growth_factor**levels)
|
|
|
|
self.hash_offset = levels * hash_table_size
|
|
self.hash_table = torch.rand(size=(hash_table_size * num_levels, features_per_level)) * 2 - 1
|
|
self.hash_table *= hash_init_scale
|
|
self.hash_table = nn.Parameter(self.hash_table)
|
|
|
|
def get_out_dim(self) -> int:
|
|
return self.num_levels * self.features_per_level
|
|
|
|
def hash_fn(self, in_tensor: TensorType[..., "num_levels", 3]) -> TensorType[..., "num_levels"]:
|
|
"""Hash tensor using method described in Instant-NGP
|
|
|
|
Args:
|
|
in_tensor (TensorType[..., "num_levels", 3]): Tensor to be hashed
|
|
|
|
Returns:
|
|
TensorType[..., "num_levels"]: Hashed tensor
|
|
"""
|
|
in_tensor = in_tensor * torch.tensor([1, 2654435761, 805459861]).to(in_tensor.device)
|
|
x = torch.bitwise_xor(in_tensor[..., 0], in_tensor[..., 1])
|
|
x = torch.bitwise_xor(x, in_tensor[..., 2])
|
|
x %= self.hash_table_size
|
|
x += self.hash_offset.to(x.device)
|
|
return x
|
|
|
|
def encode(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
assert in_tensor.shape[-1] == 3
|
|
|
|
in_tensor = in_tensor[..., None, :] # [..., 1, 3]
|
|
scaled = in_tensor * self.scalings.view(1, 1, -1, 1).to(in_tensor.device) # [..., L, 3]
|
|
scaled_c = torch.ceil(scaled).type(torch.int32)
|
|
scaled_f = torch.floor(scaled).type(torch.int32)
|
|
|
|
offset = scaled - scaled_f
|
|
|
|
hashed_0 = self.hash_fn(scaled_c) # [..., num_levels]
|
|
hashed_1 = self.hash_fn(torch.cat([scaled_c[..., 0:1], scaled_f[..., 1:2], scaled_c[..., 2:3]], dim=-1))
|
|
hashed_2 = self.hash_fn(torch.cat([scaled_f[..., 0:1], scaled_f[..., 1:2], scaled_c[..., 2:3]], dim=-1))
|
|
hashed_3 = self.hash_fn(torch.cat([scaled_f[..., 0:1], scaled_c[..., 1:2], scaled_c[..., 2:3]], dim=-1))
|
|
hashed_4 = self.hash_fn(torch.cat([scaled_c[..., 0:1], scaled_c[..., 1:2], scaled_f[..., 2:3]], dim=-1))
|
|
hashed_5 = self.hash_fn(torch.cat([scaled_c[..., 0:1], scaled_f[..., 1:2], scaled_f[..., 2:3]], dim=-1))
|
|
hashed_6 = self.hash_fn(scaled_f)
|
|
hashed_7 = self.hash_fn(torch.cat([scaled_f[..., 0:1], scaled_c[..., 1:2], scaled_f[..., 2:3]], dim=-1))
|
|
|
|
f_0 = self.hash_table[hashed_0] # [..., num_levels, features_per_level]
|
|
f_1 = self.hash_table[hashed_1]
|
|
f_2 = self.hash_table[hashed_2]
|
|
f_3 = self.hash_table[hashed_3]
|
|
f_4 = self.hash_table[hashed_4]
|
|
f_5 = self.hash_table[hashed_5]
|
|
f_6 = self.hash_table[hashed_6]
|
|
f_7 = self.hash_table[hashed_7]
|
|
|
|
f_03 = f_0 * offset[..., 0:1] + f_3 * (1 - offset[..., 0:1])
|
|
f_12 = f_1 * offset[..., 0:1] + f_2 * (1 - offset[..., 0:1])
|
|
f_56 = f_5 * offset[..., 0:1] + f_6 * (1 - offset[..., 0:1])
|
|
f_47 = f_4 * offset[..., 0:1] + f_7 * (1 - offset[..., 0:1])
|
|
|
|
f0312 = f_03 * offset[..., 1:2] + f_12 * (1 - offset[..., 1:2])
|
|
f4756 = f_47 * offset[..., 1:2] + f_56 * (1 - offset[..., 1:2])
|
|
|
|
encoded_value = f0312 * offset[..., 2:3] + f4756 * (
|
|
1 - offset[..., 2:3]
|
|
) # [..., num_levels, features_per_level]
|
|
|
|
return torch.flatten(encoded_value, start_dim=-2, end_dim=-1) # [..., num_levels * features_per_level]
|
|
|
|
|
|
class TensorVMEncoding(Encoding):
|
|
"""Learned vector-matrix encoding proposed by TensoRF"""
|
|
|
|
def __init__(self, resolution: int = 256, num_components: int = 24, init_scale: float = 0.1) -> None:
|
|
"""
|
|
Args:
|
|
resolution (int, optional): Resolution of grid. Defaults to 256.
|
|
num_components (int, optional): Number of components per dimension. Defaults to 24.
|
|
init_scale (float, optional): Initialization scale. Defaults to 0.1.
|
|
"""
|
|
super().__init__(in_dim=3)
|
|
|
|
self.resolution = resolution
|
|
self.num_components = num_components
|
|
|
|
# TODO Learning rates should be different for these
|
|
self.plane_coef = nn.Parameter(init_scale * torch.randn((3, num_components, resolution, resolution)))
|
|
self.line_coef = nn.Parameter(init_scale * torch.randn((3, num_components, resolution, 1)))
|
|
|
|
def get_out_dim(self) -> int:
|
|
return self.num_components * 3
|
|
|
|
def encode(self, in_tensor: TensorType[..., "input_dim"]) -> TensorType[..., "output_dim"]:
|
|
|
|
plane_coord = torch.stack([in_tensor[..., [0, 1]], in_tensor[..., [0, 2]], in_tensor[..., [1, 2]]]) # [3,...,2]
|
|
line_coord = torch.stack([in_tensor[..., 2], in_tensor[..., 1], in_tensor[..., 0]]) # [3, ...]
|
|
line_coord = torch.stack([torch.zeros_like(line_coord), line_coord], dim=-1) # [3, ...., 2]
|
|
|
|
# Stop gradients from going to sampler
|
|
plane_coord = plane_coord.view(3, -1, 1, 2).detach()
|
|
line_coord = line_coord.view(3, -1, 1, 2).detach()
|
|
|
|
plane_features = F.grid_sample(self.plane_coef, plane_coord, align_corners=True) # [3, Components, -1, 1]
|
|
line_features = F.grid_sample(self.line_coef, line_coord, align_corners=True) # [3, Components, -1, 1]
|
|
|
|
features = plane_features * line_features # [3, Components, -1, 1]
|
|
features = torch.moveaxis(features.view(3 * self.num_components, *in_tensor.shape[:-1]), 0, -1)
|
|
|
|
return features # [..., 3 * Components]
|
|
|
|
@torch.no_grad()
|
|
def upsample_grid(self, resolution: int) -> None:
|
|
"""Upsamples underyling feature grid
|
|
|
|
Args:
|
|
resolution (int): Target resolution.
|
|
"""
|
|
|
|
self.plane_coef.data = F.interpolate(
|
|
self.plane_coef.data, size=(resolution, resolution), mode="bilinear", align_corners=True
|
|
)
|
|
self.line_coef.data = F.interpolate(
|
|
self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True
|
|
)
|
|
|
|
self.resolution = resolution
|