mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 07:05:44 +08:00
Triplanes (#1456)
* Triplanes working * Doing the TODOs, better variable names --------- Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
1 parent
9e68a46abe
commit
50dc5d6cd7
2 files changed
+121
-10
No files matched your search
@@ -469,11 +469,94 @@ class TensorVMEncoding(Encoding):
|
||||
)
|
||||
line_coef = F.interpolate(self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True)
|
||||
|
||||
# TODO(ethan): are these torch.nn.Parameters needed?
|
||||
self.plane_coef, self.line_coef = torch.nn.Parameter(plane_coef), torch.nn.Parameter(line_coef)
|
||||
self.resolution = resolution
|
||||
|
||||
|
||||
class TriplaneEncoding(Encoding):
|
||||
"""Learned triplane encoding
|
||||
|
||||
The encoding at [i,j,k] is an n dimensional vector corresponding to the element-wise product of the
|
||||
three n dimensional vectors at plane_coeff[i,j], plane_coeff[i,k], and plane_coeff[j,k].
|
||||
|
||||
This allows for marginally more expressivity than the TensorVMEncoding, and each component is self standing
|
||||
and symmetrical, unlike with VM decomposition where we needed one component with a vector along all the x, y, z
|
||||
directions for symmetry.
|
||||
|
||||
This can be thought of as 3 planes of features perpendicular to the x, y, and z axes, respectively and intersecting
|
||||
at the origin, and the encoding being the element-wise product of the element at the projection of [i, j, k] on
|
||||
these planes.
|
||||
|
||||
The use for this is in representing a tensor decomp of a 4D embedding tensor: (x, y, z, feature_size)
|
||||
|
||||
This will return a tensor of shape (bs:..., num_components)
|
||||
|
||||
Args:
|
||||
resolution: Resolution of grid.
|
||||
num_components: The number of scalar triplanes to use (ie: output feature size)
|
||||
init_scale: The scale of the initial values of the planes
|
||||
product: Whether to use the element-wise product of the planes or the sum
|
||||
"""
|
||||
|
||||
plane_coef: TensorType[3, "num_components", "resolution", "resolution"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
resolution: int = 32,
|
||||
num_components: int = 64,
|
||||
init_scale: float = 0.1,
|
||||
reduce: Literal["sum", "product"] = "sum",
|
||||
) -> None:
|
||||
super().__init__(in_dim=3)
|
||||
|
||||
self.resolution = resolution
|
||||
self.num_components = num_components
|
||||
self.init_scale = init_scale
|
||||
self.reduce = reduce
|
||||
|
||||
self.plane_coef = nn.Parameter(
|
||||
self.init_scale * torch.randn((3, self.num_components, self.resolution, self.resolution))
|
||||
)
|
||||
|
||||
def get_out_dim(self) -> int:
|
||||
return self.num_components
|
||||
|
||||
def forward(self, in_tensor: TensorType["bs":..., 3]) -> TensorType["bs":..., "num_components", "featuresize"]:
|
||||
"""Sample features from this encoder. Expects in_tensor to be in range [0, resolution]"""
|
||||
|
||||
original_shape = in_tensor.shape
|
||||
in_tensor = in_tensor.reshape(-1, 3)
|
||||
|
||||
plane_coord = torch.stack([in_tensor[..., [0, 1]], in_tensor[..., [0, 2]], in_tensor[..., [1, 2]]], dim=0)
|
||||
|
||||
# Stop gradients from going to sampler
|
||||
plane_coord = plane_coord.detach().view(3, -1, 1, 2)
|
||||
plane_features = F.grid_sample(
|
||||
self.plane_coef, plane_coord, align_corners=True
|
||||
) # [3, num_components, flattened_bs, 1]
|
||||
|
||||
if self.reduce == "product":
|
||||
plane_features = plane_features.prod(0).squeeze(-1).T # [flattened_bs, num_components]
|
||||
else:
|
||||
plane_features = plane_features.sum(0).squeeze(-1).T
|
||||
|
||||
return plane_features.reshape(*original_shape[:-1], self.num_components)
|
||||
|
||||
@torch.no_grad()
|
||||
def upsample_grid(self, resolution: int) -> None:
|
||||
"""Upsamples underlying feature grid
|
||||
|
||||
Args:
|
||||
resolution: Target resolution.
|
||||
"""
|
||||
plane_coef = F.interpolate(
|
||||
self.plane_coef.data, size=(resolution, resolution), mode="bilinear", align_corners=True
|
||||
)
|
||||
|
||||
self.plane_coef = torch.nn.Parameter(plane_coef)
|
||||
self.resolution = resolution
|
||||
|
||||
|
||||
class SHEncoding(Encoding):
|
||||
"""Spherical harmonic encoding
|
||||
|
||||
|
||||
Reference in new issue
Block a user