mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-22 23:55:36 +08:00
compound model (wip) (#427)
* compound model now uses compound field, passing num_images in the config, working on passing image indices through ngp sampler * renaming cuda func, adding torch compound support * removing unused imports * getting num images from dataloader instead of config * fixing pylint and adding todo * undoing base config change * adding cone angle and near plane to compound model config
This commit is contained in:
1 parent
2495cfe1ee
commit
84024a4e73
10 files changed
+170
-31
No files matched your search
@@ -17,23 +17,29 @@ Field for compound nerf model, adds scene contraction and image embeddings to in
|
||||
"""
|
||||
|
||||
|
||||
from typing import Optional, Tuple
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
from torchtyping import TensorType
|
||||
|
||||
from nerfactory.cameras.rays import RaySamples
|
||||
from nerfactory.datamanagers.structs import SceneBounds
|
||||
from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField
|
||||
from nerfactory.fields.base import Field
|
||||
from nerfactory.fields.modules.embedding import Embedding
|
||||
from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.fields.modules.field_heads import (
|
||||
DensityFieldHead,
|
||||
FieldHead,
|
||||
FieldHeadNames,
|
||||
RGBFieldHead,
|
||||
)
|
||||
from nerfactory.fields.modules.mlp import MLP
|
||||
from nerfactory.fields.modules.spatial_distortions import (
|
||||
SceneContraction,
|
||||
SpatialDistortion,
|
||||
)
|
||||
from nerfactory.fields.nerf_field import NeRFField
|
||||
from nerfactory.utils.activations import trunc_exp
|
||||
|
||||
try:
|
||||
@@ -48,7 +54,7 @@ def get_normalized_directions(directions):
|
||||
return (directions + 1.0) / 2.0
|
||||
|
||||
|
||||
class TCNNCompoundField(TCNNInstantNGPField):
|
||||
class TCNNCompoundField(Field):
|
||||
"""Compound Field that uses TCNN"""
|
||||
|
||||
def __init__(
|
||||
@@ -63,7 +69,9 @@ class TCNNCompoundField(TCNNInstantNGPField):
|
||||
appearance_embedding_dim: int = 40,
|
||||
spatial_distortion: SpatialDistortion = SceneContraction(),
|
||||
) -> None:
|
||||
super().__init__(self, aabb)
|
||||
super().__init__()
|
||||
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
self.geo_feat_dim = geo_feat_dim
|
||||
|
||||
self.spatial_distortion = spatial_distortion
|
||||
@@ -71,10 +79,6 @@ class TCNNCompoundField(TCNNInstantNGPField):
|
||||
self.appearance_embedding_dim = appearance_embedding_dim
|
||||
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
|
||||
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
|
||||
self.geo_feat_dim = geo_feat_dim
|
||||
|
||||
# TODO: set this properly based on the aabb
|
||||
per_level_scale = 1.4472692012786865
|
||||
|
||||
@@ -120,6 +124,7 @@ class TCNNCompoundField(TCNNInstantNGPField):
|
||||
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
"""Computes and returns the densities."""
|
||||
# TODO: add spatial distortion to ngp sampler
|
||||
# positions = self.spatial_distortion(ray_samples.frustums.get_positions())
|
||||
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
positions_flat = positions.view(-1, 3)
|
||||
@@ -137,25 +142,36 @@ class TCNNCompoundField(TCNNInstantNGPField):
|
||||
# tcnn requires directions in the range [0,1]
|
||||
if ray_samples.camera_indices is None:
|
||||
raise AttributeError("Camera indices are not provided.")
|
||||
|
||||
camera_indices = ray_samples.camera_indices.squeeze().to(ray_samples.frustums.origins.device)
|
||||
embedded_appearance = self.embedding_appearance(camera_indices)
|
||||
|
||||
directions = get_normalized_directions(ray_samples.frustums.directions)
|
||||
directions_flat = directions.view(-1, 3)
|
||||
d = self.direction_encoding(directions_flat)
|
||||
|
||||
if self.training:
|
||||
embedded_appearance = self.embedding_appearance(camera_indices)
|
||||
else:
|
||||
embedded_appearance = torch.zeros((*directions.shape[:-1], self.appearance_embedding_dim)).to(
|
||||
directions.device
|
||||
)
|
||||
|
||||
if density_embedding is None:
|
||||
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
h = torch.cat([d, positions.view(-1, 3), embedded_appearance], dim=-1)
|
||||
else:
|
||||
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim), embedded_appearance], dim=-1)
|
||||
h = torch.cat(
|
||||
[
|
||||
d,
|
||||
density_embedding.view(-1, self.geo_feat_dim),
|
||||
embedded_appearance.view(-1, self.appearance_embedding_dim),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
|
||||
|
||||
return {FieldHeadNames.RGB: rgb}
|
||||
|
||||
|
||||
class TorchCompoundField(NeRFField):
|
||||
class TorchCompoundField(Field):
|
||||
"""
|
||||
PyTorch implementation of the compound field.
|
||||
"""
|
||||
@@ -163,26 +179,85 @@ class TorchCompoundField(NeRFField):
|
||||
def __init__(
|
||||
self,
|
||||
aabb,
|
||||
num_images: int,
|
||||
position_encoding: Encoding = HashEncoding(),
|
||||
direction_encoding: Encoding = SHEncoding(),
|
||||
base_mlp_num_layers: int = 3,
|
||||
base_mlp_layer_width: int = 64,
|
||||
head_mlp_num_layers: int = 2,
|
||||
head_mlp_layer_width: int = 32,
|
||||
appearance_embedding_dim: int = 40,
|
||||
skip_connections: Tuple = (4,),
|
||||
field_heads: Tuple[FieldHead] = (RGBFieldHead(),),
|
||||
spatial_distortion: SpatialDistortion = SceneContraction(),
|
||||
) -> None:
|
||||
super().__init__(
|
||||
position_encoding,
|
||||
direction_encoding,
|
||||
base_mlp_num_layers,
|
||||
base_mlp_layer_width,
|
||||
head_mlp_num_layers,
|
||||
head_mlp_layer_width,
|
||||
skip_connections,
|
||||
spatial_distortion=spatial_distortion,
|
||||
)
|
||||
super().__init__()
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
self.spatial_distortion = spatial_distortion
|
||||
self.num_images = num_images
|
||||
self.appearance_embedding_dim = appearance_embedding_dim
|
||||
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
|
||||
|
||||
self.position_encoding = position_encoding
|
||||
self.direction_encoding = direction_encoding
|
||||
self.spatial_distortion = spatial_distortion
|
||||
|
||||
self.mlp_base = MLP(
|
||||
in_dim=self.position_encoding.get_out_dim(),
|
||||
num_layers=base_mlp_num_layers,
|
||||
layer_width=base_mlp_layer_width,
|
||||
skip_connections=skip_connections,
|
||||
)
|
||||
|
||||
self.mlp_head = MLP(
|
||||
in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim() + self.appearance_embedding_dim,
|
||||
num_layers=head_mlp_num_layers,
|
||||
layer_width=head_mlp_layer_width,
|
||||
)
|
||||
|
||||
self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim())
|
||||
self.field_heads = nn.ModuleList(field_heads)
|
||||
for field_head in self.field_heads:
|
||||
field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore
|
||||
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
# TODO: add spatial distortion to ngp sampler
|
||||
# positions = self.spatial_distortion(ray_samples.frustums.get_positions())
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
encoded_xyz = self.position_encoding(positions)
|
||||
base_mlp_out = self.mlp_base(encoded_xyz)
|
||||
density = self.field_output_density(base_mlp_out)
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(
|
||||
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
|
||||
) -> Dict[FieldHeadNames, TensorType]:
|
||||
|
||||
if ray_samples.camera_indices is None:
|
||||
raise AttributeError("Camera indices are not provided.")
|
||||
camera_indices = ray_samples.camera_indices.squeeze().to(ray_samples.frustums.origins.device)
|
||||
if self.training:
|
||||
embedded_appearance = self.embedding_appearance(camera_indices)
|
||||
else:
|
||||
embedded_appearance = torch.zeros(
|
||||
(*ray_samples.frustums.directions.shape[:-1], self.appearance_embedding_dim)
|
||||
).to(ray_samples.frustums.directions.device)
|
||||
|
||||
outputs = {}
|
||||
for field_head in self.field_heads:
|
||||
encoded_dir = self.direction_encoding(ray_samples.frustums.directions)
|
||||
mlp_out = self.mlp_head(
|
||||
torch.cat(
|
||||
[
|
||||
encoded_dir,
|
||||
density_embedding, # type:ignore
|
||||
embedded_appearance.view(-1, self.appearance_embedding_dim),
|
||||
],
|
||||
dim=-1, # type:ignore
|
||||
)
|
||||
)
|
||||
outputs[field_head.field_head_name] = field_head(mlp_out)
|
||||
return outputs
|
||||
|
||||
|
||||
field_implementation_to_class = {"tcnn": TCNNCompoundField, "torch": TorchCompoundField}
|
||||
Reference in new issue
Block a user