mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
Ability to use vanilla pytorch instead of tcnn for Nerfacto (#1936)
* started a package to easily swap out tcnn for pytorch * trains with pytorch and not tcnn * cleaned up imports * making it easy to install on mac * removed/reverted files * merged main * reverted changes * updating config * starting to add nerfacto to tests * fixxing default value * fixed default implementation * defaults to torch if tcnn and not installed * update log * add comment * resolved pr comments * fix * update default --------- Co-authored-by: Ethan Weber <ethanweber@eecs-128-32-175-250.dhcp.berkeley.edu>
This commit is contained in:
1 parent
400ab5a313
commit
bb6925bac6
19 files changed
+295
-296
No files matched your search
@@ -17,24 +17,19 @@ Proposal network field.
|
||||
"""
|
||||
|
||||
|
||||
from typing import Optional, Tuple
|
||||
from typing import Literal, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import Tensor
|
||||
from torch import Tensor, nn
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.field_components.activations import trunc_exp
|
||||
from nerfstudio.field_components.encodings import HashEncoding
|
||||
from nerfstudio.field_components.mlp import MLP
|
||||
from nerfstudio.field_components.spatial_distortions import SpatialDistortion
|
||||
from nerfstudio.fields.base_field import Field
|
||||
|
||||
try:
|
||||
import tinycudann as tcnn
|
||||
except ImportError:
|
||||
# tinycudann module doesn't exist
|
||||
pass
|
||||
|
||||
|
||||
class HashMLPDensityField(Field):
|
||||
"""A lightweight density field module.
|
||||
@@ -61,45 +56,39 @@ class HashMLPDensityField(Field):
|
||||
base_res: int = 16,
|
||||
log2_hashmap_size: int = 18,
|
||||
features_per_level: int = 2,
|
||||
implementation: Literal["tcnn", "torch"] = "torch",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.register_buffer("aabb", aabb)
|
||||
self.spatial_distortion = spatial_distortion
|
||||
self.use_linear = use_linear
|
||||
growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
|
||||
|
||||
self.register_buffer("max_res", torch.tensor(max_res))
|
||||
self.register_buffer("num_levels", torch.tensor(num_levels))
|
||||
self.register_buffer("log2_hashmap_size", torch.tensor(log2_hashmap_size))
|
||||
|
||||
config = {
|
||||
"encoding": {
|
||||
"otype": "HashGrid",
|
||||
"n_levels": num_levels,
|
||||
"n_features_per_level": features_per_level,
|
||||
"log2_hashmap_size": log2_hashmap_size,
|
||||
"base_resolution": base_res,
|
||||
"per_level_scale": growth_factor,
|
||||
},
|
||||
"network": {
|
||||
"otype": "FullyFusedMLP",
|
||||
"activation": "ReLU",
|
||||
"output_activation": "None",
|
||||
"n_neurons": hidden_dim,
|
||||
"n_hidden_layers": num_layers - 1,
|
||||
},
|
||||
}
|
||||
self.encoding = HashEncoding(
|
||||
num_levels=num_levels,
|
||||
min_res=base_res,
|
||||
max_res=max_res,
|
||||
log2_hashmap_size=log2_hashmap_size,
|
||||
features_per_level=features_per_level,
|
||||
implementation=implementation,
|
||||
)
|
||||
|
||||
if not self.use_linear:
|
||||
self.mlp_base = tcnn.NetworkWithInputEncoding(
|
||||
n_input_dims=3,
|
||||
n_output_dims=1,
|
||||
encoding_config=config["encoding"],
|
||||
network_config=config["network"],
|
||||
network = MLP(
|
||||
in_dim=self.encoding.get_out_dim(),
|
||||
num_layers=num_layers,
|
||||
layer_width=hidden_dim,
|
||||
out_dim=1,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=None,
|
||||
implementation=implementation,
|
||||
)
|
||||
self.mlp_base = torch.nn.Sequential(self.encoding, network)
|
||||
else:
|
||||
self.encoding = tcnn.Encoding(n_input_dims=3, encoding_config=config["encoding"])
|
||||
self.linear = torch.nn.Linear(self.encoding.n_output_dims, 1)
|
||||
self.linear = torch.nn.Linear(self.encoding.get_out_dim(), 1)
|
||||
|
||||
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, None]:
|
||||
if self.spatial_distortion is not None:
|
||||
|
||||
@@ -60,6 +60,7 @@ class TCNNInstantNGPField(Field):
|
||||
num_levels: number of levels of the hashmap for the base mlp
|
||||
log2_hashmap_size: size of the hashmap for the base mlp
|
||||
max_res: maximum resolution of the hashmap for the base mlp
|
||||
mode: which implementation to use for the field
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -17,44 +17,30 @@ Field for compound nerf model, adds scene contraction and image embeddings to in
|
||||
"""
|
||||
|
||||
|
||||
from typing import Dict, Optional, Tuple, Type
|
||||
from typing import Dict, Literal, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import Tensor, nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.field_components.activations import trunc_exp
|
||||
from nerfstudio.field_components.embedding import Embedding
|
||||
from nerfstudio.field_components.encodings import Encoding, HashEncoding, SHEncoding
|
||||
from nerfstudio.field_components.encodings import HashEncoding, NeRFEncoding, SHEncoding
|
||||
from nerfstudio.field_components.field_heads import (
|
||||
DensityFieldHead,
|
||||
FieldHead,
|
||||
FieldHeadNames,
|
||||
PredNormalsFieldHead,
|
||||
RGBFieldHead,
|
||||
SemanticFieldHead,
|
||||
TransientDensityFieldHead,
|
||||
TransientRGBFieldHead,
|
||||
UncertaintyFieldHead,
|
||||
)
|
||||
from nerfstudio.field_components.mlp import MLP
|
||||
from nerfstudio.field_components.spatial_distortions import (
|
||||
SceneContraction,
|
||||
SpatialDistortion,
|
||||
)
|
||||
from nerfstudio.field_components.spatial_distortions import SpatialDistortion
|
||||
from nerfstudio.fields.base_field import Field, shift_directions_for_tcnn
|
||||
|
||||
try:
|
||||
import tinycudann as tcnn
|
||||
except ModuleNotFoundError:
|
||||
# tinycudann module doesn't exist
|
||||
pass
|
||||
|
||||
|
||||
class TCNNNerfactoField(Field):
|
||||
class NerfactoField(Field):
|
||||
"""Compound Field that uses TCNN
|
||||
|
||||
Args:
|
||||
@@ -105,6 +91,7 @@ class TCNNNerfactoField(Field):
|
||||
use_pred_normals: bool = False,
|
||||
use_average_appearance_embedding: bool = False,
|
||||
spatial_distortion: Optional[SpatialDistortion] = None,
|
||||
implementation: Literal["tcnn", "torch"] = "tcnn",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -127,102 +114,88 @@ class TCNNNerfactoField(Field):
|
||||
|
||||
base_res: int = 16
|
||||
features_per_level: int = 2
|
||||
growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
|
||||
|
||||
self.direction_encoding = tcnn.Encoding(
|
||||
n_input_dims=3,
|
||||
encoding_config={
|
||||
"otype": "SphericalHarmonics",
|
||||
"degree": 4,
|
||||
},
|
||||
self.direction_encoding = SHEncoding(
|
||||
levels=4,
|
||||
implementation=implementation,
|
||||
)
|
||||
|
||||
self.position_encoding = tcnn.Encoding(
|
||||
n_input_dims=3,
|
||||
encoding_config={"otype": "Frequency", "n_frequencies": 2},
|
||||
self.position_encoding = NeRFEncoding(
|
||||
in_dim=3, num_frequencies=2, min_freq_exp=0, max_freq_exp=2 - 1, implementation=implementation
|
||||
)
|
||||
|
||||
self.mlp_base = tcnn.NetworkWithInputEncoding(
|
||||
n_input_dims=3,
|
||||
n_output_dims=1 + self.geo_feat_dim,
|
||||
encoding_config={
|
||||
"otype": "HashGrid",
|
||||
"n_levels": num_levels,
|
||||
"n_features_per_level": features_per_level,
|
||||
"log2_hashmap_size": log2_hashmap_size,
|
||||
"base_resolution": base_res,
|
||||
"per_level_scale": growth_factor,
|
||||
},
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
"activation": "ReLU",
|
||||
"output_activation": "None",
|
||||
"n_neurons": hidden_dim,
|
||||
"n_hidden_layers": num_layers - 1,
|
||||
},
|
||||
encoder = HashEncoding(
|
||||
num_levels=num_levels,
|
||||
min_res=base_res,
|
||||
max_res=max_res,
|
||||
log2_hashmap_size=log2_hashmap_size,
|
||||
features_per_level=features_per_level,
|
||||
implementation=implementation,
|
||||
)
|
||||
network = MLP(
|
||||
in_dim=encoder.get_out_dim(),
|
||||
num_layers=num_layers,
|
||||
layer_width=hidden_dim,
|
||||
out_dim=1 + self.geo_feat_dim,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=None,
|
||||
implementation=implementation,
|
||||
)
|
||||
self.mlp_base = torch.nn.Sequential(encoder, network)
|
||||
|
||||
# transients
|
||||
if self.use_transient_embedding:
|
||||
self.transient_embedding_dim = transient_embedding_dim
|
||||
self.embedding_transient = Embedding(self.num_images, self.transient_embedding_dim)
|
||||
self.mlp_transient = tcnn.Network(
|
||||
n_input_dims=self.geo_feat_dim + self.transient_embedding_dim,
|
||||
n_output_dims=hidden_dim_transient,
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
"activation": "ReLU",
|
||||
"output_activation": "None",
|
||||
"n_neurons": hidden_dim_transient,
|
||||
"n_hidden_layers": num_layers_transient - 1,
|
||||
},
|
||||
self.mlp_transient = MLP(
|
||||
in_dim=self.geo_feat_dim + self.transient_embedding_dim,
|
||||
num_layers=num_layers_transient,
|
||||
layer_width=hidden_dim_transient,
|
||||
out_dim=hidden_dim_transient,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=None,
|
||||
implementation=implementation,
|
||||
)
|
||||
self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.n_output_dims)
|
||||
self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.n_output_dims)
|
||||
self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.n_output_dims)
|
||||
self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.get_out_dim())
|
||||
self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim())
|
||||
self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim())
|
||||
|
||||
# semantics
|
||||
if self.use_semantics:
|
||||
self.mlp_semantics = tcnn.Network(
|
||||
n_input_dims=self.geo_feat_dim,
|
||||
n_output_dims=hidden_dim_transient,
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
"activation": "ReLU",
|
||||
"output_activation": "None",
|
||||
"n_neurons": 64,
|
||||
"n_hidden_layers": 1,
|
||||
},
|
||||
self.mlp_semantics = MLP(
|
||||
in_dim=self.geo_feat_dim,
|
||||
num_layers=2,
|
||||
layer_width=64,
|
||||
out_dim=hidden_dim_transient,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=None,
|
||||
implementation=implementation,
|
||||
)
|
||||
self.field_head_semantics = SemanticFieldHead(
|
||||
in_dim=self.mlp_semantics.n_output_dims, num_classes=num_semantic_classes
|
||||
in_dim=self.mlp_semantics.get_out_dim(), num_classes=num_semantic_classes
|
||||
)
|
||||
|
||||
# predicted normals
|
||||
if self.use_pred_normals:
|
||||
self.mlp_pred_normals = tcnn.Network(
|
||||
n_input_dims=self.geo_feat_dim + self.position_encoding.n_output_dims,
|
||||
n_output_dims=hidden_dim_transient,
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
"activation": "ReLU",
|
||||
"output_activation": "None",
|
||||
"n_neurons": 64,
|
||||
"n_hidden_layers": 2,
|
||||
},
|
||||
self.mlp_pred_normals = MLP(
|
||||
in_dim=self.geo_feat_dim + self.position_encoding.get_out_dim(),
|
||||
num_layers=3,
|
||||
layer_width=64,
|
||||
out_dim=hidden_dim_transient,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=None,
|
||||
implementation=implementation,
|
||||
)
|
||||
self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.n_output_dims)
|
||||
self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.get_out_dim())
|
||||
|
||||
self.mlp_head = tcnn.Network(
|
||||
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim,
|
||||
n_output_dims=3,
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
"activation": "ReLU",
|
||||
"output_activation": "Sigmoid",
|
||||
"n_neurons": hidden_dim_color,
|
||||
"n_hidden_layers": num_layers_color - 1,
|
||||
},
|
||||
self.mlp_head = MLP(
|
||||
in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim,
|
||||
num_layers=num_layers_color,
|
||||
layer_width=hidden_dim_color,
|
||||
out_dim=3,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=nn.Sigmoid(),
|
||||
implementation=implementation,
|
||||
)
|
||||
|
||||
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
|
||||
@@ -324,102 +297,3 @@ class TCNNNerfactoField(Field):
|
||||
outputs.update({FieldHeadNames.RGB: rgb})
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
class TorchNerfactoField(Field):
|
||||
"""
|
||||
PyTorch implementation of the compound field.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aabb: Tensor,
|
||||
num_images: int,
|
||||
position_encoding: Encoding = HashEncoding(),
|
||||
direction_encoding: Encoding = SHEncoding(),
|
||||
base_mlp_num_layers: int = 3,
|
||||
base_mlp_layer_width: int = 64,
|
||||
head_mlp_num_layers: int = 2,
|
||||
head_mlp_layer_width: int = 32,
|
||||
appearance_embedding_dim: int = 40,
|
||||
skip_connections: Tuple = (4,),
|
||||
field_heads: Tuple[FieldHead] = (RGBFieldHead(),),
|
||||
spatial_distortion: SpatialDistortion = SceneContraction(),
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
self.spatial_distortion = spatial_distortion
|
||||
self.num_images = num_images
|
||||
self.appearance_embedding_dim = appearance_embedding_dim
|
||||
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
|
||||
|
||||
self.position_encoding = position_encoding
|
||||
self.direction_encoding = direction_encoding
|
||||
|
||||
self.mlp_base = MLP(
|
||||
in_dim=self.position_encoding.get_out_dim(),
|
||||
num_layers=base_mlp_num_layers,
|
||||
layer_width=base_mlp_layer_width,
|
||||
skip_connections=skip_connections,
|
||||
out_activation=nn.ReLU(),
|
||||
)
|
||||
|
||||
self.mlp_head = MLP(
|
||||
in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim() + self.appearance_embedding_dim,
|
||||
num_layers=head_mlp_num_layers,
|
||||
layer_width=head_mlp_layer_width,
|
||||
out_activation=nn.ReLU(),
|
||||
)
|
||||
|
||||
self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim())
|
||||
self.field_heads = nn.ModuleList(field_heads)
|
||||
for field_head in self.field_heads:
|
||||
field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore
|
||||
|
||||
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
|
||||
if self.spatial_distortion is not None:
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
positions = self.spatial_distortion(positions)
|
||||
else:
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
encoded_xyz = self.position_encoding(positions.view(-1, 3)).to(torch.float32)
|
||||
base_mlp_out = self.mlp_base(encoded_xyz).view(*ray_samples.frustums.shape, -1)
|
||||
density = self.field_output_density(base_mlp_out)
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(
|
||||
self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None
|
||||
) -> Dict[FieldHeadNames, Tensor]:
|
||||
outputs_shape = ray_samples.frustums.directions.shape[:-1]
|
||||
|
||||
if ray_samples.camera_indices is None:
|
||||
raise AttributeError("Camera indices are not provided.")
|
||||
camera_indices = ray_samples.camera_indices.squeeze()
|
||||
if self.training:
|
||||
embedded_appearance = self.embedding_appearance(camera_indices)
|
||||
else:
|
||||
embedded_appearance = torch.zeros(
|
||||
(*outputs_shape, self.appearance_embedding_dim),
|
||||
device=ray_samples.frustums.directions.device,
|
||||
)
|
||||
|
||||
outputs = {}
|
||||
for field_head in self.field_heads:
|
||||
directions = shift_directions_for_tcnn(ray_samples.frustums.directions)
|
||||
directions_flat = directions.view(-1, 3)
|
||||
encoded_dir = self.direction_encoding(directions_flat)
|
||||
mlp_out = self.mlp_head(
|
||||
torch.cat(
|
||||
[
|
||||
encoded_dir.view(-1, self.direction_encoding.get_out_dim()),
|
||||
density_embedding.view(-1, self.mlp_base.get_out_dim()), # type:ignore
|
||||
embedded_appearance.view(-1, self.appearance_embedding_dim),
|
||||
],
|
||||
dim=-1, # type:ignore
|
||||
)
|
||||
)
|
||||
outputs[field_head.field_head_name] = field_head(mlp_out).view(*outputs_shape, -1).to(directions)
|
||||
return outputs
|
||||
|
||||
|
||||
field_implementation_to_class: Dict[str, Type[Field]] = {"tcnn": TCNNNerfactoField, "torch": TorchNerfactoField}
|
||||
Reference in new issue
Block a user