Ability to use vanilla pytorch instead of tcnn for Nerfacto (#1936)

* started a package to easily swap out tcnn for pytorch

* trains with pytorch and not tcnn

* cleaned up imports

* making it easy to install on mac

* removed/reverted files

* merged main

* reverted changes

* updating config

* starting to add nerfacto to tests

* fixxing default value

* fixed default implementation

* defaults to torch if tcnn and not installed

* update log

* add comment

* resolved pr comments

* fix

* update default

---------

Co-authored-by: Ethan Weber <ethanweber@eecs-128-32-175-250.dhcp.berkeley.edu>
This commit is contained in:
Ethan WeberandEthan Weber authored and GitHub committed 2023-05-25 14:55:42 -07:00
1 parent 400ab5a313
commit bb6925bac6
19 files changed
+295 -296

No files matched your search

+23 -34
View File
@@ -17,24 +17,19 @@ Proposal network field.
"""
from typing import Optional, Tuple
from typing import Literal, Optional, Tuple
import numpy as np
import torch
from torch import Tensor
from torch import Tensor, nn
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.field_components.activations import trunc_exp
from nerfstudio.field_components.encodings import HashEncoding
from nerfstudio.field_components.mlp import MLP
from nerfstudio.field_components.spatial_distortions import SpatialDistortion
from nerfstudio.fields.base_field import Field
try:
import tinycudann as tcnn
except ImportError:
# tinycudann module doesn't exist
pass
class HashMLPDensityField(Field):
"""A lightweight density field module.
@@ -61,45 +56,39 @@ class HashMLPDensityField(Field):
base_res: int = 16,
log2_hashmap_size: int = 18,
features_per_level: int = 2,
implementation: Literal["tcnn", "torch"] = "torch",
) -> None:
super().__init__()
self.register_buffer("aabb", aabb)
self.spatial_distortion = spatial_distortion
self.use_linear = use_linear
growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
self.register_buffer("max_res", torch.tensor(max_res))
self.register_buffer("num_levels", torch.tensor(num_levels))
self.register_buffer("log2_hashmap_size", torch.tensor(log2_hashmap_size))
config = {
"encoding": {
"otype": "HashGrid",
"n_levels": num_levels,
"n_features_per_level": features_per_level,
"log2_hashmap_size": log2_hashmap_size,
"base_resolution": base_res,
"per_level_scale": growth_factor,
},
"network": {
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "None",
"n_neurons": hidden_dim,
"n_hidden_layers": num_layers - 1,
},
}
self.encoding = HashEncoding(
num_levels=num_levels,
min_res=base_res,
max_res=max_res,
log2_hashmap_size=log2_hashmap_size,
features_per_level=features_per_level,
implementation=implementation,
)
if not self.use_linear:
self.mlp_base = tcnn.NetworkWithInputEncoding(
n_input_dims=3,
n_output_dims=1,
encoding_config=config["encoding"],
network_config=config["network"],
network = MLP(
in_dim=self.encoding.get_out_dim(),
num_layers=num_layers,
layer_width=hidden_dim,
out_dim=1,
activation=nn.ReLU(),
out_activation=None,
implementation=implementation,
)
self.mlp_base = torch.nn.Sequential(self.encoding, network)
else:
self.encoding = tcnn.Encoding(n_input_dims=3, encoding_config=config["encoding"])
self.linear = torch.nn.Linear(self.encoding.n_output_dims, 1)
self.linear = torch.nn.Linear(self.encoding.get_out_dim(), 1)
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, None]:
if self.spatial_distortion is not None:
+1
View File
@@ -60,6 +60,7 @@ class TCNNInstantNGPField(Field):
num_levels: number of levels of the hashmap for the base mlp
log2_hashmap_size: size of the hashmap for the base mlp
max_res: maximum resolution of the hashmap for the base mlp
mode: which implementation to use for the field
"""
def __init__(
+64 -190
View File
@@ -17,44 +17,30 @@ Field for compound nerf model, adds scene contraction and image embeddings to in
"""
from typing import Dict, Optional, Tuple, Type
from typing import Dict, Literal, Optional, Tuple
import numpy as np
import torch
from torch import Tensor, nn
from torch.nn.parameter import Parameter
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.field_components.activations import trunc_exp
from nerfstudio.field_components.embedding import Embedding
from nerfstudio.field_components.encodings import Encoding, HashEncoding, SHEncoding
from nerfstudio.field_components.encodings import HashEncoding, NeRFEncoding, SHEncoding
from nerfstudio.field_components.field_heads import (
DensityFieldHead,
FieldHead,
FieldHeadNames,
PredNormalsFieldHead,
RGBFieldHead,
SemanticFieldHead,
TransientDensityFieldHead,
TransientRGBFieldHead,
UncertaintyFieldHead,
)
from nerfstudio.field_components.mlp import MLP
from nerfstudio.field_components.spatial_distortions import (
SceneContraction,
SpatialDistortion,
)
from nerfstudio.field_components.spatial_distortions import SpatialDistortion
from nerfstudio.fields.base_field import Field, shift_directions_for_tcnn
try:
import tinycudann as tcnn
except ModuleNotFoundError:
# tinycudann module doesn't exist
pass
class TCNNNerfactoField(Field):
class NerfactoField(Field):
"""Compound Field that uses TCNN
Args:
@@ -105,6 +91,7 @@ class TCNNNerfactoField(Field):
use_pred_normals: bool = False,
use_average_appearance_embedding: bool = False,
spatial_distortion: Optional[SpatialDistortion] = None,
implementation: Literal["tcnn", "torch"] = "tcnn",
) -> None:
super().__init__()
@@ -127,102 +114,88 @@ class TCNNNerfactoField(Field):
base_res: int = 16
features_per_level: int = 2
growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
self.direction_encoding = tcnn.Encoding(
n_input_dims=3,
encoding_config={
"otype": "SphericalHarmonics",
"degree": 4,
},
self.direction_encoding = SHEncoding(
levels=4,
implementation=implementation,
)
self.position_encoding = tcnn.Encoding(
n_input_dims=3,
encoding_config={"otype": "Frequency", "n_frequencies": 2},
self.position_encoding = NeRFEncoding(
in_dim=3, num_frequencies=2, min_freq_exp=0, max_freq_exp=2 - 1, implementation=implementation
)
self.mlp_base = tcnn.NetworkWithInputEncoding(
n_input_dims=3,
n_output_dims=1 + self.geo_feat_dim,
encoding_config={
"otype": "HashGrid",
"n_levels": num_levels,
"n_features_per_level": features_per_level,
"log2_hashmap_size": log2_hashmap_size,
"base_resolution": base_res,
"per_level_scale": growth_factor,
},
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "None",
"n_neurons": hidden_dim,
"n_hidden_layers": num_layers - 1,
},
encoder = HashEncoding(
num_levels=num_levels,
min_res=base_res,
max_res=max_res,
log2_hashmap_size=log2_hashmap_size,
features_per_level=features_per_level,
implementation=implementation,
)
network = MLP(
in_dim=encoder.get_out_dim(),
num_layers=num_layers,
layer_width=hidden_dim,
out_dim=1 + self.geo_feat_dim,
activation=nn.ReLU(),
out_activation=None,
implementation=implementation,
)
self.mlp_base = torch.nn.Sequential(encoder, network)
# transients
if self.use_transient_embedding:
self.transient_embedding_dim = transient_embedding_dim
self.embedding_transient = Embedding(self.num_images, self.transient_embedding_dim)
self.mlp_transient = tcnn.Network(
n_input_dims=self.geo_feat_dim + self.transient_embedding_dim,
n_output_dims=hidden_dim_transient,
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "None",
"n_neurons": hidden_dim_transient,
"n_hidden_layers": num_layers_transient - 1,
},
self.mlp_transient = MLP(
in_dim=self.geo_feat_dim + self.transient_embedding_dim,
num_layers=num_layers_transient,
layer_width=hidden_dim_transient,
out_dim=hidden_dim_transient,
activation=nn.ReLU(),
out_activation=None,
implementation=implementation,
)
self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.n_output_dims)
self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.n_output_dims)
self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.n_output_dims)
self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.get_out_dim())
self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim())
self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim())
# semantics
if self.use_semantics:
self.mlp_semantics = tcnn.Network(
n_input_dims=self.geo_feat_dim,
n_output_dims=hidden_dim_transient,
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "None",
"n_neurons": 64,
"n_hidden_layers": 1,
},
self.mlp_semantics = MLP(
in_dim=self.geo_feat_dim,
num_layers=2,
layer_width=64,
out_dim=hidden_dim_transient,
activation=nn.ReLU(),
out_activation=None,
implementation=implementation,
)
self.field_head_semantics = SemanticFieldHead(
in_dim=self.mlp_semantics.n_output_dims, num_classes=num_semantic_classes
in_dim=self.mlp_semantics.get_out_dim(), num_classes=num_semantic_classes
)
# predicted normals
if self.use_pred_normals:
self.mlp_pred_normals = tcnn.Network(
n_input_dims=self.geo_feat_dim + self.position_encoding.n_output_dims,
n_output_dims=hidden_dim_transient,
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "None",
"n_neurons": 64,
"n_hidden_layers": 2,
},
self.mlp_pred_normals = MLP(
in_dim=self.geo_feat_dim + self.position_encoding.get_out_dim(),
num_layers=3,
layer_width=64,
out_dim=hidden_dim_transient,
activation=nn.ReLU(),
out_activation=None,
implementation=implementation,
)
self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.n_output_dims)
self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.get_out_dim())
self.mlp_head = tcnn.Network(
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim,
n_output_dims=3,
network_config={
"otype": "FullyFusedMLP",
"activation": "ReLU",
"output_activation": "Sigmoid",
"n_neurons": hidden_dim_color,
"n_hidden_layers": num_layers_color - 1,
},
self.mlp_head = MLP(
in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim,
num_layers=num_layers_color,
layer_width=hidden_dim_color,
out_dim=3,
activation=nn.ReLU(),
out_activation=nn.Sigmoid(),
implementation=implementation,
)
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
@@ -324,102 +297,3 @@ class TCNNNerfactoField(Field):
outputs.update({FieldHeadNames.RGB: rgb})
return outputs
class TorchNerfactoField(Field):
"""
PyTorch implementation of the compound field.
"""
def __init__(
self,
aabb: Tensor,
num_images: int,
position_encoding: Encoding = HashEncoding(),
direction_encoding: Encoding = SHEncoding(),
base_mlp_num_layers: int = 3,
base_mlp_layer_width: int = 64,
head_mlp_num_layers: int = 2,
head_mlp_layer_width: int = 32,
appearance_embedding_dim: int = 40,
skip_connections: Tuple = (4,),
field_heads: Tuple[FieldHead] = (RGBFieldHead(),),
spatial_distortion: SpatialDistortion = SceneContraction(),
) -> None:
super().__init__()
self.aabb = Parameter(aabb, requires_grad=False)
self.spatial_distortion = spatial_distortion
self.num_images = num_images
self.appearance_embedding_dim = appearance_embedding_dim
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
self.position_encoding = position_encoding
self.direction_encoding = direction_encoding
self.mlp_base = MLP(
in_dim=self.position_encoding.get_out_dim(),
num_layers=base_mlp_num_layers,
layer_width=base_mlp_layer_width,
skip_connections=skip_connections,
out_activation=nn.ReLU(),
)
self.mlp_head = MLP(
in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim() + self.appearance_embedding_dim,
num_layers=head_mlp_num_layers,
layer_width=head_mlp_layer_width,
out_activation=nn.ReLU(),
)
self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim())
self.field_heads = nn.ModuleList(field_heads)
for field_head in self.field_heads:
field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
if self.spatial_distortion is not None:
positions = ray_samples.frustums.get_positions()
positions = self.spatial_distortion(positions)
else:
positions = ray_samples.frustums.get_positions()
encoded_xyz = self.position_encoding(positions.view(-1, 3)).to(torch.float32)
base_mlp_out = self.mlp_base(encoded_xyz).view(*ray_samples.frustums.shape, -1)
density = self.field_output_density(base_mlp_out)
return density, base_mlp_out
def get_outputs(
self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None
) -> Dict[FieldHeadNames, Tensor]:
outputs_shape = ray_samples.frustums.directions.shape[:-1]
if ray_samples.camera_indices is None:
raise AttributeError("Camera indices are not provided.")
camera_indices = ray_samples.camera_indices.squeeze()
if self.training:
embedded_appearance = self.embedding_appearance(camera_indices)
else:
embedded_appearance = torch.zeros(
(*outputs_shape, self.appearance_embedding_dim),
device=ray_samples.frustums.directions.device,
)
outputs = {}
for field_head in self.field_heads:
directions = shift_directions_for_tcnn(ray_samples.frustums.directions)
directions_flat = directions.view(-1, 3)
encoded_dir = self.direction_encoding(directions_flat)
mlp_out = self.mlp_head(
torch.cat(
[
encoded_dir.view(-1, self.direction_encoding.get_out_dim()),
density_embedding.view(-1, self.mlp_base.get_out_dim()), # type:ignore
embedded_appearance.view(-1, self.appearance_embedding_dim),
],
dim=-1, # type:ignore
)
)
outputs[field_head.field_head_name] = field_head(mlp_out).view(*outputs_shape, -1).to(directions)
return outputs
field_implementation_to_class: Dict[str, Type[Field]] = {"tcnn": TCNNNerfactoField, "torch": TorchNerfactoField}