diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index e167ce567..4f1d82238 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -64,14 +64,16 @@ class MachineConfig(PrintableConfig): seed: int = 42 """random seed initialization""" - num_gpus: int = 1 - """total number of gpus available for train/eval""" + num_devices: int = 1 + """total number of devices (e.g., gpus) available for train/eval""" num_machines: int = 1 """total number of distributed machines available (for DDP)""" machine_rank: int = 0 """current machine's rank (for DDP)""" dist_url: str = "auto" """distributed connection point (for DDP)""" + device_type: Literal["cpu", "cuda", "mps"] = "cuda" + """device type to use for training""" @dataclass diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index e786b9ab2..a69c8d68e 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -119,7 +119,9 @@ class Trainer: self.config = config self.local_rank = local_rank self.world_size = world_size - self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}" + self.device: TORCH_DEVICE = config.machine.device_type + if self.device == "cuda": + self.device += f":{local_rank}" self.mixed_precision: bool = self.config.mixed_precision self.use_grad_scaler: bool = self.mixed_precision or self.config.use_grad_scaler self.training_state: Literal["training", "paused", "completed"] = "training" @@ -459,9 +461,10 @@ class Trainer: """ self.optimizers.zero_grad_all() - cpu_or_cuda_str: str = self.device.split(":")[0] - with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision): + device_type: str = self.device.split(":")[0] if "cuda" in self.device else "cpu" + + with torch.autocast(device_type=device_type, enabled=self.mixed_precision): _, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step) loss = functools.reduce(torch.add, loss_dict.values()) self.grad_scaler.scale(loss).backward() # type: ignore diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py index 71d3e4a6c..48ec17556 100644 --- a/nerfstudio/field_components/encodings.py +++ b/nerfstudio/field_components/encodings.py @@ -109,7 +109,13 @@ class NeRFEncoding(Encoding): """ def __init__( - self, in_dim: int, num_frequencies: int, min_freq_exp: float, max_freq_exp: float, include_input: bool = False + self, + in_dim: int, + num_frequencies: int, + min_freq_exp: float, + max_freq_exp: float, + include_input: bool = False, + implementation: Literal["tcnn", "torch"] = "torch", ) -> None: super().__init__(in_dim) @@ -118,6 +124,18 @@ class NeRFEncoding(Encoding): self.max_freq = max_freq_exp self.include_input = include_input + self.tcnn_encoding = None + if implementation == "tcnn" and not TCNN_EXISTS: + print_tcnn_speed_warning("NeRFEncoding") + elif implementation == "tcnn": + encoding_config = {"otype": "Frequency", "n_frequencies": num_frequencies} + assert min_freq_exp == 0, "tcnn only supports min_freq_exp = 0" + assert max_freq_exp == num_frequencies - 1, "tcnn only supports max_freq_exp = num_frequencies - 1" + self.tcnn_encoding = tcnn.Encoding( + n_input_dims=in_dim, + encoding_config=encoding_config, + ) + def get_out_dim(self) -> int: if self.in_dim is None: raise ValueError("Input dimension has not been set") @@ -126,7 +144,7 @@ class NeRFEncoding(Encoding): out_dim += self.in_dim return out_dim - def forward( + def pytorch_fwd( self, in_tensor: Float[Tensor, "*bs input_dim"], covs: Optional[Float[Tensor, "*bs input_dim input_dim"]] = None, @@ -158,6 +176,13 @@ class NeRFEncoding(Encoding): encoded_inputs = torch.cat([encoded_inputs, in_tensor], dim=-1) return encoded_inputs + def forward( + self, in_tensor: Float[Tensor, "*bs input_dim"], covs: Optional[Float[Tensor, "*bs input_dim input_dim"]] = None + ) -> Float[Tensor, "*bs output_dim"]: + if self.tcnn_encoding is not None: + return self.tcnn_encoding(in_tensor) + return self.pytorch_fwd(in_tensor, covs) + class RFFEncoding(Encoding): """Random Fourier Feature encoding. Supports integrated encodings. @@ -239,7 +264,7 @@ class HashEncoding(Encoding): log2_hashmap_size: int = 19, features_per_level: int = 2, hash_init_scale: float = 0.001, - implementation: Literal["tcnn", "torch"] = "tcnn", + implementation: Literal["tcnn", "torch"] = "torch", interpolation: Optional[Literal["Nearest", "Linear", "Smoothstep"]] = None, ) -> None: super().__init__(in_dim=3) @@ -258,7 +283,7 @@ class HashEncoding(Encoding): self.hash_table = nn.Parameter(self.hash_table) self.tcnn_encoding = None - if not TCNN_EXISTS and implementation == "tcnn": + if implementation == "tcnn" and not TCNN_EXISTS: print_tcnn_speed_warning("HashEncoding") elif implementation == "tcnn": encoding_config = { @@ -277,7 +302,7 @@ class HashEncoding(Encoding): encoding_config=encoding_config, ) - if not TCNN_EXISTS or self.tcnn_encoding is None: + if self.tcnn_encoding is None: assert ( interpolation is None or interpolation == "Linear" ), f"interpolation '{interpolation}' is not supported for torch encoding backend" @@ -348,7 +373,7 @@ class HashEncoding(Encoding): return torch.flatten(encoded_value, start_dim=-2, end_dim=-1) # [..., num_levels * features_per_level] def forward(self, in_tensor: Float[Tensor, "*bs input_dim"]) -> Float[Tensor, "*bs output_dim"]: - if TCNN_EXISTS and self.tcnn_encoding is not None: + if self.tcnn_encoding is not None: return self.tcnn_encoding(in_tensor) return self.pytorch_fwd(in_tensor) @@ -647,7 +672,7 @@ class SHEncoding(Encoding): levels: Number of spherical harmonic levels to encode. """ - def __init__(self, levels: int = 4) -> None: + def __init__(self, levels: int = 4, implementation: Literal["tcnn", "torch"] = "torch") -> None: super().__init__(in_dim=3) if levels <= 0 or levels > 4: @@ -655,9 +680,28 @@ class SHEncoding(Encoding): self.levels = levels + self.tcnn_encoding = None + if implementation == "tcnn" and not TCNN_EXISTS: + print_tcnn_speed_warning("SHEncoding") + elif implementation == "tcnn": + encoding_config = { + "otype": "SphericalHarmonics", + "degree": levels, + } + self.tcnn_encoding = tcnn.Encoding( + n_input_dims=3, + encoding_config=encoding_config, + ) + def get_out_dim(self) -> int: return self.levels**2 @torch.no_grad() - def forward(self, in_tensor: Float[Tensor, "*bs input_dim"]) -> Float[Tensor, "*bs output_dim"]: + def pytorch_fwd(self, in_tensor: Float[Tensor, "*bs input_dim"]) -> Float[Tensor, "*bs output_dim"]: + """Forward pass using pytorch. Significantly slower than TCNN implementation.""" return components_from_spherical_harmonics(levels=self.levels, directions=in_tensor) + + def forward(self, in_tensor: Float[Tensor, "*bs input_dim"]) -> Float[Tensor, "*bs output_dim"]: + if self.tcnn_encoding is not None: + return self.tcnn_encoding(in_tensor) + return self.pytorch_fwd(in_tensor) diff --git a/nerfstudio/field_components/mlp.py b/nerfstudio/field_components/mlp.py index c51ed5f3b..2d766e233 100644 --- a/nerfstudio/field_components/mlp.py +++ b/nerfstudio/field_components/mlp.py @@ -15,13 +15,51 @@ """ Multi Layer Perceptron """ -from typing import Optional, Set, Tuple +from typing import Literal, Optional, Set, Tuple, Union import torch from jaxtyping import Float from torch import Tensor, nn from nerfstudio.field_components.base_field_component import FieldComponent +from nerfstudio.utils.printing import print_tcnn_speed_warning + +from nerfstudio.utils.rich_utils import CONSOLE + +try: + import tinycudann as tcnn + + TCNN_EXISTS = True +except ModuleNotFoundError: + TCNN_EXISTS = False + + +def activation_to_tcnn_string(activation: Union[nn.Module, None]) -> str: + """Converts a torch.nn activation function to a string that can be used to + initialize a TCNN activation function. + + Args: + activation: torch.nn activation function + Returns: + str: TCNN activation function string + """ + + if isinstance(activation, nn.ReLU): + return "ReLU" + if isinstance(activation, nn.LeakyReLU): + return "Leaky ReLU" + if isinstance(activation, nn.Sigmoid): + return "Sigmoid" + if isinstance(activation, nn.Softplus): + return "Softplus" + if isinstance(activation, nn.Tanh): + return "Tanh" + if isinstance(activation, type(None)): + return "None" + tcnn_documentation_url = "https://github.com/NVlabs/tiny-cuda-nn/blob/master/DOCUMENTATION.md#activation-functions" + raise ValueError( + f"TCNN activation {activation} not supported for now.\nSee {tcnn_documentation_url} for TCNN documentation." + ) class MLP(FieldComponent): @@ -45,6 +83,7 @@ class MLP(FieldComponent): skip_connections: Optional[Tuple[int]] = None, activation: Optional[nn.Module] = nn.ReLU(), out_activation: Optional[nn.Module] = None, + implementation: Literal["tcnn", "torch"] = "torch", ) -> None: super().__init__() self.in_dim = in_dim @@ -57,7 +96,43 @@ class MLP(FieldComponent): self.activation = activation self.out_activation = out_activation self.net = None - self.build_nn_modules() + + self.tcnn_encoding = None + if implementation == "torch": + self.build_nn_modules() + elif implementation == "tcnn" and not TCNN_EXISTS: + print_tcnn_speed_warning("MLP") + elif implementation == "tcnn": + activation_str = activation_to_tcnn_string(activation) + output_activation_str = activation_to_tcnn_string(out_activation) + if layer_width in [16, 32, 64, 128]: + network_config = { + "otype": "FullyFusedMLP", + "activation": activation_str, + "output_activation": output_activation_str, + "n_neurons": layer_width, + "n_hidden_layers": num_layers - 1, + } + else: + CONSOLE.line() + CONSOLE.print("[bold yellow]WARNING: Using slower TCNN CutlassMLP instead of TCNN FullyFusedMLP") + CONSOLE.print( + "[bold yellow]Use layer width of 16, 32, 64, or 128 to use the faster TCNN FullyFusedMLP." + ) + CONSOLE.line() + network_config = { + "otype": "CutlassMLP", + "activation": activation_str, + "output_activation": output_activation_str, + "n_neurons": layer_width, + "n_hidden_layers": num_layers - 1, + } + + self.tcnn_encoding = tcnn.Network( + n_input_dims=in_dim, + n_output_dims=out_dim, + network_config=network_config, + ) def build_nn_modules(self) -> None: """Initialize multi-layer perceptron.""" @@ -76,7 +151,7 @@ class MLP(FieldComponent): layers.append(nn.Linear(self.layer_width, self.out_dim)) self.layers = nn.ModuleList(layers) - def forward(self, in_tensor: Float[Tensor, "*bs in_dim"]) -> Float[Tensor, "*bs out_dim"]: + def pytorch_fwd(self, in_tensor: Float[Tensor, "*bs in_dim"]) -> Float[Tensor, "*bs out_dim"]: """Process input with a multilayer perceptron. Args: @@ -96,3 +171,8 @@ class MLP(FieldComponent): if self.out_activation is not None: x = self.out_activation(x) return x + + def forward(self, in_tensor: Float[Tensor, "*bs in_dim"]) -> Float[Tensor, "*bs out_dim"]: + if self.tcnn_encoding is not None: + return self.tcnn_encoding(in_tensor) + return self.pytorch_fwd(in_tensor) diff --git a/nerfstudio/fields/density_fields.py b/nerfstudio/fields/density_fields.py index 47eec223a..75fc5dc78 100644 --- a/nerfstudio/fields/density_fields.py +++ b/nerfstudio/fields/density_fields.py @@ -17,24 +17,19 @@ Proposal network field. """ -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple -import numpy as np import torch -from torch import Tensor +from torch import Tensor, nn from nerfstudio.cameras.rays import RaySamples from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp +from nerfstudio.field_components.encodings import HashEncoding +from nerfstudio.field_components.mlp import MLP from nerfstudio.field_components.spatial_distortions import SpatialDistortion from nerfstudio.fields.base_field import Field -try: - import tinycudann as tcnn -except ImportError: - # tinycudann module doesn't exist - pass - class HashMLPDensityField(Field): """A lightweight density field module. @@ -61,45 +56,39 @@ class HashMLPDensityField(Field): base_res: int = 16, log2_hashmap_size: int = 18, features_per_level: int = 2, + implementation: Literal["tcnn", "torch"] = "torch", ) -> None: super().__init__() self.register_buffer("aabb", aabb) self.spatial_distortion = spatial_distortion self.use_linear = use_linear - growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) self.register_buffer("max_res", torch.tensor(max_res)) self.register_buffer("num_levels", torch.tensor(num_levels)) self.register_buffer("log2_hashmap_size", torch.tensor(log2_hashmap_size)) - config = { - "encoding": { - "otype": "HashGrid", - "n_levels": num_levels, - "n_features_per_level": features_per_level, - "log2_hashmap_size": log2_hashmap_size, - "base_resolution": base_res, - "per_level_scale": growth_factor, - }, - "network": { - "otype": "FullyFusedMLP", - "activation": "ReLU", - "output_activation": "None", - "n_neurons": hidden_dim, - "n_hidden_layers": num_layers - 1, - }, - } + self.encoding = HashEncoding( + num_levels=num_levels, + min_res=base_res, + max_res=max_res, + log2_hashmap_size=log2_hashmap_size, + features_per_level=features_per_level, + implementation=implementation, + ) if not self.use_linear: - self.mlp_base = tcnn.NetworkWithInputEncoding( - n_input_dims=3, - n_output_dims=1, - encoding_config=config["encoding"], - network_config=config["network"], + network = MLP( + in_dim=self.encoding.get_out_dim(), + num_layers=num_layers, + layer_width=hidden_dim, + out_dim=1, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, ) + self.mlp_base = torch.nn.Sequential(self.encoding, network) else: - self.encoding = tcnn.Encoding(n_input_dims=3, encoding_config=config["encoding"]) - self.linear = torch.nn.Linear(self.encoding.n_output_dims, 1) + self.linear = torch.nn.Linear(self.encoding.get_out_dim(), 1) def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, None]: if self.spatial_distortion is not None: diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index 3941e7ec9..63b01eff6 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -60,6 +60,7 @@ class TCNNInstantNGPField(Field): num_levels: number of levels of the hashmap for the base mlp log2_hashmap_size: size of the hashmap for the base mlp max_res: maximum resolution of the hashmap for the base mlp + mode: which implementation to use for the field """ def __init__( diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index e828ac664..898d344d3 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -17,44 +17,30 @@ Field for compound nerf model, adds scene contraction and image embeddings to in """ -from typing import Dict, Optional, Tuple, Type +from typing import Dict, Literal, Optional, Tuple -import numpy as np import torch from torch import Tensor, nn -from torch.nn.parameter import Parameter from nerfstudio.cameras.rays import RaySamples from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp from nerfstudio.field_components.embedding import Embedding -from nerfstudio.field_components.encodings import Encoding, HashEncoding, SHEncoding +from nerfstudio.field_components.encodings import HashEncoding, NeRFEncoding, SHEncoding from nerfstudio.field_components.field_heads import ( - DensityFieldHead, - FieldHead, FieldHeadNames, PredNormalsFieldHead, - RGBFieldHead, SemanticFieldHead, TransientDensityFieldHead, TransientRGBFieldHead, UncertaintyFieldHead, ) from nerfstudio.field_components.mlp import MLP -from nerfstudio.field_components.spatial_distortions import ( - SceneContraction, - SpatialDistortion, -) +from nerfstudio.field_components.spatial_distortions import SpatialDistortion from nerfstudio.fields.base_field import Field, shift_directions_for_tcnn -try: - import tinycudann as tcnn -except ModuleNotFoundError: - # tinycudann module doesn't exist - pass - -class TCNNNerfactoField(Field): +class NerfactoField(Field): """Compound Field that uses TCNN Args: @@ -105,6 +91,7 @@ class TCNNNerfactoField(Field): use_pred_normals: bool = False, use_average_appearance_embedding: bool = False, spatial_distortion: Optional[SpatialDistortion] = None, + implementation: Literal["tcnn", "torch"] = "tcnn", ) -> None: super().__init__() @@ -127,102 +114,88 @@ class TCNNNerfactoField(Field): base_res: int = 16 features_per_level: int = 2 - growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) - self.direction_encoding = tcnn.Encoding( - n_input_dims=3, - encoding_config={ - "otype": "SphericalHarmonics", - "degree": 4, - }, + self.direction_encoding = SHEncoding( + levels=4, + implementation=implementation, ) - self.position_encoding = tcnn.Encoding( - n_input_dims=3, - encoding_config={"otype": "Frequency", "n_frequencies": 2}, + self.position_encoding = NeRFEncoding( + in_dim=3, num_frequencies=2, min_freq_exp=0, max_freq_exp=2 - 1, implementation=implementation ) - self.mlp_base = tcnn.NetworkWithInputEncoding( - n_input_dims=3, - n_output_dims=1 + self.geo_feat_dim, - encoding_config={ - "otype": "HashGrid", - "n_levels": num_levels, - "n_features_per_level": features_per_level, - "log2_hashmap_size": log2_hashmap_size, - "base_resolution": base_res, - "per_level_scale": growth_factor, - }, - network_config={ - "otype": "FullyFusedMLP", - "activation": "ReLU", - "output_activation": "None", - "n_neurons": hidden_dim, - "n_hidden_layers": num_layers - 1, - }, + encoder = HashEncoding( + num_levels=num_levels, + min_res=base_res, + max_res=max_res, + log2_hashmap_size=log2_hashmap_size, + features_per_level=features_per_level, + implementation=implementation, ) + network = MLP( + in_dim=encoder.get_out_dim(), + num_layers=num_layers, + layer_width=hidden_dim, + out_dim=1 + self.geo_feat_dim, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.mlp_base = torch.nn.Sequential(encoder, network) # transients if self.use_transient_embedding: self.transient_embedding_dim = transient_embedding_dim self.embedding_transient = Embedding(self.num_images, self.transient_embedding_dim) - self.mlp_transient = tcnn.Network( - n_input_dims=self.geo_feat_dim + self.transient_embedding_dim, - n_output_dims=hidden_dim_transient, - network_config={ - "otype": "FullyFusedMLP", - "activation": "ReLU", - "output_activation": "None", - "n_neurons": hidden_dim_transient, - "n_hidden_layers": num_layers_transient - 1, - }, + self.mlp_transient = MLP( + in_dim=self.geo_feat_dim + self.transient_embedding_dim, + num_layers=num_layers_transient, + layer_width=hidden_dim_transient, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, ) - self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.n_output_dims) - self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.n_output_dims) - self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.n_output_dims) + self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.get_out_dim()) + self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim()) + self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim()) # semantics if self.use_semantics: - self.mlp_semantics = tcnn.Network( - n_input_dims=self.geo_feat_dim, - n_output_dims=hidden_dim_transient, - network_config={ - "otype": "FullyFusedMLP", - "activation": "ReLU", - "output_activation": "None", - "n_neurons": 64, - "n_hidden_layers": 1, - }, + self.mlp_semantics = MLP( + in_dim=self.geo_feat_dim, + num_layers=2, + layer_width=64, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, ) self.field_head_semantics = SemanticFieldHead( - in_dim=self.mlp_semantics.n_output_dims, num_classes=num_semantic_classes + in_dim=self.mlp_semantics.get_out_dim(), num_classes=num_semantic_classes ) # predicted normals if self.use_pred_normals: - self.mlp_pred_normals = tcnn.Network( - n_input_dims=self.geo_feat_dim + self.position_encoding.n_output_dims, - n_output_dims=hidden_dim_transient, - network_config={ - "otype": "FullyFusedMLP", - "activation": "ReLU", - "output_activation": "None", - "n_neurons": 64, - "n_hidden_layers": 2, - }, + self.mlp_pred_normals = MLP( + in_dim=self.geo_feat_dim + self.position_encoding.get_out_dim(), + num_layers=3, + layer_width=64, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, ) - self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.n_output_dims) + self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.get_out_dim()) - self.mlp_head = tcnn.Network( - n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim, - n_output_dims=3, - network_config={ - "otype": "FullyFusedMLP", - "activation": "ReLU", - "output_activation": "Sigmoid", - "n_neurons": hidden_dim_color, - "n_hidden_layers": num_layers_color - 1, - }, + self.mlp_head = MLP( + in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim, + num_layers=num_layers_color, + layer_width=hidden_dim_color, + out_dim=3, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, ) def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: @@ -324,102 +297,3 @@ class TCNNNerfactoField(Field): outputs.update({FieldHeadNames.RGB: rgb}) return outputs - - -class TorchNerfactoField(Field): - """ - PyTorch implementation of the compound field. - """ - - def __init__( - self, - aabb: Tensor, - num_images: int, - position_encoding: Encoding = HashEncoding(), - direction_encoding: Encoding = SHEncoding(), - base_mlp_num_layers: int = 3, - base_mlp_layer_width: int = 64, - head_mlp_num_layers: int = 2, - head_mlp_layer_width: int = 32, - appearance_embedding_dim: int = 40, - skip_connections: Tuple = (4,), - field_heads: Tuple[FieldHead] = (RGBFieldHead(),), - spatial_distortion: SpatialDistortion = SceneContraction(), - ) -> None: - super().__init__() - self.aabb = Parameter(aabb, requires_grad=False) - self.spatial_distortion = spatial_distortion - self.num_images = num_images - self.appearance_embedding_dim = appearance_embedding_dim - self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) - - self.position_encoding = position_encoding - self.direction_encoding = direction_encoding - - self.mlp_base = MLP( - in_dim=self.position_encoding.get_out_dim(), - num_layers=base_mlp_num_layers, - layer_width=base_mlp_layer_width, - skip_connections=skip_connections, - out_activation=nn.ReLU(), - ) - - self.mlp_head = MLP( - in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim() + self.appearance_embedding_dim, - num_layers=head_mlp_num_layers, - layer_width=head_mlp_layer_width, - out_activation=nn.ReLU(), - ) - - self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim()) - self.field_heads = nn.ModuleList(field_heads) - for field_head in self.field_heads: - field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore - - def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: - if self.spatial_distortion is not None: - positions = ray_samples.frustums.get_positions() - positions = self.spatial_distortion(positions) - else: - positions = ray_samples.frustums.get_positions() - encoded_xyz = self.position_encoding(positions.view(-1, 3)).to(torch.float32) - base_mlp_out = self.mlp_base(encoded_xyz).view(*ray_samples.frustums.shape, -1) - density = self.field_output_density(base_mlp_out) - return density, base_mlp_out - - def get_outputs( - self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None - ) -> Dict[FieldHeadNames, Tensor]: - outputs_shape = ray_samples.frustums.directions.shape[:-1] - - if ray_samples.camera_indices is None: - raise AttributeError("Camera indices are not provided.") - camera_indices = ray_samples.camera_indices.squeeze() - if self.training: - embedded_appearance = self.embedding_appearance(camera_indices) - else: - embedded_appearance = torch.zeros( - (*outputs_shape, self.appearance_embedding_dim), - device=ray_samples.frustums.directions.device, - ) - - outputs = {} - for field_head in self.field_heads: - directions = shift_directions_for_tcnn(ray_samples.frustums.directions) - directions_flat = directions.view(-1, 3) - encoded_dir = self.direction_encoding(directions_flat) - mlp_out = self.mlp_head( - torch.cat( - [ - encoded_dir.view(-1, self.direction_encoding.get_out_dim()), - density_embedding.view(-1, self.mlp_base.get_out_dim()), # type:ignore - embedded_appearance.view(-1, self.appearance_embedding_dim), - ], - dim=-1, # type:ignore - ) - ) - outputs[field_head.field_head_name] = field_head(mlp_out).view(*outputs_shape, -1).to(directions) - return outputs - - -field_implementation_to_class: Dict[str, Type[Field]] = {"tcnn": TCNNNerfactoField, "torch": TorchNerfactoField} diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index cd589cad3..604f306e6 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -77,9 +77,9 @@ def outer( def lossfun_outer( - t: Float[Tensor, "*batch num_samples+1"], + t: Float[Tensor, "*batch num_samples_1"], w: Float[Tensor, "*batch num_samples"], - t_env: Float[Tensor, "*batch num_samples+1"], + t_env: Float[Tensor, "*batch num_samples_1"], w_env: Float[Tensor, "*batch num_samples"], ): """ diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index 59d6d7b82..c0b4b883d 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -17,7 +17,7 @@ Collection of sampling strategies """ from abc import abstractmethod -from typing import Callable, List, Optional, Protocol, Tuple, Union +from typing import Any, Callable, List, Optional, Protocol, Tuple, Union import torch from jaxtyping import Float @@ -42,10 +42,10 @@ class Sampler(nn.Module): self.num_samples = num_samples @abstractmethod - def generate_ray_samples(self) -> RaySamples: + def generate_ray_samples(self) -> Any: """Generate Ray Samples""" - def forward(self, *args, **kwargs) -> RaySamples: + def forward(self, *args, **kwargs) -> Any: """Generate ray samples""" return self.generate_ray_samples(*args, **kwargs) diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 046167f72..1feb22adb 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -21,7 +21,7 @@ from __future__ import annotations from abc import abstractmethod from collections import defaultdict from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List, Optional, Tuple, Type, Union import torch from torch import nn @@ -115,7 +115,7 @@ class Model(nn.Module): """ @abstractmethod - def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: + def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]: """Takes in a Ray Bundle and returns a dictionary of outputs. Args: @@ -126,7 +126,7 @@ class Model(nn.Module): Outputs of model. (ie. rendered colors) """ - def forward(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: + def forward(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]: """Run forward starting with a ray bundle. This outputs different things depending on the configuration of the model and whether or not the batch is provided (whether or not we are training basically) diff --git a/nerfstudio/models/base_surface_model.py b/nerfstudio/models/base_surface_model.py index b5e2d373a..f7af6e2c1 100644 --- a/nerfstudio/models/base_surface_model.py +++ b/nerfstudio/models/base_surface_model.py @@ -33,7 +33,7 @@ from nerfstudio.cameras.rays import RayBundle from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction -from nerfstudio.fields.nerfacto_field import TCNNNerfactoField +from nerfstudio.fields.nerfacto_field import NerfactoField from nerfstudio.fields.sdf_field import SDFFieldConfig from nerfstudio.fields.vanilla_nerf_field import NeRFField from nerfstudio.model_components.losses import ( @@ -124,7 +124,7 @@ class SurfaceModel(Model): # background model if self.config.background_model == "grid": - self.field_background = TCNNNerfactoField( + self.field_background = NerfactoField( self.scene_box.aabb, spatial_distortion=self.scene_contraction, num_images=self.num_train_data, diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index 1eff0c836..e8ac52b96 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -36,7 +36,7 @@ from nerfstudio.engine.callbacks import ( ) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction -from nerfstudio.fields.nerfacto_field import TCNNNerfactoField +from nerfstudio.fields.nerfacto_field import NerfactoField from nerfstudio.model_components.losses import MSELoss from nerfstudio.model_components.ray_samplers import VolumetricSampler from nerfstudio.model_components.renderers import ( @@ -94,7 +94,7 @@ class NGPModel(Model): """ config: InstantNGPModelConfig - field: TCNNNerfactoField + field: NerfactoField def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None: super().__init__(config=config, **kwargs) @@ -108,7 +108,7 @@ class NGPModel(Model): else: scene_contraction = SceneContraction(order=float("inf")) - self.field = TCNNNerfactoField( + self.field = NerfactoField( aabb=self.scene_box.aabb, num_images=self.num_train_data, log2_hashmap_size=self.config.log2_hashmap_size, diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 6d46c86c3..80ead5951 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -37,7 +37,7 @@ from nerfstudio.engine.callbacks import ( from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField -from nerfstudio.fields.nerfacto_field import TCNNNerfactoField +from nerfstudio.fields.nerfacto_field import NerfactoField from nerfstudio.model_components.losses import ( MSELoss, distortion_loss, @@ -130,6 +130,8 @@ class NerfactoModelConfig(ModelConfig): """Whether to disable scene contraction or not.""" use_gradient_scaling: bool = False """Use gradient scaler where the gradients are lower for points closer to the camera.""" + implementation: Literal["tcnn", "torch"] = "tcnn" + """Which implementation to use for the model.""" class NerfactoModel(Model): @@ -151,7 +153,7 @@ class NerfactoModel(Model): scene_contraction = SceneContraction(order=float("inf")) # Fields - self.field = TCNNNerfactoField( + self.field = NerfactoField( self.scene_box.aabb, hidden_dim=self.config.hidden_dim, num_levels=self.config.num_levels, @@ -163,6 +165,7 @@ class NerfactoModel(Model): num_images=self.num_train_data, use_pred_normals=self.config.predict_normals, use_average_appearance_embedding=self.config.use_average_appearance_embedding, + implementation=self.config.implementation, ) self.density_fns = [] @@ -172,7 +175,12 @@ class NerfactoModel(Model): if self.config.use_same_proposal_network: assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed." prop_net_args = self.config.proposal_net_args_list[0] - network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction, **prop_net_args) + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) self.proposal_networks.append(network) self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)]) else: @@ -182,6 +190,7 @@ class NerfactoModel(Model): self.scene_box.aabb, spatial_distortion=scene_contraction, **prop_net_args, + implementation=self.config.implementation, ) self.proposal_networks.append(network) self.density_fns.extend([network.density_fn for network in self.proposal_networks]) diff --git a/nerfstudio/models/neus_facto.py b/nerfstudio/models/neus_facto.py index 1f7104620..5c1a3ec9d 100644 --- a/nerfstudio/models/neus_facto.py +++ b/nerfstudio/models/neus_facto.py @@ -48,7 +48,7 @@ class NeuSFactoModelConfig(NeuSModelConfig): """NeusFacto Model Config""" _target: Type = field(default_factory=lambda: NeuSFactoModel) - num_proposal_samples_per_ray: Tuple[int, int] = (256, 96) + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) """Number of samples per ray for the proposal network.""" num_neus_samples_per_ray: int = 48 """Number of samples per ray for the nerf network.""" diff --git a/nerfstudio/models/semantic_nerfw.py b/nerfstudio/models/semantic_nerfw.py index a006ca3b2..c465e65b9 100644 --- a/nerfstudio/models/semantic_nerfw.py +++ b/nerfstudio/models/semantic_nerfw.py @@ -38,7 +38,7 @@ from nerfstudio.engine.callbacks import ( from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField -from nerfstudio.fields.nerfacto_field import TCNNNerfactoField +from nerfstudio.fields.nerfacto_field import NerfactoField from nerfstudio.model_components.losses import MSELoss, distortion_loss, interlevel_loss from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler from nerfstudio.model_components.renderers import ( @@ -90,7 +90,7 @@ class SemanticNerfWModel(Model): raise ValueError("Transient embedding is not fully working for semantic nerf-w.") # Fields - self.field = TCNNNerfactoField( + self.field = NerfactoField( self.scene_box.aabb, num_levels=self.config.num_levels, max_res=self.config.max_res, diff --git a/nerfstudio/scripts/train.py b/nerfstudio/scripts/train.py index 69b86c33a..cd5e73324 100644 --- a/nerfstudio/scripts/train.py +++ b/nerfstudio/scripts/train.py @@ -49,7 +49,7 @@ import random import socket import traceback from datetime import timedelta -from typing import Any, Callable, Optional +from typing import Any, Callable, Literal, Optional import numpy as np import torch @@ -104,11 +104,12 @@ def _distributed_worker( local_rank: int, main_func: Callable, world_size: int, - num_gpus_per_machine: int, + num_devices_per_machine: int, machine_rank: int, dist_url: str, config: TrainerConfig, timeout: timedelta = DEFAULT_TIMEOUT, + device_type: Literal["cpu", "cuda", "mps"] = "cuda", ) -> Any: """Spawned distributed worker that handles the initialization of process group and handles the training process on multiple processes. @@ -117,7 +118,7 @@ def _distributed_worker( local_rank: Current rank of process. main_func: Function that will be called by the distributed workers. world_size: Total number of gpus available. - num_gpus_per_machine: Number of GPUs per machine. + num_devices_per_machine: Number of GPUs per machine. machine_rank: Rank of this machine. dist_url: URL to connect to for distributed jobs, including protocol E.g., "tcp://127.0.0.1:8686". @@ -132,24 +133,24 @@ def _distributed_worker( Any: TODO: determine the return type """ assert torch.cuda.is_available(), "cuda is not available. Please check your installation." - global_rank = machine_rank * num_gpus_per_machine + local_rank + global_rank = machine_rank * num_devices_per_machine + local_rank dist.init_process_group( - backend="nccl", + backend="nccl" if device_type == "cuda" else "gloo", init_method=dist_url, world_size=world_size, rank=global_rank, timeout=timeout, ) assert comms.LOCAL_PROCESS_GROUP is None - num_machines = world_size // num_gpus_per_machine + num_machines = world_size // num_devices_per_machine for i in range(num_machines): - ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)) + ranks_on_i = list(range(i * num_devices_per_machine, (i + 1) * num_devices_per_machine)) pg = dist.new_group(ranks_on_i) if i == machine_rank: comms.LOCAL_PROCESS_GROUP = pg - assert num_gpus_per_machine <= torch.cuda.device_count() + assert num_devices_per_machine <= torch.cuda.device_count() output = main_func(local_rank, world_size, config, global_rank) comms.synchronize() dist.destroy_process_group() @@ -158,29 +159,32 @@ def _distributed_worker( def launch( main_func: Callable, - num_gpus_per_machine: int, + num_devices_per_machine: int, num_machines: int = 1, machine_rank: int = 0, dist_url: str = "auto", config: Optional[TrainerConfig] = None, timeout: timedelta = DEFAULT_TIMEOUT, + device_type: Literal["cpu", "cuda", "mps"] = "cuda", ) -> None: """Function that spawns multiple processes to call on main_func Args: main_func (Callable): function that will be called by the distributed workers - num_gpus_per_machine (int): number of GPUs per machine + num_devices_per_machine (int): number of GPUs per machine num_machines (int, optional): total number of machines machine_rank (int, optional): rank of this machine. dist_url (str, optional): url to connect to for distributed jobs. config (TrainerConfig, optional): config file specifying training regimen. timeout (timedelta, optional): timeout of the distributed workers. + device_type: type of device to use for training. """ assert config is not None - world_size = num_machines * num_gpus_per_machine - if world_size <= 1: - # world_size=0 uses one CPU in one process. - # world_size=1 uses one GPU in one process. + world_size = num_machines * num_devices_per_machine + if world_size == 0: + raise ValueError("world_size cannot be 0") + elif world_size == 1: + # uses one process try: main_func(local_rank=0, world_size=world_size, config=config) except KeyboardInterrupt: @@ -199,17 +203,9 @@ def launch( process_context = mp.spawn( _distributed_worker, - nprocs=num_gpus_per_machine, + nprocs=num_devices_per_machine, join=False, - args=( - main_func, - world_size, - num_gpus_per_machine, - machine_rank, - dist_url, - config, - timeout, - ), + args=(main_func, world_size, num_devices_per_machine, machine_rank, dist_url, config, timeout, device_type), ) # process_context won't be None because join=False, so it's okay to assert this # for Pylance reasons @@ -245,7 +241,8 @@ def main(config: TrainerConfig) -> None: launch( main_func=train_loop, - num_gpus_per_machine=config.machine.num_gpus, + num_devices_per_machine=config.machine.num_devices, + device_type=config.machine.device_type, num_machines=config.machine.num_machines, machine_rank=config.machine.machine_rank, dist_url=config.machine.dist_url, diff --git a/nerfstudio/utils/printing.py b/nerfstudio/utils/printing.py index 859a3dc1b..1fc4e99a8 100644 --- a/nerfstudio/utils/printing.py +++ b/nerfstudio/utils/printing.py @@ -19,10 +19,10 @@ from math import floor, log from nerfstudio.utils.rich_utils import CONSOLE -def print_tcnn_speed_warning(method_name: str): +def print_tcnn_speed_warning(module_name: str): """Prints a warning about the speed of the TCNN.""" CONSOLE.line() - CONSOLE.print(f"[bold yellow]WARNING: Using a slow implementation of {method_name}. ") + CONSOLE.print(f"[bold yellow]WARNING: Using a slow implementation for the {module_name} module. ") CONSOLE.print( "[bold yellow]:person_running: :person_running: " + "Install tcnn for speedups :person_running: :person_running:" diff --git a/tests/field_components/test_encodings.py b/tests/field_components/test_encodings.py index dc59b79ff..a7dbb7f2c 100644 --- a/tests/field_components/test_encodings.py +++ b/tests/field_components/test_encodings.py @@ -154,10 +154,9 @@ def test_tensor_hash_encoder(): ) assert encoder.get_out_dim() == out_dim - in_tensor = torch.zeros((10, 3)) - in_tensor[..., 1] = 1 - encoded = encoder(in_tensor) - assert encoded.shape == (10, out_dim) + in_tensor = torch.rand((10, 3)) + encoded_torch = encoder(in_tensor) + assert encoded_torch.shape == (10, out_dim) encoder = encodings.HashEncoding( num_levels=num_levels, @@ -165,10 +164,8 @@ def test_tensor_hash_encoder(): log2_hashmap_size=5, implementation="tcnn", ) - in_tensor = torch.zeros((10, 3)) - in_tensor[..., 1] = 1 - encoded = encoder(in_tensor) - assert encoded.shape == (10, out_dim) + encoded_tcnn = encoder(in_tensor).float().cpu() + assert encoded_tcnn.shape == (10, out_dim) def test_kplane_encoder(): diff --git a/tests/test_train.py b/tests/test_train.py index a559a0217..5cc5dcb51 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -20,7 +20,6 @@ BLACKLIST = [ "semantic-nerfw", "instant-ngp", "instant-ngp-bounded", - "nerfacto", "nerfacto-big", "volinga", "phototourism", @@ -34,7 +33,11 @@ BLACKLIST = [ def set_reduced_config(config: TrainerConfig): """Reducing the config settings to speedup test""" - config.machine.num_gpus = 0 + config.machine.device_type = "cpu" + if hasattr(config.pipeline.model, "implementation"): + setattr(config.pipeline.model, "implementation", "torch") + config.mixed_precision = False + config.use_grad_scaler = False config.max_num_iterations = 2 # reduce dataset factors; set dataset to test config.pipeline.datamanager.dataparser = BlenderDataParserConfig(data=Path("tests/data/lego_test"))