Ability to use vanilla pytorch instead of tcnn for Nerfacto (#1936)

* started a package to easily swap out tcnn for pytorch

* trains with pytorch and not tcnn

* cleaned up imports

* making it easy to install on mac

* removed/reverted files

* merged main

* reverted changes

* updating config

* starting to add nerfacto to tests

* fixxing default value

* fixed default implementation

* defaults to torch if tcnn and not installed

* update log

* add comment

* resolved pr comments

* fix

* update default

---------

Co-authored-by: Ethan Weber <ethanweber@eecs-128-32-175-250.dhcp.berkeley.edu>
This commit is contained in:
Ethan WeberandEthan Weber authored and GitHub committed 2023-05-25 14:55:42 -07:00
1 parent 400ab5a313
commit bb6925bac6
19 files changed
+295 -296

No files matched your search

+3 -3
View File
@@ -21,7 +21,7 @@ from __future__ import annotations
from abc import abstractmethod
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple, Type
from typing import Any, Dict, List, Optional, Tuple, Type, Union
import torch
from torch import nn
@@ -115,7 +115,7 @@ class Model(nn.Module):
"""
@abstractmethod
def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]:
"""Takes in a Ray Bundle and returns a dictionary of outputs.
Args:
@@ -126,7 +126,7 @@ class Model(nn.Module):
Outputs of model. (ie. rendered colors)
"""
def forward(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
def forward(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]:
"""Run forward starting with a ray bundle. This outputs different things depending on the configuration
of the model and whether or not the batch is provided (whether or not we are training basically)
+2 -2
View File
@@ -33,7 +33,7 @@ from nerfstudio.cameras.rays import RayBundle
from nerfstudio.field_components.encodings import NeRFEncoding
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
from nerfstudio.fields.nerfacto_field import NerfactoField
from nerfstudio.fields.sdf_field import SDFFieldConfig
from nerfstudio.fields.vanilla_nerf_field import NeRFField
from nerfstudio.model_components.losses import (
@@ -124,7 +124,7 @@ class SurfaceModel(Model):
# background model
if self.config.background_model == "grid":
self.field_background = TCNNNerfactoField(
self.field_background = NerfactoField(
self.scene_box.aabb,
spatial_distortion=self.scene_contraction,
num_images=self.num_train_data,
+3 -3
View File
@@ -36,7 +36,7 @@ from nerfstudio.engine.callbacks import (
)
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
from nerfstudio.fields.nerfacto_field import NerfactoField
from nerfstudio.model_components.losses import MSELoss
from nerfstudio.model_components.ray_samplers import VolumetricSampler
from nerfstudio.model_components.renderers import (
@@ -94,7 +94,7 @@ class NGPModel(Model):
"""
config: InstantNGPModelConfig
field: TCNNNerfactoField
field: NerfactoField
def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None:
super().__init__(config=config, **kwargs)
@@ -108,7 +108,7 @@ class NGPModel(Model):
else:
scene_contraction = SceneContraction(order=float("inf"))
self.field = TCNNNerfactoField(
self.field = NerfactoField(
aabb=self.scene_box.aabb,
num_images=self.num_train_data,
log2_hashmap_size=self.config.log2_hashmap_size,
+12 -3
View File
@@ -37,7 +37,7 @@ from nerfstudio.engine.callbacks import (
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.density_fields import HashMLPDensityField
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
from nerfstudio.fields.nerfacto_field import NerfactoField
from nerfstudio.model_components.losses import (
MSELoss,
distortion_loss,
@@ -130,6 +130,8 @@ class NerfactoModelConfig(ModelConfig):
"""Whether to disable scene contraction or not."""
use_gradient_scaling: bool = False
"""Use gradient scaler where the gradients are lower for points closer to the camera."""
implementation: Literal["tcnn", "torch"] = "tcnn"
"""Which implementation to use for the model."""
class NerfactoModel(Model):
@@ -151,7 +153,7 @@ class NerfactoModel(Model):
scene_contraction = SceneContraction(order=float("inf"))
# Fields
self.field = TCNNNerfactoField(
self.field = NerfactoField(
self.scene_box.aabb,
hidden_dim=self.config.hidden_dim,
num_levels=self.config.num_levels,
@@ -163,6 +165,7 @@ class NerfactoModel(Model):
num_images=self.num_train_data,
use_pred_normals=self.config.predict_normals,
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
implementation=self.config.implementation,
)
self.density_fns = []
@@ -172,7 +175,12 @@ class NerfactoModel(Model):
if self.config.use_same_proposal_network:
assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed."
prop_net_args = self.config.proposal_net_args_list[0]
network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction, **prop_net_args)
network = HashMLPDensityField(
self.scene_box.aabb,
spatial_distortion=scene_contraction,
**prop_net_args,
implementation=self.config.implementation,
)
self.proposal_networks.append(network)
self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)])
else:
@@ -182,6 +190,7 @@ class NerfactoModel(Model):
self.scene_box.aabb,
spatial_distortion=scene_contraction,
**prop_net_args,
implementation=self.config.implementation,
)
self.proposal_networks.append(network)
self.density_fns.extend([network.density_fn for network in self.proposal_networks])
+1 -1
View File
@@ -48,7 +48,7 @@ class NeuSFactoModelConfig(NeuSModelConfig):
"""NeusFacto Model Config"""
_target: Type = field(default_factory=lambda: NeuSFactoModel)
num_proposal_samples_per_ray: Tuple[int, int] = (256, 96)
num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
"""Number of samples per ray for the proposal network."""
num_neus_samples_per_ray: int = 48
"""Number of samples per ray for the nerf network."""
+2 -2
View File
@@ -38,7 +38,7 @@ from nerfstudio.engine.callbacks import (
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.density_fields import HashMLPDensityField
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
from nerfstudio.fields.nerfacto_field import NerfactoField
from nerfstudio.model_components.losses import MSELoss, distortion_loss, interlevel_loss
from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler
from nerfstudio.model_components.renderers import (
@@ -90,7 +90,7 @@ class SemanticNerfWModel(Model):
raise ValueError("Transient embedding is not fully working for semantic nerf-w.")
# Fields
self.field = TCNNNerfactoField(
self.field = NerfactoField(
self.scene_box.aabb,
num_levels=self.config.num_levels,
max_res=self.config.max_res,