mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 07:05:44 +08:00
Ability to use vanilla pytorch instead of tcnn for Nerfacto (#1936)
* started a package to easily swap out tcnn for pytorch * trains with pytorch and not tcnn * cleaned up imports * making it easy to install on mac * removed/reverted files * merged main * reverted changes * updating config * starting to add nerfacto to tests * fixxing default value * fixed default implementation * defaults to torch if tcnn and not installed * update log * add comment * resolved pr comments * fix * update default --------- Co-authored-by: Ethan Weber <ethanweber@eecs-128-32-175-250.dhcp.berkeley.edu>
This commit is contained in:
1 parent
400ab5a313
commit
bb6925bac6
19 files changed
+295
-296
No files matched your search
@@ -21,7 +21,7 @@ from __future__ import annotations
|
||||
from abc import abstractmethod
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -115,7 +115,7 @@ class Model(nn.Module):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
|
||||
def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]:
|
||||
"""Takes in a Ray Bundle and returns a dictionary of outputs.
|
||||
|
||||
Args:
|
||||
@@ -126,7 +126,7 @@ class Model(nn.Module):
|
||||
Outputs of model. (ie. rendered colors)
|
||||
"""
|
||||
|
||||
def forward(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
|
||||
def forward(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]:
|
||||
"""Run forward starting with a ray bundle. This outputs different things depending on the configuration
|
||||
of the model and whether or not the batch is provided (whether or not we are training basically)
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.field_components.encodings import NeRFEncoding
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.field_components.spatial_distortions import SceneContraction
|
||||
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
|
||||
from nerfstudio.fields.nerfacto_field import NerfactoField
|
||||
from nerfstudio.fields.sdf_field import SDFFieldConfig
|
||||
from nerfstudio.fields.vanilla_nerf_field import NeRFField
|
||||
from nerfstudio.model_components.losses import (
|
||||
@@ -124,7 +124,7 @@ class SurfaceModel(Model):
|
||||
|
||||
# background model
|
||||
if self.config.background_model == "grid":
|
||||
self.field_background = TCNNNerfactoField(
|
||||
self.field_background = NerfactoField(
|
||||
self.scene_box.aabb,
|
||||
spatial_distortion=self.scene_contraction,
|
||||
num_images=self.num_train_data,
|
||||
|
||||
@@ -36,7 +36,7 @@ from nerfstudio.engine.callbacks import (
|
||||
)
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.field_components.spatial_distortions import SceneContraction
|
||||
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
|
||||
from nerfstudio.fields.nerfacto_field import NerfactoField
|
||||
from nerfstudio.model_components.losses import MSELoss
|
||||
from nerfstudio.model_components.ray_samplers import VolumetricSampler
|
||||
from nerfstudio.model_components.renderers import (
|
||||
@@ -94,7 +94,7 @@ class NGPModel(Model):
|
||||
"""
|
||||
|
||||
config: InstantNGPModelConfig
|
||||
field: TCNNNerfactoField
|
||||
field: NerfactoField
|
||||
|
||||
def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None:
|
||||
super().__init__(config=config, **kwargs)
|
||||
@@ -108,7 +108,7 @@ class NGPModel(Model):
|
||||
else:
|
||||
scene_contraction = SceneContraction(order=float("inf"))
|
||||
|
||||
self.field = TCNNNerfactoField(
|
||||
self.field = NerfactoField(
|
||||
aabb=self.scene_box.aabb,
|
||||
num_images=self.num_train_data,
|
||||
log2_hashmap_size=self.config.log2_hashmap_size,
|
||||
|
||||
@@ -37,7 +37,7 @@ from nerfstudio.engine.callbacks import (
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.field_components.spatial_distortions import SceneContraction
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
|
||||
from nerfstudio.fields.nerfacto_field import NerfactoField
|
||||
from nerfstudio.model_components.losses import (
|
||||
MSELoss,
|
||||
distortion_loss,
|
||||
@@ -130,6 +130,8 @@ class NerfactoModelConfig(ModelConfig):
|
||||
"""Whether to disable scene contraction or not."""
|
||||
use_gradient_scaling: bool = False
|
||||
"""Use gradient scaler where the gradients are lower for points closer to the camera."""
|
||||
implementation: Literal["tcnn", "torch"] = "tcnn"
|
||||
"""Which implementation to use for the model."""
|
||||
|
||||
|
||||
class NerfactoModel(Model):
|
||||
@@ -151,7 +153,7 @@ class NerfactoModel(Model):
|
||||
scene_contraction = SceneContraction(order=float("inf"))
|
||||
|
||||
# Fields
|
||||
self.field = TCNNNerfactoField(
|
||||
self.field = NerfactoField(
|
||||
self.scene_box.aabb,
|
||||
hidden_dim=self.config.hidden_dim,
|
||||
num_levels=self.config.num_levels,
|
||||
@@ -163,6 +165,7 @@ class NerfactoModel(Model):
|
||||
num_images=self.num_train_data,
|
||||
use_pred_normals=self.config.predict_normals,
|
||||
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
|
||||
implementation=self.config.implementation,
|
||||
)
|
||||
|
||||
self.density_fns = []
|
||||
@@ -172,7 +175,12 @@ class NerfactoModel(Model):
|
||||
if self.config.use_same_proposal_network:
|
||||
assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed."
|
||||
prop_net_args = self.config.proposal_net_args_list[0]
|
||||
network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction, **prop_net_args)
|
||||
network = HashMLPDensityField(
|
||||
self.scene_box.aabb,
|
||||
spatial_distortion=scene_contraction,
|
||||
**prop_net_args,
|
||||
implementation=self.config.implementation,
|
||||
)
|
||||
self.proposal_networks.append(network)
|
||||
self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)])
|
||||
else:
|
||||
@@ -182,6 +190,7 @@ class NerfactoModel(Model):
|
||||
self.scene_box.aabb,
|
||||
spatial_distortion=scene_contraction,
|
||||
**prop_net_args,
|
||||
implementation=self.config.implementation,
|
||||
)
|
||||
self.proposal_networks.append(network)
|
||||
self.density_fns.extend([network.density_fn for network in self.proposal_networks])
|
||||
|
||||
@@ -48,7 +48,7 @@ class NeuSFactoModelConfig(NeuSModelConfig):
|
||||
"""NeusFacto Model Config"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: NeuSFactoModel)
|
||||
num_proposal_samples_per_ray: Tuple[int, int] = (256, 96)
|
||||
num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
|
||||
"""Number of samples per ray for the proposal network."""
|
||||
num_neus_samples_per_ray: int = 48
|
||||
"""Number of samples per ray for the nerf network."""
|
||||
|
||||
@@ -38,7 +38,7 @@ from nerfstudio.engine.callbacks import (
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.field_components.spatial_distortions import SceneContraction
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.fields.nerfacto_field import TCNNNerfactoField
|
||||
from nerfstudio.fields.nerfacto_field import NerfactoField
|
||||
from nerfstudio.model_components.losses import MSELoss, distortion_loss, interlevel_loss
|
||||
from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler
|
||||
from nerfstudio.model_components.renderers import (
|
||||
@@ -90,7 +90,7 @@ class SemanticNerfWModel(Model):
|
||||
raise ValueError("Transient embedding is not fully working for semantic nerf-w.")
|
||||
|
||||
# Fields
|
||||
self.field = TCNNNerfactoField(
|
||||
self.field = NerfactoField(
|
||||
self.scene_box.aabb,
|
||||
num_levels=self.config.num_levels,
|
||||
max_res=self.config.max_res,
|
||||
|
||||
Reference in new issue
Block a user