mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 06:35:36 +08:00
NeusFacto (#1654)
* add neus_facto method * fix missing pixsfm from video process * fix dev errors and small typos --------- Co-authored-by: Ethan Weber <eweb0124@gmail.com>
This commit is contained in:
1 parent
fd6cc84808
commit
8126d47d8c
7 files changed
+348
-4
No files matched your search
@@ -45,9 +45,11 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConf
|
||||
from nerfstudio.engine.schedulers import (
|
||||
CosineDecaySchedulerConfig,
|
||||
ExponentialDecaySchedulerConfig,
|
||||
MultiStepSchedulerConfig,
|
||||
)
|
||||
from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind
|
||||
from nerfstudio.fields.sdf_field import SDFFieldConfig
|
||||
from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig
|
||||
from nerfstudio.models.instant_ngp import InstantNGPModelConfig
|
||||
from nerfstudio.models.mipnerf import MipNerfModel
|
||||
@@ -55,6 +57,7 @@ from nerfstudio.models.nerfacto import NerfactoModelConfig
|
||||
from nerfstudio.models.nerfplayer_nerfacto import NerfplayerNerfactoModelConfig
|
||||
from nerfstudio.models.nerfplayer_ngp import NerfplayerNGPModelConfig
|
||||
from nerfstudio.models.neus import NeuSModelConfig
|
||||
from nerfstudio.models.neus_facto import NeuSFactoModelConfig
|
||||
from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig
|
||||
from nerfstudio.models.tensorf import TensoRFModelConfig
|
||||
from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig
|
||||
@@ -516,6 +519,56 @@ method_configs["neus"] = TrainerConfig(
|
||||
vis="viewer",
|
||||
)
|
||||
|
||||
method_configs["neus-facto"] = TrainerConfig(
|
||||
method_name="neus-facto",
|
||||
steps_per_eval_image=5000,
|
||||
steps_per_eval_batch=5000,
|
||||
steps_per_save=2000,
|
||||
steps_per_eval_all_images=1000000, # set to a very large model so we don't eval with all images
|
||||
max_num_iterations=20001,
|
||||
mixed_precision=False,
|
||||
pipeline=VanillaPipelineConfig(
|
||||
datamanager=SDFDataManagerConfig(
|
||||
dataparser=SDFStudioDataParserConfig(),
|
||||
train_num_rays_per_batch=2048,
|
||||
eval_num_rays_per_batch=2048,
|
||||
camera_optimizer=CameraOptimizerConfig(
|
||||
mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2)
|
||||
),
|
||||
),
|
||||
model=NeuSFactoModelConfig(
|
||||
# proposal network allows for signifanctly smaller sdf/color network
|
||||
sdf_field=SDFFieldConfig(
|
||||
use_grid_feature=True,
|
||||
num_layers=2,
|
||||
num_layers_color=2,
|
||||
hidden_dim=256,
|
||||
bias=0.5,
|
||||
beta_init=0.8,
|
||||
use_appearance_embedding=False,
|
||||
),
|
||||
background_model="none",
|
||||
eval_num_rays_per_chunk=2048,
|
||||
),
|
||||
),
|
||||
optimizers={
|
||||
"proposal_networks": {
|
||||
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
|
||||
"scheduler": MultiStepSchedulerConfig(max_steps=20001, milestones=(10000, 1500, 18000)),
|
||||
},
|
||||
"fields": {
|
||||
"optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15),
|
||||
"scheduler": CosineDecaySchedulerConfig(warm_up_end=500, learning_rate_alpha=0.05, max_steps=20001),
|
||||
},
|
||||
"field_background": {
|
||||
"optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15),
|
||||
"scheduler": CosineDecaySchedulerConfig(warm_up_end=500, learning_rate_alpha=0.05, max_steps=20001),
|
||||
},
|
||||
},
|
||||
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
|
||||
vis="viewer",
|
||||
)
|
||||
|
||||
external_methods, external_descriptions = discover_methods()
|
||||
method_configs.update(external_methods)
|
||||
descriptions.update(external_descriptions)
|
||||
|
||||
@@ -55,7 +55,7 @@ class SDFStudioDataParserConfig(DataParserConfig):
|
||||
"""
|
||||
skip_every_for_val_split: int = 1
|
||||
"""sub sampling validation images"""
|
||||
auto_orient: bool = False
|
||||
auto_orient: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, Type
|
||||
from typing import Optional, Tuple, Type
|
||||
|
||||
import numpy as np
|
||||
from torch.optim import Optimizer, lr_scheduler
|
||||
@@ -56,6 +56,34 @@ class Scheduler:
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MultiStepSchedulerConfig(SchedulerConfig):
|
||||
"""Config for multi step scheduler where lr decays by gamma every milestone"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: MultiStepScheduler)
|
||||
"""target class to instantiate"""
|
||||
max_steps: int = 1000000
|
||||
"""The maximum number of steps."""
|
||||
gamma: float = 0.33
|
||||
"""The learning rate decay factor."""
|
||||
milestones: Tuple[int, ...] = (500000, 750000, 900000)
|
||||
"""The milestone steps at which to decay the learning rate."""
|
||||
|
||||
|
||||
class MultiStepScheduler(Scheduler):
|
||||
"""Multi step scheduler where lr decays by gamma every milestone"""
|
||||
|
||||
config: MultiStepSchedulerConfig
|
||||
|
||||
def get_scheduler(self, optimizer: Optimizer, lr_init: float) -> lr_scheduler._LRScheduler:
|
||||
scheduler = lr_scheduler.MultiStepLR(
|
||||
optimizer=optimizer,
|
||||
milestones=self.config.milestones,
|
||||
gamma=self.config.gamma,
|
||||
)
|
||||
return scheduler
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExponentialDecaySchedulerConfig(SchedulerConfig):
|
||||
"""Config for exponential decay scheduler with warmup"""
|
||||
@@ -111,7 +139,7 @@ class ExponentialDecayScheduler(Scheduler):
|
||||
|
||||
@dataclass
|
||||
class CosineDecaySchedulerConfig(SchedulerConfig):
|
||||
"""Basic scheduler config with self-defined exponential decay schedule"""
|
||||
"""Config for cosine decay schedule"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: CosineDecayScheduler)
|
||||
"""target class to instantiate"""
|
||||
@@ -124,7 +152,7 @@ class CosineDecaySchedulerConfig(SchedulerConfig):
|
||||
|
||||
|
||||
class CosineDecayScheduler(Scheduler):
|
||||
"""Starts with a flat lr schedule until it reaches N epochs then applies a given scheduler"""
|
||||
"""Cosine decay scheduler with linear warmup"""
|
||||
|
||||
config: CosineDecaySchedulerConfig
|
||||
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Implementation of NeuS similar to nerfacto where proposal sampler is used.
|
||||
Based on SDFStudio https://github.com/autonomousvision/sdfstudio/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.engine.callbacks import (
|
||||
TrainingCallback,
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.model_components.losses import interlevel_loss
|
||||
from nerfstudio.model_components.ray_samplers import (
|
||||
ProposalNetworkSampler,
|
||||
UniformSampler,
|
||||
)
|
||||
from nerfstudio.models.neus import NeuSModel, NeuSModelConfig
|
||||
from nerfstudio.utils import colormaps
|
||||
|
||||
|
||||
@dataclass
|
||||
class NeuSFactoModelConfig(NeuSModelConfig):
|
||||
"""NeusFacto Model Config"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: NeuSFactoModel)
|
||||
num_proposal_samples_per_ray: Tuple[int, int] = (256, 96)
|
||||
"""Number of samples per ray for the proposal network."""
|
||||
num_neus_samples_per_ray: int = 48
|
||||
"""Number of samples per ray for the nerf network."""
|
||||
proposal_update_every: int = 5
|
||||
"""Sample every n steps after the warmup"""
|
||||
proposal_warmup: int = 5000
|
||||
"""Scales n from 1 to proposal_update_every over this many steps"""
|
||||
num_proposal_iterations: int = 2
|
||||
"""Number of proposal network iterations."""
|
||||
use_same_proposal_network: bool = False
|
||||
"""Use the same proposal network. Otherwise use different ones."""
|
||||
proposal_net_args_list: List[Dict] = field(
|
||||
default_factory=lambda: [
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 64},
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
|
||||
]
|
||||
)
|
||||
"""Arguments for the proposal density fields."""
|
||||
interlevel_loss_mult: float = 1.0
|
||||
"""Proposal loss multiplier."""
|
||||
use_proposal_weight_anneal: bool = True
|
||||
"""Whether to use proposal weight annealing."""
|
||||
proposal_weights_anneal_slope: float = 10.0
|
||||
"""Slope of the annealing function for the proposal weights."""
|
||||
proposal_weights_anneal_max_num_iters: int = 1000
|
||||
"""Max num iterations for the annealing function."""
|
||||
use_single_jitter: bool = True
|
||||
"""Whether use single jitter or not for the proposal networks."""
|
||||
|
||||
|
||||
class NeuSFactoModel(NeuSModel):
|
||||
"""NeuSFactoModel extends NeuSModel for a more efficient sampling strategy.
|
||||
|
||||
The model improves the rendering speed and quality by incorporating a learning-based
|
||||
proposal distribution to guide the sampling process.(similar to mipnerf-360)
|
||||
|
||||
Args:
|
||||
config: NeuS configuration to instantiate model
|
||||
"""
|
||||
|
||||
config: NeuSFactoModelConfig
|
||||
|
||||
def populate_modules(self):
|
||||
"""Instantiate modules and fields, including proposal networks."""
|
||||
super().populate_modules()
|
||||
|
||||
self.density_fns = []
|
||||
num_prop_nets = self.config.num_proposal_iterations
|
||||
# Build the proposal network(s)
|
||||
self.proposal_networks = torch.nn.ModuleList()
|
||||
if self.config.use_same_proposal_network:
|
||||
assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed."
|
||||
prop_net_args = self.config.proposal_net_args_list[0]
|
||||
network = HashMLPDensityField(
|
||||
self.scene_box.aabb, spatial_distortion=self.scene_contraction, **prop_net_args
|
||||
)
|
||||
self.proposal_networks.append(network)
|
||||
self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)])
|
||||
else:
|
||||
for i in range(num_prop_nets):
|
||||
prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)]
|
||||
network = HashMLPDensityField(
|
||||
self.scene_box.aabb,
|
||||
spatial_distortion=self.scene_contraction,
|
||||
**prop_net_args,
|
||||
)
|
||||
self.proposal_networks.append(network)
|
||||
self.density_fns.extend([network.density_fn for network in self.proposal_networks])
|
||||
|
||||
# update proposal network every iterations
|
||||
update_schedule = lambda step: -1
|
||||
|
||||
initial_sampler = UniformSampler(single_jitter=self.config.use_single_jitter)
|
||||
self.proposal_sampler = ProposalNetworkSampler(
|
||||
num_nerf_samples_per_ray=self.config.num_neus_samples_per_ray,
|
||||
num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
|
||||
num_proposal_network_iterations=self.config.num_proposal_iterations,
|
||||
single_jitter=self.config.use_single_jitter,
|
||||
update_sched=update_schedule,
|
||||
initial_sampler=initial_sampler,
|
||||
)
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
"""Return a dictionary with the parameters of the proposal networks."""
|
||||
param_groups = super().get_param_groups()
|
||||
param_groups["proposal_networks"] = list(self.proposal_networks.parameters())
|
||||
return param_groups
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
callbacks = super().get_training_callbacks(training_callback_attributes)
|
||||
|
||||
if self.config.use_proposal_weight_anneal:
|
||||
# anneal the weights of the proposal network before doing PDF sampling
|
||||
N = self.config.proposal_weights_anneal_max_num_iters
|
||||
|
||||
def set_anneal(step: int):
|
||||
# https://arxiv.org/pdf/2111.12077.pdf eq. 18
|
||||
train_frac = np.clip(step / N, 0, 1)
|
||||
bias = lambda x, b: (b * x) / ((b - 1) * x + 1)
|
||||
anneal = bias(train_frac, self.config.proposal_weights_anneal_slope)
|
||||
self.proposal_sampler.set_anneal(anneal)
|
||||
|
||||
callbacks.append(
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=set_anneal,
|
||||
)
|
||||
)
|
||||
callbacks.append(
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=self.proposal_sampler.step_cb,
|
||||
)
|
||||
)
|
||||
|
||||
return callbacks
|
||||
|
||||
def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict[str, Any]:
|
||||
"""Sample rays using proposal networks and compute the corresponding field outputs."""
|
||||
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
|
||||
|
||||
field_outputs = self.field(ray_samples, return_alphas=True)
|
||||
weights, transmittance = ray_samples.get_weights_and_transmittance_from_alphas(
|
||||
field_outputs[FieldHeadNames.ALPHA]
|
||||
)
|
||||
bg_transmittance = transmittance[:, -1, :]
|
||||
|
||||
weights_list.append(weights)
|
||||
ray_samples_list.append(ray_samples)
|
||||
|
||||
samples_and_field_outputs = {
|
||||
"ray_samples": ray_samples,
|
||||
"field_outputs": field_outputs,
|
||||
"weights": weights,
|
||||
"bg_transmittance": bg_transmittance,
|
||||
"weights_list": weights_list,
|
||||
"ray_samples_list": ray_samples_list,
|
||||
}
|
||||
return samples_and_field_outputs
|
||||
|
||||
def get_loss_dict(
|
||||
self, outputs: Dict[str, Any], batch: Dict[str, Any], metrics_dict: Optional[Dict[str, Any]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Compute the loss dictionary, including interlevel loss for proposal networks."""
|
||||
loss_dict = super().get_loss_dict(outputs, batch, metrics_dict)
|
||||
|
||||
if self.training:
|
||||
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
self, outputs: Dict[str, Any], batch: Dict[str, Any]
|
||||
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
|
||||
"""Compute image metrics and images, including the proposal depth for each iteration."""
|
||||
metrics_dict, images_dict = super().get_image_metrics_and_images(outputs, batch)
|
||||
for i in range(self.config.num_proposal_iterations):
|
||||
key = f"prop_depth_{i}"
|
||||
prop_depth_i = colormaps.apply_depth_colormap(
|
||||
outputs[key],
|
||||
accumulation=outputs["accumulation"],
|
||||
)
|
||||
images_dict[key] = prop_depth_i
|
||||
|
||||
return metrics_dict, images_dict
|
||||
Reference in new issue
Block a user