diff --git a/.vscode/launch.json b/.vscode/launch.json index 118282928..da293a07b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -112,5 +112,42 @@ "--include_mono_prior=True" ] }, + { + "name": "Python: NeuS-facto on Replica", + "type": "python", + "request": "launch", + "program": "scripts/train.py", + "console": "integratedTerminal", + "justMyCode": false, + "env": {"CUDA_VISIBLE_DEVICES":"0"}, + "args": [ + "neus-facto", + "--pipeline.model.sdf-field.inside-outside=True", + "--pipeline.model.mono-depth-loss-mult=0.1", + "--pipeline.model.mono-normal-loss-mult=0.05", + "--vis=wandb", + "sdfstudio-data", + "--data=data/sdfstudio-demo-data/replica-room0", + "--include_mono_prior=True" + ] + }, + { + "name": "Python: NeuS-facto on DTU (no mono-prior)", + "type": "python", + "request": "launch", + "program": "scripts/train.py", + "console": "integratedTerminal", + "justMyCode": false, + "env": {"CUDA_VISIBLE_DEVICES":"0"}, + "args": [ + "neus-facto", + "--pipeline.model.sdf-field.inside-outside=False", + "--vis=wandb", + "sdfstudio-data", + "--data=data/sdfstudio-demo-data/dtu-scan65", + "--include_mono_prior=False", + "--auto-orient=True" + ] + }, ] } diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 0357d0f9b..8842f4e46 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -45,9 +45,11 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConf from nerfstudio.engine.schedulers import ( CosineDecaySchedulerConfig, ExponentialDecaySchedulerConfig, + MultiStepSchedulerConfig, ) from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind +from nerfstudio.fields.sdf_field import SDFFieldConfig from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel @@ -55,6 +57,7 @@ from nerfstudio.models.nerfacto import NerfactoModelConfig from nerfstudio.models.nerfplayer_nerfacto import NerfplayerNerfactoModelConfig from nerfstudio.models.nerfplayer_ngp import NerfplayerNGPModelConfig from nerfstudio.models.neus import NeuSModelConfig +from nerfstudio.models.neus_facto import NeuSFactoModelConfig from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig @@ -516,6 +519,56 @@ method_configs["neus"] = TrainerConfig( vis="viewer", ) +method_configs["neus-facto"] = TrainerConfig( + method_name="neus-facto", + steps_per_eval_image=5000, + steps_per_eval_batch=5000, + steps_per_save=2000, + steps_per_eval_all_images=1000000, # set to a very large model so we don't eval with all images + max_num_iterations=20001, + mixed_precision=False, + pipeline=VanillaPipelineConfig( + datamanager=SDFDataManagerConfig( + dataparser=SDFStudioDataParserConfig(), + train_num_rays_per_batch=2048, + eval_num_rays_per_batch=2048, + camera_optimizer=CameraOptimizerConfig( + mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) + ), + ), + model=NeuSFactoModelConfig( + # proposal network allows for signifanctly smaller sdf/color network + sdf_field=SDFFieldConfig( + use_grid_feature=True, + num_layers=2, + num_layers_color=2, + hidden_dim=256, + bias=0.5, + beta_init=0.8, + use_appearance_embedding=False, + ), + background_model="none", + eval_num_rays_per_chunk=2048, + ), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": MultiStepSchedulerConfig(max_steps=20001, milestones=(10000, 1500, 18000)), + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15), + "scheduler": CosineDecaySchedulerConfig(warm_up_end=500, learning_rate_alpha=0.05, max_steps=20001), + }, + "field_background": { + "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15), + "scheduler": CosineDecaySchedulerConfig(warm_up_end=500, learning_rate_alpha=0.05, max_steps=20001), + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", +) + external_methods, external_descriptions = discover_methods() method_configs.update(external_methods) descriptions.update(external_descriptions) diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py index 2b01f77d2..97c7a299d 100644 --- a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py @@ -55,7 +55,7 @@ class SDFStudioDataParserConfig(DataParserConfig): """ skip_every_for_val_split: int = 1 """sub sampling validation images""" - auto_orient: bool = False + auto_orient: bool = True @dataclass diff --git a/nerfstudio/engine/schedulers.py b/nerfstudio/engine/schedulers.py index 48a656650..301d9eb5f 100644 --- a/nerfstudio/engine/schedulers.py +++ b/nerfstudio/engine/schedulers.py @@ -18,7 +18,7 @@ from abc import abstractmethod from dataclasses import dataclass, field -from typing import Optional, Type +from typing import Optional, Tuple, Type import numpy as np from torch.optim import Optimizer, lr_scheduler @@ -56,6 +56,34 @@ class Scheduler: """ +@dataclass +class MultiStepSchedulerConfig(SchedulerConfig): + """Config for multi step scheduler where lr decays by gamma every milestone""" + + _target: Type = field(default_factory=lambda: MultiStepScheduler) + """target class to instantiate""" + max_steps: int = 1000000 + """The maximum number of steps.""" + gamma: float = 0.33 + """The learning rate decay factor.""" + milestones: Tuple[int, ...] = (500000, 750000, 900000) + """The milestone steps at which to decay the learning rate.""" + + +class MultiStepScheduler(Scheduler): + """Multi step scheduler where lr decays by gamma every milestone""" + + config: MultiStepSchedulerConfig + + def get_scheduler(self, optimizer: Optimizer, lr_init: float) -> lr_scheduler._LRScheduler: + scheduler = lr_scheduler.MultiStepLR( + optimizer=optimizer, + milestones=self.config.milestones, + gamma=self.config.gamma, + ) + return scheduler + + @dataclass class ExponentialDecaySchedulerConfig(SchedulerConfig): """Config for exponential decay scheduler with warmup""" @@ -111,7 +139,7 @@ class ExponentialDecayScheduler(Scheduler): @dataclass class CosineDecaySchedulerConfig(SchedulerConfig): - """Basic scheduler config with self-defined exponential decay schedule""" + """Config for cosine decay schedule""" _target: Type = field(default_factory=lambda: CosineDecayScheduler) """target class to instantiate""" @@ -124,7 +152,7 @@ class CosineDecaySchedulerConfig(SchedulerConfig): class CosineDecayScheduler(Scheduler): - """Starts with a flat lr schedule until it reaches N epochs then applies a given scheduler""" + """Cosine decay scheduler with linear warmup""" config: CosineDecaySchedulerConfig diff --git a/nerfstudio/models/neus_facto.py b/nerfstudio/models/neus_facto.py new file mode 100644 index 000000000..554f0d90f --- /dev/null +++ b/nerfstudio/models/neus_facto.py @@ -0,0 +1,222 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Implementation of NeuS similar to nerfacto where proposal sampler is used. +Based on SDFStudio https://github.com/autonomousvision/sdfstudio/ +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, Type + +import numpy as np +import torch +from torch.nn import Parameter + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.model_components.losses import interlevel_loss +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.models.neus import NeuSModel, NeuSModelConfig +from nerfstudio.utils import colormaps + + +@dataclass +class NeuSFactoModelConfig(NeuSModelConfig): + """NeusFacto Model Config""" + + _target: Type = field(default_factory=lambda: NeuSFactoModel) + num_proposal_samples_per_ray: Tuple[int, int] = (256, 96) + """Number of samples per ray for the proposal network.""" + num_neus_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 64}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256}, + ] + ) + """Arguments for the proposal density fields.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + use_proposal_weight_anneal: bool = True + """Whether to use proposal weight annealing.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 1000 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + + +class NeuSFactoModel(NeuSModel): + """NeuSFactoModel extends NeuSModel for a more efficient sampling strategy. + + The model improves the rendering speed and quality by incorporating a learning-based + proposal distribution to guide the sampling process.(similar to mipnerf-360) + + Args: + config: NeuS configuration to instantiate model + """ + + config: NeuSFactoModelConfig + + def populate_modules(self): + """Instantiate modules and fields, including proposal networks.""" + super().populate_modules() + + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + if self.config.use_same_proposal_network: + assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed." + prop_net_args = self.config.proposal_net_args_list[0] + network = HashMLPDensityField( + self.scene_box.aabb, spatial_distortion=self.scene_contraction, **prop_net_args + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)]) + else: + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=self.scene_contraction, + **prop_net_args, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + # update proposal network every iterations + update_schedule = lambda step: -1 + + initial_sampler = UniformSampler(single_jitter=self.config.use_single_jitter) + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_neus_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + initial_sampler=initial_sampler, + ) + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Return a dictionary with the parameters of the proposal networks.""" + param_groups = super().get_param_groups() + param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) + return param_groups + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + callbacks = super().get_training_callbacks(training_callback_attributes) + + if self.config.use_proposal_weight_anneal: + # anneal the weights of the proposal network before doing PDF sampling + N = self.config.proposal_weights_anneal_max_num_iters + + def set_anneal(step: int): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / N, 0, 1) + bias = lambda x, b: (b * x) / ((b - 1) * x + 1) + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ) + ) + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.proposal_sampler.step_cb, + ) + ) + + return callbacks + + def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict[str, Any]: + """Sample rays using proposal networks and compute the corresponding field outputs.""" + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + + field_outputs = self.field(ray_samples, return_alphas=True) + weights, transmittance = ray_samples.get_weights_and_transmittance_from_alphas( + field_outputs[FieldHeadNames.ALPHA] + ) + bg_transmittance = transmittance[:, -1, :] + + weights_list.append(weights) + ray_samples_list.append(ray_samples) + + samples_and_field_outputs = { + "ray_samples": ray_samples, + "field_outputs": field_outputs, + "weights": weights, + "bg_transmittance": bg_transmittance, + "weights_list": weights_list, + "ray_samples_list": ray_samples_list, + } + return samples_and_field_outputs + + def get_loss_dict( + self, outputs: Dict[str, Any], batch: Dict[str, Any], metrics_dict: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """Compute the loss dictionary, including interlevel loss for proposal networks.""" + loss_dict = super().get_loss_dict(outputs, batch, metrics_dict) + + if self.training: + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, Any], batch: Dict[str, Any] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Compute image metrics and images, including the proposal depth for each iteration.""" + metrics_dict, images_dict = super().get_image_metrics_and_images(outputs, batch) + for i in range(self.config.num_proposal_iterations): + key = f"prop_depth_{i}" + prop_depth_i = colormaps.apply_depth_colormap( + outputs[key], + accumulation=outputs["accumulation"], + ) + images_dict[key] = prop_depth_i + + return metrics_dict, images_dict diff --git a/scripts/process_data.py b/scripts/process_data.py index 2eaa9e11c..e11541024 100644 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -268,6 +268,8 @@ class ProcessVideo: sfm_tool: Literal["any", "colmap", "hloc"] = "any" """Structure from motion tool to use. Colmap will use sift features, hloc can use many modern methods such as superpoint features and superglue matcher""" + refine_pixsfm: bool = False + """If True, runs refinement using Pixel Perfect SFM. Only has an effect if sfm_tool is hloc""" feature_type: Literal[ "any", "sift", @@ -410,6 +412,7 @@ class ProcessVideo: matching_method=self.matching_method, feature_type=feature_type, matcher_type=matcher_type, + refine_pixsfm=self.refine_pixsfm, ) else: CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting") diff --git a/tests/test_train.py b/tests/test_train.py index f795f6060..238408192 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -28,6 +28,7 @@ BLACKLIST = [ "nerfplayer-ngp", "nerfplayer-nerfacto", "neus", + "neus-facto", ]