diff --git a/.vscode/launch.json b/.vscode/launch.json index 935d7ae8b..118282928 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -92,6 +92,25 @@ "program": "scripts/train.py", "console": "integratedTerminal", "args": ["semantic_nerf"] - } + }, + { + "name": "Python: NeuS on Replica", + "type": "python", + "request": "launch", + "program": "scripts/train.py", + "console": "integratedTerminal", + "justMyCode": false, + "env": {"CUDA_VISIBLE_DEVICES":"0"}, + "args": [ + "neus", + "--pipeline.model.sdf-field.inside-outside=True", + "--pipeline.model.mono-depth-loss-mult=0.1", + "--pipeline.model.mono-normal-loss-mult=0.05", + "--vis=wandb", + "sdfstudio-data", + "--data=data/sdfstudio-demo-data/replica-room0", + "--include_mono_prior=True" + ] + }, ] } diff --git a/nerfstudio/cameras/rays.py b/nerfstudio/cameras/rays.py index fd4e8129e..0184b4c80 100644 --- a/nerfstudio/cameras/rays.py +++ b/nerfstudio/cameras/rays.py @@ -17,7 +17,7 @@ Some ray datastructures. """ import random from dataclasses import dataclass -from typing import Callable, Dict, Optional, Union +from typing import Callable, Dict, Optional, Tuple, Union import torch from torchtyping import TensorType @@ -56,6 +56,14 @@ class Frustums(TensorDataclass): pos = pos + self.offsets return pos + def get_start_positions(self) -> TensorType[..., 3]: + """Calulates "start" position of frustum. + + Returns: + xyz positions. + """ + return self.origins + self.directions * self.starts + def set_offsets(self, offsets): """Sets offsets for this frustum for computing positions""" self.offsets = offsets @@ -140,6 +148,27 @@ class RaySamples(TensorDataclass): return weights + @staticmethod + def get_weights_and_transmittance_from_alphas( + alphas: TensorType[..., "num_samples", 1], weights_only: bool = False + ) -> Tuple[TensorType[..., "num_samples", 1], TensorType[..., "num_samples", 1]]: + """Return weights based on predicted alphas + Args: + alphas: Predicted alphas (maybe from sdf) for samples along ray + weights_only: If function should return only weights + Returns: + Tuple of weights and transmittance for each sample + """ + + transmittance = torch.cumprod( + torch.cat([torch.ones((*alphas.shape[:1], 1, 1), device=alphas.device), 1.0 - alphas + 1e-7], 1), 1 + ) + + weights = alphas * transmittance[:, :-1, :] + if weights_only: + return weights + return weights, transmittance + @dataclass class RayBundle(TensorDataclass): diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index e888747b4..5d799fb01 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -27,6 +27,7 @@ from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig from nerfstudio.data.datamanagers.depth_datamanager import DepthDataManagerConfig +from nerfstudio.data.datamanagers.sdf_datamanager import SDFDataManagerConfig from nerfstudio.data.datamanagers.semantic_datamanager import SemanticDataManagerConfig from nerfstudio.data.datamanagers.variable_res_datamanager import ( VariableResDataManagerConfig, @@ -41,9 +42,13 @@ from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataPars from nerfstudio.data.dataparsers.phototourism_dataparser import ( PhototourismDataParserConfig, ) +from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig -from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig +from nerfstudio.engine.schedulers import ( + CosineDecaySchedulerConfig, + ExponentialDecaySchedulerConfig, +) from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig @@ -52,6 +57,7 @@ from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.nerfacto import NerfactoModelConfig from nerfstudio.models.nerfplayer_nerfacto import NerfplayerNerfactoModelConfig from nerfstudio.models.nerfplayer_ngp import NerfplayerNGPModelConfig +from nerfstudio.models.neus import NeuSModelConfig from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig @@ -74,6 +80,7 @@ descriptions = { "phototourism": "Uses the Phototourism data.", "nerfplayer-nerfacto": "NeRFPlayer with nerfacto backbone.", "nerfplayer-ngp": "NeRFPlayer with InstantNGP backbone.", + "neus": "Implementation of NeuS. (slow)", } method_configs["nerfacto"] = TrainerConfig( @@ -430,6 +437,39 @@ method_configs["nerfplayer-ngp"] = TrainerConfig( vis="viewer", ) +method_configs["neus"] = TrainerConfig( + method_name="neus", + steps_per_eval_image=500, + steps_per_eval_batch=5000, + steps_per_save=20000, + steps_per_eval_all_images=1000000, # set to a very large number so we don't eval with all images + max_num_iterations=100000, + mixed_precision=False, + pipeline=VanillaPipelineConfig( + datamanager=SDFDataManagerConfig( + dataparser=SDFStudioDataParserConfig(), + train_num_rays_per_batch=1024, + eval_num_rays_per_batch=1024, + camera_optimizer=CameraOptimizerConfig( + mode="off", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) + ), + ), + model=NeuSModelConfig(eval_num_rays_per_chunk=1024), + ), + optimizers={ + "fields": { + "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15), + "scheduler": CosineDecaySchedulerConfig(warm_up_end=5000, learning_rate_alpha=0.05, max_steps=300000), + }, + "field_background": { + "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15), + "scheduler": CosineDecaySchedulerConfig(warm_up_end=5000, learning_rate_alpha=0.05, max_steps=300000), + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", +) + external_methods, external_descriptions = discover_methods() method_configs.update(external_methods) descriptions.update(external_descriptions) diff --git a/nerfstudio/data/datamanagers/sdf_datamanager.py b/nerfstudio/data/datamanagers/sdf_datamanager.py new file mode 100644 index 000000000..e1aca46df --- /dev/null +++ b/nerfstudio/data/datamanagers/sdf_datamanager.py @@ -0,0 +1,48 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SDF datamanager. +""" + +from dataclasses import dataclass, field +from typing import Type + +from nerfstudio.data.datamanagers import base_datamanager +from nerfstudio.data.datasets.sdf_dataset import SDFDataset + + +@dataclass +class SDFDataManagerConfig(base_datamanager.VanillaDataManagerConfig): + """A sdf datamanager - required to use with .setup()""" + + _target: Type = field(default_factory=lambda: SDFDataManager) + + +class SDFDataManager(base_datamanager.VanillaDataManager): # pylint: disable=abstract-method + """Data manager implementation for data that also requires processing sdf data. + Args: + config: the DataManagerConfig used to instantiate class + """ + + def create_train_dataset(self) -> SDFDataset: + self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") + return SDFDataset( + dataparser_outputs=self.train_dataparser_outputs, + ) + + def create_eval_dataset(self) -> SDFDataset: + return SDFDataset( + dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split), + ) diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py index 43781a00b..9b692c65b 100644 --- a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py @@ -107,6 +107,7 @@ class SDFStudio(DataParser): fy = torch.stack(fy) cx = torch.stack(cx) cy = torch.stack(cy) + c2w_colmap = torch.stack(camera_to_worlds) camera_to_worlds = torch.stack(camera_to_worlds) # Convert from COLMAP's/OPENCV's camera coordinate system to nerfstudio @@ -151,7 +152,8 @@ class SDFStudio(DataParser): "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, "normal_filenames": normal_filenames if len(normal_filenames) > 0 else None, "transform": transform, - "camera_to_worlds": camera_to_worlds if len(camera_to_worlds) > 0 else None, + # required for normal maps, these are in colmap format so they require c2w before conversion + "camera_to_worlds": c2w_colmap if len(c2w_colmap) > 0 else None, "include_mono_prior": self.config.include_mono_prior, }, ) diff --git a/nerfstudio/data/datasets/sdf_dataset.py b/nerfstudio/data/datasets/sdf_dataset.py index 48e0cff6d..db0bb9796 100644 --- a/nerfstudio/data/datasets/sdf_dataset.py +++ b/nerfstudio/data/datasets/sdf_dataset.py @@ -42,7 +42,7 @@ class SDFDataset(InputDataset): self.normal_filenames = self.metadata["normal_filenames"] self.camera_to_worlds = self.metadata["camera_to_worlds"] # can be none if auto orient not enabled in dataparser - self.transform = self.metadata["normal_filenames"] + self.transform = self.metadata["transform"] self.include_mono_prior = self.metadata["include_mono_prior"] def get_metadata(self, data: Dict) -> Dict: @@ -57,8 +57,8 @@ class SDFDataset(InputDataset): depth_image, normal_image = self.get_depths_and_normals( depth_filepath=depth_filepath, normal_filename=normal_filepath, camtoworld=camtoworld ) - metadata["depth_image"] = depth_image - metadata["normal_image"] = normal_image + metadata["depth"] = depth_image + metadata["normal"] = normal_image return metadata diff --git a/nerfstudio/engine/schedulers.py b/nerfstudio/engine/schedulers.py index 0b6586ffc..48a656650 100644 --- a/nerfstudio/engine/schedulers.py +++ b/nerfstudio/engine/schedulers.py @@ -107,3 +107,36 @@ class ExponentialDecayScheduler(Scheduler): scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=func) return scheduler + + +@dataclass +class CosineDecaySchedulerConfig(SchedulerConfig): + """Basic scheduler config with self-defined exponential decay schedule""" + + _target: Type = field(default_factory=lambda: CosineDecayScheduler) + """target class to instantiate""" + warm_up_end: int = 5000 + """Iteration number where warmp ends""" + learning_rate_alpha: float = 0.05 + """Learning rate alpha value""" + max_steps: int = 300000 + """The maximum number of steps.""" + + +class CosineDecayScheduler(Scheduler): + """Starts with a flat lr schedule until it reaches N epochs then applies a given scheduler""" + + config: CosineDecaySchedulerConfig + + def get_scheduler(self, optimizer: Optimizer, lr_init: float) -> lr_scheduler._LRScheduler: + def func(step): + if step < self.config.warm_up_end: + learning_factor = step / self.config.warm_up_end + else: + alpha = self.config.learning_rate_alpha + progress = (step - self.config.warm_up_end) / (self.config.max_steps - self.config.warm_up_end) + learning_factor = (np.cos(np.pi * progress) + 1.0) * 0.5 * (1 - alpha) + alpha + return learning_factor + + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=func) + return scheduler diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index cba0ee7ea..6fffb5111 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -37,6 +37,9 @@ class FieldHeadNames(Enum): TRANSIENT_RGB = "transient_rgb" TRANSIENT_DENSITY = "transient_density" SEMANTICS = "semantics" + SDF = "sdf" + ALPHA = "alpha" + GRADIENT = "gradient" class FieldHead(FieldComponent): diff --git a/nerfstudio/fields/base_field.py b/nerfstudio/fields/base_field.py index e20774d88..3bcc51d56 100644 --- a/nerfstudio/fields/base_field.py +++ b/nerfstudio/fields/base_field.py @@ -17,16 +17,26 @@ Base class for the graphs. """ from abc import abstractmethod -from typing import Dict, Optional, Tuple +from dataclasses import dataclass, field +from typing import Dict, Optional, Tuple, Type import torch from torch import nn from torchtyping import TensorType from nerfstudio.cameras.rays import Frustums, RaySamples +from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.field_components.field_heads import FieldHeadNames +@dataclass +class FieldConfig(InstantiateConfig): + """Configuration for field instantiation""" + + _target: Type = field(default_factory=lambda: Field) + """target class to instantiate""" + + class Field(nn.Module): """Base class for fields.""" diff --git a/nerfstudio/fields/sdf_field.py b/nerfstudio/fields/sdf_field.py new file mode 100644 index 000000000..e9fd10cb8 --- /dev/null +++ b/nerfstudio/fields/sdf_field.py @@ -0,0 +1,458 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Field for SDF based model, rather then estimating density to generate a surface, +a signed distance function (SDF) for surface representation is used to help with extracting high fidelity surfaces +""" + +from dataclasses import dataclass, field +from typing import Dict, Optional, Type + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.parameter import Parameter +from torchtyping import TensorType +from typing_extensions import Literal + +from nerfstudio.cameras.rays import RaySamples +from nerfstudio.field_components.embedding import Embedding +from nerfstudio.field_components.encodings import NeRFEncoding +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SpatialDistortion +from nerfstudio.fields.base_field import Field, FieldConfig + +try: + import tinycudann as tcnn +except ImportError: + # tinycudann module doesn't exist + pass + + +class LearnedVariance(nn.Module): + """Variance network in NeuS + + Args: + init_val: initial value in NeuS variance network + """ + + def __init__(self, init_val): + super().__init__() + self.register_parameter("variance", nn.Parameter(init_val * torch.ones(1), requires_grad=True)) + + def forward(self, x: TensorType[1]) -> TensorType[1]: + """Returns current variance value""" + return torch.ones([len(x), 1], device=x.device) * torch.exp(self.variance * 10.0) + + def get_variance(self) -> TensorType[1]: + """return current variance value""" + return torch.exp(self.variance * 10.0).clip(1e-6, 1e6) + + +@dataclass +class SDFFieldConfig(FieldConfig): + """SDF Field Config""" + + _target: Type = field(default_factory=lambda: SDFField) + num_layers: int = 8 + """Number of layers for geometric network""" + hidden_dim: int = 256 + """Number of hidden dimension of geometric network""" + geo_feat_dim: int = 256 + """Dimension of geometric feature""" + num_layers_color: int = 4 + """Number of layers for color network""" + hidden_dim_color: int = 256 + """Number of hidden dimension of color network""" + appearance_embedding_dim: int = 32 + """Dimension of appearance embedding""" + use_appearance_embedding: bool = False + """Dimension of appearance embedding""" + bias: float = 0.8 + """sphere size of geometric initializaion""" + geometric_init: bool = True + """Whether to use geometric initialization""" + inside_outside: bool = True + """whether to revert signed distance value, set to True for indoor scene""" + weight_norm: bool = True + """Whether to use weight norm for linear layer""" + use_grid_feature: bool = False + """Whether to use multi-resolution feature grids""" + divide_factor: float = 2.0 + """Normalization factor for multi-resolution grids""" + beta_init: float = 0.1 + """Init learnable beta value for transformation of sdf to density""" + encoding_type: Literal["hash", "periodic", "tensorf_vm"] = "hash" + num_levels = 16 + """Number of encoding levels""" + max_res = 2048 + """Maximum resolution of the encoding""" + base_res = 16 + """Base resolution of the encoding""" + log2_hashmap_size = 19 + """Size of the hash map""" + features_per_level = 2 + """Number of features per encoding level""" + use_hash = True + """Whether to use hash encoding""" + smoothstep = True + """Whether to use the smoothstep function""" + + +class SDFField(Field): + """ + A field for Signed Distance Functions (SDF). + + Args: + config: The configuration for the SDF field. + aabb: An axis-aligned bounding box for the SDF field. + num_images: The number of images for embedding appearance. + use_average_appearance_embedding: Whether to use average appearance embedding. Defaults to False. + spatial_distortion: The spatial distortion. Defaults to None. + """ + + config: SDFFieldConfig + + def __init__( + self, + config: SDFFieldConfig, + aabb: TensorType[2, 3], + num_images: int, + use_average_appearance_embedding: bool = False, + spatial_distortion: Optional[SpatialDistortion] = None, + ) -> None: + super().__init__() + self.config = config + + self.aabb = Parameter(aabb, requires_grad=False) + + self.spatial_distortion = spatial_distortion + self.num_images = num_images + + self.embedding_appearance = Embedding(self.num_images, self.config.appearance_embedding_dim) + self.use_average_appearance_embedding = use_average_appearance_embedding + self.use_grid_feature = self.config.use_grid_feature + self.divide_factor = self.config.divide_factor + + growth_factor = np.exp((np.log(config.max_res) - np.log(config.base_res)) / (config.num_levels - 1)) + + if self.config.encoding_type == "hash": + # feature encoding + self.encoding = tcnn.Encoding( + n_input_dims=3, + encoding_config={ + "otype": "HashGrid" if config.use_hash else "DenseGrid", + "n_levels": config.num_levels, + "n_features_per_level": config.features_per_level, + "log2_hashmap_size": config.log2_hashmap_size, + "base_resolution": config.base_res, + "per_level_scale": growth_factor, + "interpolation": "Smoothstep" if config.smoothstep else "Linear", + }, + ) + + # we concat inputs position ourselves + self.position_encoding = NeRFEncoding( + in_dim=3, num_frequencies=6, min_freq_exp=0.0, max_freq_exp=5.0, include_input=False + ) + + self.direction_encoding = NeRFEncoding( + in_dim=3, num_frequencies=4, min_freq_exp=0.0, max_freq_exp=3.0, include_input=True + ) + + # initialize geometric network + self.initialize_geo_layers() + + # deviation_network to compute alpha from sdf from NeuS + self.deviation_network = LearnedVariance(init_val=self.config.beta_init) + + # color network + dims = [self.config.hidden_dim_color for _ in range(self.config.num_layers_color)] + # point, view_direction, normal, feature, embedding + in_dim = ( + 3 + + self.direction_encoding.get_out_dim() + + 3 + + self.config.geo_feat_dim + + self.embedding_appearance.get_out_dim() + ) + dims = [in_dim] + dims + [3] + self.num_layers_color = len(dims) + + for l in range(0, self.num_layers_color - 1): + out_dim = dims[l + 1] + lin = nn.Linear(dims[l], out_dim) + + if self.config.weight_norm: + lin = nn.utils.weight_norm(lin) + setattr(self, "clin" + str(l), lin) + + self.softplus = nn.Softplus(beta=100) + self.relu = nn.ReLU() + self.sigmoid = torch.nn.Sigmoid() + + self._cos_anneal_ratio = 1.0 + + def initialize_geo_layers(self) -> None: + """ + Initialize layers for geometric network (sdf) + """ + # MLP with geometric initialization + dims = [self.config.hidden_dim for _ in range(self.config.num_layers)] + in_dim = 3 + self.position_encoding.get_out_dim() + self.encoding.n_output_dims + dims = [in_dim] + dims + [1 + self.config.geo_feat_dim] + self.num_layers = len(dims) + self.skip_in = [4] + + for l in range(0, self.num_layers - 1): + if l + 1 in self.skip_in: + out_dim = dims[l + 1] - dims[0] + else: + out_dim = dims[l + 1] + + lin = nn.Linear(dims[l], out_dim) + + if self.config.geometric_init: + if l == self.num_layers - 2: + if not self.config.inside_outside: + torch.nn.init.normal_(lin.weight, mean=np.sqrt(np.pi) / np.sqrt(dims[l]), std=0.0001) + torch.nn.init.constant_(lin.bias, -self.config.bias) + else: + torch.nn.init.normal_(lin.weight, mean=-np.sqrt(np.pi) / np.sqrt(dims[l]), std=0.0001) + torch.nn.init.constant_(lin.bias, self.config.bias) + elif l == 0: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.constant_(lin.weight[:, 3:], 0.0) + torch.nn.init.normal_(lin.weight[:, :3], 0.0, np.sqrt(2) / np.sqrt(out_dim)) + elif l in self.skip_in: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.normal_(lin.weight, 0.0, np.sqrt(2) / np.sqrt(out_dim)) + torch.nn.init.constant_(lin.weight[:, -(dims[0] - 3) :], 0.0) + else: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.normal_(lin.weight, 0.0, np.sqrt(2) / np.sqrt(out_dim)) + + if self.config.weight_norm: + lin = nn.utils.weight_norm(lin) + setattr(self, "glin" + str(l), lin) + + def set_cos_anneal_ratio(self, anneal: float) -> None: + """Set the anneal value for the proposal network.""" + self._cos_anneal_ratio = anneal + + def forward_geonetwork(self, inputs: TensorType[..., 3]) -> TensorType[..., "geo-features+1"]: + """forward the geonetwork""" + if self.use_grid_feature: + positions = self.spatial_distortion(inputs) + + positions = (positions + 1.0) / 2.0 + feature = self.encoding(positions) + else: + feature = torch.zeros_like(inputs[:, :1].repeat(1, self.encoding.n_output_dims)) + + pe = self.position_encoding(inputs) + + inputs = torch.cat((inputs, pe, feature), dim=-1) + + # Pass through layers + outputs = inputs + + for l in range(0, self.num_layers - 1): + lin = getattr(self, "glin" + str(l)) + + if l in self.skip_in: + outputs = torch.cat([outputs, inputs], 1) / np.sqrt(2) + + outputs = lin(outputs) + + if l < self.num_layers - 2: + outputs = self.softplus(outputs) + return outputs + + def get_sdf(self, ray_samples: RaySamples) -> TensorType["num_samples", -1, 1]: + """predict the sdf value for ray samples""" + positions = ray_samples.frustums.get_start_positions() + positions_flat = positions.view(-1, 3) + hidden_output = self.forward_geonetwork(positions_flat).view(*ray_samples.frustums.shape, -1) + sdf, _ = torch.split(hidden_output, [1, self.config.geo_feat_dim], dim=-1) + return sdf + + def get_alpha( + self, + ray_samples: RaySamples, + sdf: Optional[TensorType["num_samples", -1, 1]] = None, + gradients: Optional[TensorType["num_samples", -1, 1]] = None, + ) -> TensorType["num_samples", -1, 1]: + """compute alpha from sdf as in NeuS""" + if sdf is None or gradients is None: + inputs = ray_samples.frustums.get_start_positions() + inputs.requires_grad_(True) + with torch.enable_grad(): + hidden_output = self.forward_geonetwork(inputs) + sdf, _ = torch.split(hidden_output, [1, self.config.geo_feat_dim], dim=-1) + d_output = torch.ones_like(sdf, requires_grad=False, device=sdf.device) + gradients = torch.autograd.grad( + outputs=sdf, + inputs=inputs, + grad_outputs=d_output, + create_graph=True, + retain_graph=True, + only_inputs=True, + )[0] + + inv_s = self.deviation_network.get_variance() # Single parameter + + true_cos = (ray_samples.frustums.directions * gradients).sum(-1, keepdim=True) + + # anneal as NeuS + cos_anneal_ratio = self._cos_anneal_ratio + + # "cos_anneal_ratio" grows from 0 to 1 in the beginning training iterations. The anneal strategy below makes + # the cos value "not dead" at the beginning training iterations, for better convergence. + iter_cos = -( + F.relu(-true_cos * 0.5 + 0.5) * (1.0 - cos_anneal_ratio) + F.relu(-true_cos) * cos_anneal_ratio + ) # always non-positive + + # Estimate signed distances at section points + estimated_next_sdf = sdf + iter_cos * ray_samples.deltas * 0.5 + estimated_prev_sdf = sdf - iter_cos * ray_samples.deltas * 0.5 + + prev_cdf = torch.sigmoid(estimated_prev_sdf * inv_s) + next_cdf = torch.sigmoid(estimated_next_sdf * inv_s) + + p = prev_cdf - next_cdf + c = prev_cdf + + alpha = ((p + 1e-5) / (c + 1e-5)).clip(0.0, 1.0) + + return alpha + + def get_density(self, ray_samples: RaySamples): + raise NotImplementedError + + def get_colors( + self, + points: TensorType[..., 3], + directions: TensorType[..., 3], + normals: TensorType[..., 3], + geo_features: TensorType[..., "geo-feat-dim"], + camera_indices: TensorType, + ) -> TensorType[..., 3]: + """compute colors""" + d = self.direction_encoding(directions) + + # appearance + if self.training: + embedded_appearance = self.embedding_appearance(camera_indices) + # set it to zero if don't use it + if not self.config.use_appearance_embedding: + embedded_appearance = torch.zeros_like(embedded_appearance) + else: + if self.use_average_appearance_embedding: + embedded_appearance = torch.ones( + (*directions.shape[:-1], self.config.appearance_embedding_dim), device=directions.device + ) * self.embedding_appearance.mean(dim=0) + else: + embedded_appearance = torch.zeros( + (*directions.shape[:-1], self.config.appearance_embedding_dim), device=directions.device + ) + + hidden_input = torch.cat( + [ + points, + d, + normals, + geo_features.view(-1, self.config.geo_feat_dim), + embedded_appearance.view(-1, self.config.appearance_embedding_dim), + ], + dim=-1, + ) + + for layer in range(0, self.num_layers_color - 1): + lin = getattr(self, "clin" + str(layer)) + + hidden_input = lin(hidden_input) + + if layer < self.num_layers_color - 2: + hidden_input = self.relu(hidden_input) + + rgb = self.sigmoid(hidden_input) + + return rgb + + def get_outputs( + self, + ray_samples: RaySamples, + density_embedding: Optional[TensorType] = None, + return_alphas: bool = False, + ) -> Dict[FieldHeadNames, TensorType]: + """compute output of ray samples""" + if ray_samples.camera_indices is None: + raise AttributeError("Camera indices are not provided.") + + outputs = {} + + camera_indices = ray_samples.camera_indices.squeeze() + + inputs = ray_samples.frustums.get_start_positions() + inputs = inputs.view(-1, 3) + + directions = ray_samples.frustums.directions + directions_flat = directions.reshape(-1, 3) + + inputs.requires_grad_(True) + with torch.enable_grad(): + hidden_output = self.forward_geonetwork(inputs) + sdf, geo_feature = torch.split(hidden_output, [1, self.config.geo_feat_dim], dim=-1) + d_output = torch.ones_like(sdf, requires_grad=False, device=sdf.device) + gradients = torch.autograd.grad( + outputs=sdf, inputs=inputs, grad_outputs=d_output, create_graph=True, retain_graph=True, only_inputs=True + )[0] + + rgb = self.get_colors(inputs, directions_flat, gradients, geo_feature, camera_indices) + + rgb = rgb.view(*ray_samples.frustums.directions.shape[:-1], -1) + sdf = sdf.view(*ray_samples.frustums.directions.shape[:-1], -1) + gradients = gradients.view(*ray_samples.frustums.directions.shape[:-1], -1) + normals = torch.nn.functional.normalize(gradients, p=2, dim=-1) + + outputs.update( + { + FieldHeadNames.RGB: rgb, + FieldHeadNames.SDF: sdf, + FieldHeadNames.NORMALS: normals, + FieldHeadNames.GRADIENT: gradients, + } + ) + + if return_alphas: + alphas = self.get_alpha(ray_samples, sdf, gradients) + outputs.update({FieldHeadNames.ALPHA: alphas}) + + return outputs + + def forward( + self, ray_samples: RaySamples, compute_normals: bool = False, return_alphas: bool = False + ) -> Dict[FieldHeadNames, TensorType]: + """Evaluates the field at points along the ray. + + Args: + ray_samples: Samples to evaluate field on. + compute normals: not currently used in this implementation. + return_alphas: Whether to return alpha values + """ + field_outputs = self.get_outputs(ray_samples, return_alphas=return_alphas) + return field_outputs diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 419a8a1f4..174c131cc 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -23,7 +23,7 @@ from torchtyping import TensorType from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples -from nerfstudio.utils.math import masked_reduction +from nerfstudio.utils.math import masked_reduction, normalized_depth_scale_and_shift L1Loss = nn.L1Loss MSELoss = nn.MSELoss @@ -355,7 +355,8 @@ class MiDaSMSELoss(nn.Module): """ summed_mask = torch.sum(mask, (1, 2)) image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2)) - image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type) + # multiply by 2 magic number? + image_loss = masked_reduction(image_loss, 2 * summed_mask, self.reduction_type) return image_loss @@ -456,47 +457,6 @@ class ScaleAndShiftInvariantLoss(nn.Module): self.__prediction_ssi = None - @classmethod - def compute_scale_and_shift( - cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] - ): - """ - More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss - This function computes scale/shift required to normalizes predicted depth map, - to allow for using normalized depth maps as input from monocular depth estimation networks. - These networks are trained such that they predict normalized depth maps. - - Solves for scale/shift using a least squares approach with a closed form solution: - Based on: - https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7 - Args: - prediction: predicted depth map - target: ground truth depth map - mask: mask of valid pixels - Returns: - scale and shift for depth prediction - """ - # system matrix: A = [[a_00, a_01], [a_10, a_11]] - a_00 = torch.sum(mask * prediction * prediction, (1, 2)) - a_01 = torch.sum(mask * prediction, (1, 2)) - a_11 = torch.sum(mask, (1, 2)) - - # right hand side: b = [b_0, b_1] - b_0 = torch.sum(mask * prediction * target, (1, 2)) - b_1 = torch.sum(mask * target, (1, 2)) - - # solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b - scale = torch.zeros_like(b_0) - shift = torch.zeros_like(b_1) - - det = a_00 * a_11 - a_01 * a_01 - valid = det.nonzero() - - scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid] - shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid] - - return scale, shift - def forward( self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] ) -> TensorType[0]: @@ -508,7 +468,7 @@ class ScaleAndShiftInvariantLoss(nn.Module): Returns: scale and shift invariant loss """ - scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask) + scale, shift = normalized_depth_scale_and_shift(prediction, target, mask) self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1) total = self.__data_loss(self.__prediction_ssi, target, mask) diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index 866218ba1..d75bfec94 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -17,7 +17,7 @@ Collection of sampling strategies """ from abc import abstractmethod -from typing import Callable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple, Union import nerfacc import torch @@ -598,3 +598,166 @@ class ProposalNetworkSampler(Sampler): assert ray_samples is not None return ray_samples, weights_list, ray_samples_list + + +class NeuSSampler(Sampler): + """NeuS sampler that uses a sdf network to generate samples with fixed variance value in each iterations.""" + + def __init__( + self, + num_samples: int = 64, + num_samples_importance: int = 64, + num_samples_outside: int = 32, + num_upsample_steps: int = 4, + base_variance: float = 64, + single_jitter: bool = True, + ) -> None: + super().__init__() + self.num_samples = num_samples + self.num_samples_importance = num_samples_importance + self.num_samples_outside = num_samples_outside + self.num_upsample_steps = num_upsample_steps + self.base_variance = base_variance + self.single_jitter = single_jitter + + # samplers + self.uniform_sampler = UniformSampler(single_jitter=single_jitter) + self.pdf_sampler = PDFSampler( + include_original=False, + single_jitter=single_jitter, + histogram_padding=1e-5, + ) + self.outside_sampler = LinearDisparitySampler() + + def generate_ray_samples( + self, + ray_bundle: Optional[RayBundle] = None, + sdf_fn: Optional[Callable] = None, + ray_samples: Optional[RaySamples] = None, + ) -> Union[Tuple[RaySamples, torch.Tensor], RaySamples]: + assert ray_bundle is not None + assert sdf_fn is not None + + # Start with uniform sampling + if ray_samples is None: + ray_samples = self.uniform_sampler(ray_bundle, num_samples=self.num_samples) + + total_iters = 0 + sorted_index = None + new_samples = ray_samples + + base_variance = self.base_variance + + while total_iters < self.num_upsample_steps: + + with torch.no_grad(): + new_sdf = sdf_fn(new_samples) + + # merge sdf predictions + if sorted_index is not None: + sdf_merge = torch.cat([sdf.squeeze(-1), new_sdf.squeeze(-1)], -1) + sdf = torch.gather(sdf_merge, 1, sorted_index).unsqueeze(-1) + else: + sdf = new_sdf + + # compute with fix variances + alphas = self.rendering_sdf_with_fixed_inv_s( + ray_samples, sdf.reshape(ray_samples.shape), inv_s=base_variance * 2**total_iters + ) + + weights = ray_samples.get_weights_and_transmittance_from_alphas(alphas[..., None], weights_only=True) + weights = torch.cat((weights, torch.zeros_like(weights[:, :1])), dim=1) + + new_samples = self.pdf_sampler( + ray_bundle, + ray_samples, + weights, + num_samples=self.num_samples_importance // self.num_upsample_steps, + ) + + ray_samples, sorted_index = self.merge_ray_samples(ray_bundle, ray_samples, new_samples) + + total_iters += 1 + + return ray_samples + + @staticmethod + def rendering_sdf_with_fixed_inv_s( + ray_samples: RaySamples, sdf: TensorType["num_samples", -1], inv_s: int + ) -> TensorType["num_samples", -1]: + """ + rendering given a fixed inv_s as NeuS + + Args: + ray_samples: samples along ray + sdf: sdf values along ray + inv_s: fixed variance value + Returns: + alpha value + """ + batch_size = ray_samples.shape[0] + prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:] + deltas = ray_samples.deltas[:, :-1, 0] + mid_sdf = (prev_sdf + next_sdf) * 0.5 + cos_val = (next_sdf - prev_sdf) / (deltas + 1e-5) + + # ---------------------------------------------------------------------------------------------------------- + # Use min value of [ cos, prev_cos ] + # Though it makes the sampling (not rendering) a little bit biased, this strategy can make the sampling more + # robust when meeting situations like below: + # + # SDF + # ^ + # |\ -----x----... + # | \ / + # | x x + # |---\----/-------------> 0 level + # | \ / + # | \/ + # | + # ---------------------------------------------------------------------------------------------------------- + prev_cos_val = torch.cat([torch.zeros([batch_size, 1], device=sdf.device), cos_val[:, :-1]], dim=-1) + cos_val = torch.stack([prev_cos_val, cos_val], dim=-1) + cos_val, _ = torch.min(cos_val, dim=-1, keepdim=False) + cos_val = cos_val.clip(-1e3, 0.0) + + dist = deltas + prev_esti_sdf = mid_sdf - cos_val * dist * 0.5 + next_esti_sdf = mid_sdf + cos_val * dist * 0.5 + prev_cdf = torch.sigmoid(prev_esti_sdf * inv_s) + next_cdf = torch.sigmoid(next_esti_sdf * inv_s) + alpha = (prev_cdf - next_cdf + 1e-5) / (prev_cdf + 1e-5) + + return alpha + + @staticmethod + def merge_ray_samples(ray_bundle: RayBundle, ray_samples_1: RaySamples, ray_samples_2: RaySamples): + """Merge two set of ray samples and return sorted index which can be used to merge sdf values + Args: + ray_samples_1 : ray_samples to merge + ray_samples_2 : ray_samples to merge + """ + + starts_1 = ray_samples_1.spacing_starts[..., 0] + starts_2 = ray_samples_2.spacing_starts[..., 0] + + ends = torch.maximum(ray_samples_1.spacing_ends[..., -1:, 0], ray_samples_2.spacing_ends[..., -1:, 0]) + + bins, sorted_index = torch.sort(torch.cat([starts_1, starts_2], -1), -1) + + bins = torch.cat([bins, ends], dim=-1) + + # Stop gradients + bins = bins.detach() + + euclidean_bins = ray_samples_1.spacing_to_euclidean_fn(bins) + + ray_samples = ray_bundle.get_ray_samples( + bin_starts=euclidean_bins[..., :-1, None], + bin_ends=euclidean_bins[..., 1:, None], + spacing_starts=bins[..., :-1, None], + spacing_ends=bins[..., 1:, None], + spacing_to_euclidean_fn=ray_samples_1.spacing_to_euclidean_fn, + ) + + return ray_samples, sorted_index diff --git a/nerfstudio/models/base_surface_model.py b/nerfstudio/models/base_surface_model.py new file mode 100644 index 000000000..c83a07032 --- /dev/null +++ b/nerfstudio/models/base_surface_model.py @@ -0,0 +1,397 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Implementation of Base surface model. +""" + +from __future__ import annotations + +from abc import abstractmethod +from dataclasses import dataclass, field +from typing import Dict, List, Tuple, Type + +import torch +import torch.nn.functional as F +from torch.nn import Parameter +from torchmetrics import PeakSignalNoiseRatio +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity +from typing_extensions import Literal + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.field_components.encodings import NeRFEncoding +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.nerfacto_field import TCNNNerfactoField +from nerfstudio.fields.sdf_field import SDFFieldConfig +from nerfstudio.fields.vanilla_nerf_field import NeRFField +from nerfstudio.model_components.losses import ( + L1Loss, + MSELoss, + ScaleAndShiftInvariantLoss, + monosdf_normal_loss, +) +from nerfstudio.model_components.ray_samplers import LinearDisparitySampler +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + RGBRenderer, + SemanticRenderer, +) +from nerfstudio.model_components.scene_colliders import AABBBoxCollider, NearFarCollider +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps +from nerfstudio.utils.colors import get_color +from nerfstudio.utils.math import normalized_depth_scale_and_shift + + +@dataclass +class SurfaceModelConfig(ModelConfig): + """Surface Model Config""" + + _target: Type = field(default_factory=lambda: SurfaceModel) + near_plane: float = 0.05 + """How far along the ray to start sampling.""" + far_plane: float = 4.0 + """How far along the ray to stop sampling.""" + far_plane_bg: float = 1000.0 + """How far along the ray to stop sampling of the background model.""" + background_color: Literal["random", "last_sample", "white", "black"] = "black" + """Whether to randomize the background color.""" + use_average_appearance_embedding: bool = False + """Whether to use average appearance embedding or zeros for inference.""" + eikonal_loss_mult: float = 0.1 + """Monocular normal consistency loss multiplier.""" + fg_mask_loss_mult: float = 0.01 + """Foreground mask loss multiplier.""" + mono_normal_loss_mult: float = 0.0 + """Monocular normal consistency loss multiplier.""" + mono_depth_loss_mult: float = 0.0 + """Monocular depth consistency loss multiplier.""" + sdf_field: SDFFieldConfig = SDFFieldConfig() + """Config for SDF Field""" + background_model: Literal["grid", "mlp", "none"] = "mlp" + """background models""" + num_samples_outside: int = 32 + """Number of samples outside the bounding sphere for backgound""" + periodic_tvl_mult: float = 0.0 + """Total variational loss mutliplier""" + overwrite_near_far_plane: bool = False + """whether to use near and far collider from command line""" + + +class SurfaceModel(Model): + """Base surface model + + Args: + config: Base surface model configuration to instantiate model + """ + + config: SurfaceModelConfig + + def populate_modules(self): + """Set the fields and modules.""" + super().populate_modules() + + self.scene_contraction = SceneContraction(order=float("inf")) + + # Can we also use contraction for sdf? + # Fields + self.field = self.config.sdf_field.setup( + aabb=self.scene_box.aabb, + spatial_distortion=self.scene_contraction, + num_images=self.num_train_data, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, + ) + + # Collider + self.collider = AABBBoxCollider(self.scene_box, near_plane=0.05) + + # command line near and far has highest priority + if self.config.overwrite_near_far_plane: + self.collider = NearFarCollider(near_plane=self.config.near_plane, far_plane=self.config.far_plane) + + # background model + if self.config.background_model == "grid": + self.field_background = TCNNNerfactoField( + self.scene_box.aabb, + spatial_distortion=self.scene_contraction, + num_images=self.num_train_data, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, + ) + elif self.config.background_model == "mlp": + position_encoding = NeRFEncoding( + in_dim=3, num_frequencies=10, min_freq_exp=0.0, max_freq_exp=9.0, include_input=True + ) + direction_encoding = NeRFEncoding( + in_dim=3, num_frequencies=4, min_freq_exp=0.0, max_freq_exp=3.0, include_input=True + ) + + self.field_background = NeRFField( + position_encoding=position_encoding, + direction_encoding=direction_encoding, + spatial_distortion=self.scene_contraction, + ) + else: + # dummy background model + self.field_background = Parameter(torch.ones(1), requires_grad=False) + + self.sampler_bg = LinearDisparitySampler(num_samples=self.config.num_samples_outside) + + # renderers + background_color = ( + get_color(self.config.background_color) + if self.config.background_color in set(["white", "black"]) + else self.config.background_color + ) + self.renderer_rgb = RGBRenderer(background_color=background_color) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer(method="expected") + self.renderer_normal = SemanticRenderer() + + # losses + self.rgb_loss = L1Loss() + self.eikonal_loss = MSELoss() + self.depth_loss = ScaleAndShiftInvariantLoss(alpha=0.5, scales=1) + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity() + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + param_groups = {} + param_groups["fields"] = list(self.field.parameters()) + if self.config.background_model != "none": + param_groups["field_background"] = list(self.field_background.parameters()) + else: + param_groups["field_background"] = list(self.field_background) + return param_groups + + @abstractmethod + def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: + """Takes in a Ray Bundle and returns a dictionary of samples and field output. + + Args: + ray_bundle: Input bundle of rays. This raybundle should have all the + needed information to compute the outputs. + + Returns: + Outputs of model. (ie. rendered colors) + """ + + def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: + """Takes in a Ray Bundle and returns a dictionary of outputs. + + Args: + ray_bundle: Input bundle of rays. This raybundle should have all the + needed information to compute the outputs. + + Returns: + Outputs of model. (ie. rendered colors) + """ + samples_and_field_outputs = self.sample_and_forward_field(ray_bundle=ray_bundle) + + # shortcuts + field_outputs = samples_and_field_outputs["field_outputs"] + ray_samples = samples_and_field_outputs["ray_samples"] + weights = samples_and_field_outputs["weights"] + bg_transmittance = samples_and_field_outputs["bg_transmittance"] + + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + # the rendered depth is point-to-point distance and we should convert to depth + depth = depth / ray_bundle.metadata["directions_norm"] + + normal = self.renderer_normal(semantics=field_outputs[FieldHeadNames.NORMALS], weights=weights) + accumulation = self.renderer_accumulation(weights=weights) + + # background model + if self.config.background_model != "none": + # sample inversely from far to 1000 and points and forward the bg model + ray_bundle.nears = ray_bundle.fars + ray_bundle.fars = torch.ones_like(ray_bundle.fars) * self.config.far_plane_bg + + ray_samples_bg = self.sampler_bg(ray_bundle) + # use the same background model for both density field and occupancy field + field_outputs_bg = self.field_background(ray_samples_bg) + weights_bg = ray_samples_bg.get_weights(field_outputs_bg[FieldHeadNames.DENSITY]) + + rgb_bg = self.renderer_rgb(rgb=field_outputs_bg[FieldHeadNames.RGB], weights=weights_bg) + depth_bg = self.renderer_depth(weights=weights_bg, ray_samples=ray_samples_bg) + accumulation_bg = self.renderer_accumulation(weights=weights_bg) + + # merge background color to forgound color + rgb = rgb + bg_transmittance * rgb_bg + + bg_outputs = { + "bg_rgb": rgb_bg, + "bg_accumulation": accumulation_bg, + "bg_depth": depth_bg, + "bg_weights": weights_bg, + } + else: + bg_outputs = {} + + outputs = { + "rgb": rgb, + "accumulation": accumulation, + "depth": depth, + "normal": normal, + "weights": weights, + # used to scale z_vals for free space and sdf loss + "directions_norm": ray_bundle.metadata["directions_norm"], + } + outputs.update(bg_outputs) + + if self.training: + grad_points = field_outputs[FieldHeadNames.GRADIENT] + outputs.update({"eik_grad": grad_points}) + outputs.update(samples_and_field_outputs) + + if "weights_list" in samples_and_field_outputs: + weights_list = samples_and_field_outputs["weights_list"] + ray_samples_list = samples_and_field_outputs["ray_samples_list"] + + for i in range(len(weights_list) - 1): + outputs[f"prop_depth_{i}"] = self.renderer_depth( + weights=weights_list[i], ray_samples=ray_samples_list[i] + ) + # this is used only in viewer + outputs["normal_vis"] = (outputs["normal"] + 1.0) / 2.0 + return outputs + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + """Computes and returns the losses dict. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + metrics_dict: dictionary of metrics, some of which we can use for loss + """ + loss_dict = {} + image = batch["image"].to(self.device) + loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"]) + if self.training: + # eikonal loss + grad_theta = outputs["eik_grad"] + loss_dict["eikonal_loss"] = ((grad_theta.norm(2, dim=-1) - 1) ** 2).mean() * self.config.eikonal_loss_mult + + # foreground mask loss + if "fg_mask" in batch and self.config.fg_mask_loss_mult > 0.0: + fg_label = batch["fg_mask"].float().to(self.device) + weights_sum = outputs["weights"].sum(dim=1).clip(1e-3, 1.0 - 1e-3) + loss_dict["fg_mask_loss"] = ( + F.binary_cross_entropy(weights_sum, fg_label) * self.config.fg_mask_loss_mult + ) + + # monocular normal loss + if "normal" in batch and self.config.mono_normal_loss_mult > 0.0: + normal_gt = batch["normal"].to(self.device) + normal_pred = outputs["normal"] + loss_dict["normal_loss"] = ( + monosdf_normal_loss(normal_pred, normal_gt) * self.config.mono_normal_loss_mult + ) + + # monocular depth loss + if "depth" in batch and self.config.mono_depth_loss_mult > 0.0: + depth_gt = batch["depth"].to(self.device)[..., None] + depth_pred = outputs["depth"] + + mask = torch.ones_like(depth_gt).reshape(1, 32, -1).bool() + loss_dict["depth_loss"] = ( + self.depth_loss(depth_pred.reshape(1, 32, -1), (depth_gt * 50 + 0.5).reshape(1, 32, -1), mask) + * self.config.mono_depth_loss_mult + ) + + return loss_dict + + def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]: + """Compute and returns metrics. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + """ + metrics_dict = {} + image = batch["image"].to(self.device) + metrics_dict["psnr"] = self.psnr(outputs["rgb"], image) + return metrics_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Writes the test image outputs. + Args: + outputs: Outputs of the model. + batch: Batch of data. + + Returns: + A dictionary of metrics. + """ + image = batch["image"].to(self.device) + rgb = outputs["rgb"] + acc = colormaps.apply_colormap(outputs["accumulation"]) + + normal = outputs["normal"] + normal = (normal + 1.0) / 2.0 + + combined_rgb = torch.cat([image, rgb], dim=1) + combined_acc = torch.cat([acc], dim=1) + if "depth" in batch: + depth_gt = batch["depth"].to(self.device) + depth_pred = outputs["depth"] + + # align to predicted depth and normalize + scale, shift = normalized_depth_scale_and_shift( + depth_pred[None, ..., 0], depth_gt[None, ...], depth_gt[None, ...] > 0.0 + ) + depth_pred = depth_pred * scale + shift + + combined_depth = torch.cat([depth_gt[..., None], depth_pred], dim=1) + combined_depth = colormaps.apply_depth_colormap(combined_depth) + else: + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + combined_depth = torch.cat([depth], dim=1) + + if "normal" in batch: + normal_gt = (batch["normal"].to(self.device) + 1.0) / 2.0 + combined_normal = torch.cat([normal_gt, normal], dim=1) + else: + combined_normal = torch.cat([normal], dim=1) + + images_dict = { + "img": combined_rgb, + "accumulation": combined_acc, + "depth": combined_depth, + "normal": combined_normal, + } + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + image = torch.moveaxis(image, -1, 0)[None, ...] + rgb = torch.moveaxis(rgb, -1, 0)[None, ...] + + psnr = self.psnr(image, rgb) + ssim = self.ssim(image, rgb) + lpips = self.lpips(image, rgb) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + return metrics_dict, images_dict diff --git a/nerfstudio/models/neus.py b/nerfstudio/models/neus.py new file mode 100644 index 000000000..0e88b6843 --- /dev/null +++ b/nerfstudio/models/neus.py @@ -0,0 +1,119 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Implementation of NeuS. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Type + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.model_components.ray_samplers import NeuSSampler +from nerfstudio.models.base_surface_model import SurfaceModel, SurfaceModelConfig + + +@dataclass +class NeuSModelConfig(SurfaceModelConfig): + """NeuS Model Config""" + + _target: Type = field(default_factory=lambda: NeuSModel) + num_samples: int = 64 + """Number of uniform samples""" + num_samples_importance: int = 64 + """Number of importance samples""" + num_up_sample_steps: int = 4 + """number of up sample step, 1 for simple coarse-to-fine sampling""" + base_variance: float = 64 + """fixed base variance in NeuS sampler, the inv_s will be base * 2 ** iter during upsample""" + perturb: bool = True + """use to use perturb for the sampled points""" + + +class NeuSModel(SurfaceModel): + """NeuS model + + Args: + config: NeuS configuration to instantiate model + """ + + config: NeuSModelConfig + + def populate_modules(self): + """Set the fields and modules.""" + super().populate_modules() + + self.sampler = NeuSSampler( + num_samples=self.config.num_samples, + num_samples_importance=self.config.num_samples_importance, + num_samples_outside=self.config.num_samples_outside, + num_upsample_steps=self.config.num_up_sample_steps, + base_variance=self.config.base_variance, + ) + + self.anneal_end = 50000 + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + callbacks = [] + # anneal for cos in NeuS + if self.anneal_end > 0: + + def set_anneal(step): + anneal = min([1.0, step / self.anneal_end]) + self.field.set_cos_anneal_ratio(anneal) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ) + ) + + return callbacks + + def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict: + ray_samples = self.sampler(ray_bundle, sdf_fn=self.field.get_sdf) + field_outputs = self.field(ray_samples, return_alphas=True) + weights, transmittance = ray_samples.get_weights_and_transmittance_from_alphas( + field_outputs[FieldHeadNames.ALPHA] + ) + bg_transmittance = transmittance[:, -1, :] + + samples_and_field_outputs = { + "ray_samples": ray_samples, + "field_outputs": field_outputs, + "weights": weights, + "bg_transmittance": bg_transmittance, + } + return samples_and_field_outputs + + def get_metrics_dict(self, outputs, batch) -> Dict: + metrics_dict = super().get_metrics_dict(outputs, batch) + if self.training: + # training statics + metrics_dict["s_val"] = self.field.deviation_network.get_variance().item() + metrics_dict["inv_s"] = 1.0 / self.field.deviation_network.get_variance().item() + + return metrics_dict diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index b0a0aaec4..f54b6509a 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -311,3 +311,44 @@ def masked_reduction( input_tensor[valid] = input_tensor[valid] / mask[valid] input_tensor = torch.mean(input_tensor) return input_tensor + + +def normalized_depth_scale_and_shift( + prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] +): + """ + More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss + This function computes scale/shift required to normalizes predicted depth map, + to allow for using normalized depth maps as input from monocular depth estimation networks. + These networks are trained such that they predict normalized depth maps. + + Solves for scale/shift using a least squares approach with a closed form solution: + Based on: + https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7 + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + scale and shift for depth prediction + """ + # system matrix: A = [[a_00, a_01], [a_10, a_11]] + a_00 = torch.sum(mask * prediction * prediction, (1, 2)) + a_01 = torch.sum(mask * prediction, (1, 2)) + a_11 = torch.sum(mask, (1, 2)) + + # right hand side: b = [b_0, b_1] + b_0 = torch.sum(mask * prediction * target, (1, 2)) + b_1 = torch.sum(mask * target, (1, 2)) + + # solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b + scale = torch.zeros_like(b_0) + shift = torch.zeros_like(b_1) + + det = a_00 * a_11 - a_01 * a_01 + valid = det.nonzero() + + scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid] + shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid] + + return scale, shift diff --git a/tests/test_train.py b/tests/test_train.py index 89aa1c29a..b38e84205 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -26,6 +26,7 @@ BLACKLIST = [ "depth-nerfacto", "nerfplayer-ngp", "nerfplayer-nerfacto", + "neus", ]