diff --git a/docs/nerfology/models/index.md b/docs/nerfology/models/index.md index d1e50bce0..6424b60c9 100644 --- a/docs/nerfology/models/index.md +++ b/docs/nerfology/models/index.md @@ -20,6 +20,5 @@ In addition to their implementations, we have provided guides that walk through Mip-NeRF Mip-NeRF 360 Instant-NGP - NeRF-W - Semantic NeRF + Semantic NeRF-W ``` diff --git a/docs/nerfology/models/nerf_w.md b/docs/nerfology/models/nerf_w.md deleted file mode 100644 index f935e681f..000000000 --- a/docs/nerfology/models/nerf_w.md +++ /dev/null @@ -1,15 +0,0 @@ -# NeRF-W - -

NeRF in the Wild

- -```{button-link} https://nerf-w.github.io/ -:color: primary -:outline: -Paper Website -``` - -### Running Model - -```bash -python scripts/train.py --config-name=graph_nerf_w.yaml -``` diff --git a/docs/nerfology/models/semantic_nerf.md b/docs/nerfology/models/semantic_nerf.md deleted file mode 100644 index 0a86bbc9d..000000000 --- a/docs/nerfology/models/semantic_nerf.md +++ /dev/null @@ -1,7 +0,0 @@ -# Semantic NeRF - -### Running Model - -```bash -python scripts/train.py --config-name=graph_semantic_nerf.yaml -``` diff --git a/docs/nerfology/models/semantic_nerfw.md b/docs/nerfology/models/semantic_nerfw.md new file mode 100644 index 000000000..79f1c5b35 --- /dev/null +++ b/docs/nerfology/models/semantic_nerfw.md @@ -0,0 +1,31 @@ +# Semantic NeRF-W + +

Semantic NeRF

+ +```{button-link} https://shuaifengzhi.com/Semantic-NeRF/ +:color: primary +:outline: +Paper Website +``` + +

NeRF in the Wild

+ +```{button-link} https://nerf-w.github.io/ +:color: primary +:outline: +Paper Website +``` + +```{admonition} Notice +The transient embeddings are still under development. Please stay tuned. +``` + +### Running Model + +```bash +ns-train semantic-nerfw +``` + +This model defaults to using the "friends" dataset from the paper ["The One Where They Reconstructed 3D Humans and Environments in TV Shows"](https://ethanweber.me/sitcoms3D/). + + \ No newline at end of file diff --git a/docs/reference/api/models.rst b/docs/reference/api/models.rst index 6428b8064..0421fbc24 100644 --- a/docs/reference/api/models.rst +++ b/docs/reference/api/models.rst @@ -17,10 +17,10 @@ Instant NGP :members: :show-inheritance: -Semantic NeRF +Semantic NeRF-W ---------------- -.. automodule:: nerfstudio.models.semantic_nerf +.. automodule:: nerfstudio.models.semantic_nerfw :members: :show-inheritance: diff --git a/nerfstudio/configs/model_configs.py b/nerfstudio/configs/model_configs.py index 9b1ed8fe4..c7301f6ba 100644 --- a/nerfstudio/configs/model_configs.py +++ b/nerfstudio/configs/model_configs.py @@ -35,8 +35,7 @@ from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.mipnerf_360 import MipNerf360Model from nerfstudio.models.nerfacto import NerfactoModelConfig -from nerfstudio.models.nerfw import NerfWModelConfig -from nerfstudio.models.semantic_nerf import SemanticNerfModel +from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig @@ -49,8 +48,7 @@ descriptions = { "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for bounded synthetic data.", "mipnerf-360": "High quality model for unbounded 360 degree scenes. [red]*slow*", "mipnerf": "High quality model for bounded scenes. [red]*slow*", - "nerfw": "Model designed to handle inconsistent appearance between images. [red]*slow*", - "semantic-nerf": "Model that predicts dense semantic segmentations. [red]*slow*", + "semantic-nerfw": "Predicts semantic segmentations and filters out transient objects.", "vanilla-nerf": "Original NeRF model. [red]*slow*", "tensorf": "Fast model designed for bounded scenes.", } @@ -137,42 +135,27 @@ model_configs["mipnerf"] = Config( }, ) -model_configs["nerfw"] = Config( - method_name="nerfw", +model_configs["semantic-nerfw"] = Config( + method_name="semantic-nerfw", + trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True), pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig( - dataparser=FriendsDataParserConfig(), + dataparser=FriendsDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=8192 ), - model=NerfWModelConfig(), + model=SemanticNerfWModelConfig(eval_num_rays_per_chunk=1 << 16), ), optimizers={ - "fields": { - "optimizer": RAdamOptimizerConfig(lr=5e-4, eps=1e-08), + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": None, - } - }, -) - - -model_configs["semantic-nerf"] = Config( - method_name="semantic-nerf", - pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( - dataparser=FriendsDataParserConfig(), - ), - model=VanillaModelConfig( - _target=SemanticNerfModel, - loss_coefficients={"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0, "semantic_loss_fine": 0.05}, - num_coarse_samples=64, - num_importance_samples=64, - ), - ), - optimizers={ + }, "fields": { - "optimizer": RAdamOptimizerConfig(lr=5e-4, eps=1e-08), + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": None, - } + }, }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 16), + vis="viewer", ) model_configs["vanilla-nerf"] = Config( diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 1dce5468b..7b0e00a99 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -60,6 +60,8 @@ class DataparserOutputs: """color of dataset background""" scene_box: SceneBox = SceneBox() """scene box of dataset. Could be used to bound the scene, or to provide the scene scale depending on model.""" + semantics: Optional[Semantics] = None + """Semantics information.""" # we support additional input information/formats including mask/depth/semantics. additional_inputs: Dict[str, Any] = to_immutable_dict({}) """Dictionary of additional dataset information (e.g. semantics/point clouds/masks). diff --git a/nerfstudio/data/dataparsers/friends_dataparser.py b/nerfstudio/data/dataparsers/friends_dataparser.py index 05cd37cc8..7a1afa69e 100644 --- a/nerfstudio/data/dataparsers/friends_dataparser.py +++ b/nerfstudio/data/dataparsers/friends_dataparser.py @@ -51,10 +51,15 @@ def get_semantics_and_masks(image_idx: int, semantics: Semantics): thing_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None] mask = (thing_semantics != person_index).to(torch.float32) # 1 where valid # handle semantics + # stuff stuff_image_filename = semantics.stuff_filenames[image_idx] pil_image = Image.open(stuff_image_filename) stuff_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None] - return {"mask": mask, "semantics": stuff_semantics} + # thing + thing_image_filename = semantics.thing_filenames[image_idx] + pil_image = Image.open(thing_image_filename) + thing_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None] + return {"mask": mask, "semantics_stuff": stuff_semantics, "semantics_thing": thing_semantics} @dataclass @@ -67,8 +72,8 @@ class FriendsDataParserConfig(DataParserConfig): """Directory specifying location of data.""" include_semantics: bool = True """whether or not to include loading of semantics data""" - downscale_factor: int = 8 - scene_scale: float = 4.0 + downscale_factor: int = 4 + scene_scale: float = 2.0 """ Sets the bounding cube to have edge length of this size. The longest dimension of the Friends axis-aligned bbox will be scaled to this value. @@ -187,5 +192,6 @@ class Friends(DataParser): cameras=cameras, scene_box=scene_box, additional_inputs={"semantics": {"func": get_semantics_and_masks, "kwargs": {"semantics": semantics}}}, + semantics=semantics, ) return dataparser_outputs diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index e982f7ae1..8ba19ba63 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -49,11 +49,13 @@ def collate_image_dataset_batch(batch: Dict, num_rays_per_batch: int, keep_full_ c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) image = batch["image"][c, y, x] - mask, semantics = None, None + mask, semantics_stuff, semantics_thing = None, None, None if "mask" in batch: mask = batch["mask"][c, y, x] - if "semantics" in batch: - semantics = batch["semantics"][c, y, x] + if "semantics_stuff" in batch: + semantics_stuff = batch["semantics_stuff"][c, y, x] + if "semantics_thing" in batch: + semantics_thing = batch["semantics_thing"][c, y, x] assert image.shape == (num_rays_per_batch, 3), image.shape # Needed to correct the random indices to their actual camera idx locations. @@ -66,8 +68,10 @@ def collate_image_dataset_batch(batch: Dict, num_rays_per_batch: int, keep_full_ } if mask is not None: collated_batch["mask"] = mask - if semantics is not None: - collated_batch["semantics"] = semantics + if semantics_stuff is not None: + collated_batch["semantics_stuff"] = semantics_stuff + if semantics_thing is not None: + collated_batch["semantics_thing"] = semantics_thing if keep_full_image: collated_batch["full_image"] = batch["image"] diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index 336a2d04f..c11a827d1 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -34,6 +34,8 @@ class FieldHeadNames(Enum): TRANSIENT_RGB = "transient_rgb" TRANSIENT_DENSITY = "transient_density" SEMANTICS = "semantics" + SEMANTICS_STUFF = "semantics_stuff" + SEMANTICS_THING = "semantics_thing" class FieldHead(FieldComponent): @@ -179,3 +181,33 @@ class SemanticFieldHead(FieldHead): def __init__(self, num_classes: int, in_dim: Optional[int] = None) -> None: super().__init__(in_dim=in_dim, out_dim=num_classes, field_head_name=FieldHeadNames.SEMANTICS, activation=None) + + +class SemanticStuffFieldHead(FieldHead): + """Semantic stuff output + + Args: + num_classes: Number of semantic classes + in_dim: input dimension. If not defined in constructor, it must be set later. + activation: output head activation + """ + + def __init__(self, num_classes: int, in_dim: Optional[int] = None) -> None: + super().__init__( + in_dim=in_dim, out_dim=num_classes, field_head_name=FieldHeadNames.SEMANTICS_STUFF, activation=None + ) + + +class SemanticThingFieldHead(FieldHead): + """Semantic thing output + + Args: + num_classes: Number of semantic classes + in_dim: input dimension. If not defined in constructor, it must be set later. + activation: output head activation + """ + + def __init__(self, num_classes: int, in_dim: Optional[int] = None) -> None: + super().__init__( + in_dim=in_dim, out_dim=num_classes, field_head_name=FieldHeadNames.SEMANTICS_THING, activation=None + ) diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index e6c3873da..86e479e48 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -35,6 +35,11 @@ from nerfstudio.field_components.field_heads import ( FieldHead, FieldHeadNames, RGBFieldHead, + SemanticStuffFieldHead, + SemanticThingFieldHead, + TransientDensityFieldHead, + TransientRGBFieldHead, + UncertaintyFieldHead, ) from nerfstudio.field_components.mlp import MLP from nerfstudio.field_components.spatial_distortions import ( @@ -70,6 +75,7 @@ class TCNNNerfactoField(Field): num_layers_color: number of hidden layers for color network hidden_dim_color: dimension of hidden layers for color network appearance_embedding_dim: dimension of appearance embedding + transient_embedding_dim: dimension of transient embedding use_average_appearance_embedding: whether to use average appearance embedding or zeros for inference spatial_distortion: spatial distortion to apply to the scene """ @@ -82,8 +88,15 @@ class TCNNNerfactoField(Field): hidden_dim: int = 64, geo_feat_dim: int = 15, num_layers_color: int = 3, + num_layers_transient: int = 2, hidden_dim_color: int = 64, + hidden_dim_transient: int = 64, appearance_embedding_dim: int = 32, + transient_embedding_dim: int = 16, + use_transient_embedding: bool = False, + use_semantics: bool = False, + num_semantics_thing_classes: int = 100, + num_semantics_stuff_classes: int = 100, use_average_appearance_embedding: bool = False, spatial_distortion: Optional[SpatialDistortion] = None, ) -> None: @@ -97,6 +110,8 @@ class TCNNNerfactoField(Field): self.appearance_embedding_dim = appearance_embedding_dim self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) self.use_average_appearance_embedding = use_average_appearance_embedding + self.use_transient_embedding = use_transient_embedding + self.use_semantics = use_semantics num_levels = 16 max_res = 1024 @@ -133,6 +148,45 @@ class TCNNNerfactoField(Field): }, ) + # transients + if self.use_transient_embedding: + self.transient_embedding_dim = transient_embedding_dim + self.embedding_transient = Embedding(self.num_images, self.transient_embedding_dim) + self.mlp_transient = tcnn.Network( + n_input_dims=self.geo_feat_dim + self.transient_embedding_dim, + n_output_dims=hidden_dim_transient, + network_config={ + "otype": "FullyFusedMLP", + "activation": "ReLU", + "output_activation": "None", + "n_neurons": hidden_dim_transient, + "n_hidden_layers": num_layers_transient - 1, + }, + ) + self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.n_output_dims) + self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.n_output_dims) + self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.n_output_dims) + + # semantics + if self.use_semantics: + self.mlp_semantics = tcnn.Network( + n_input_dims=self.geo_feat_dim, + n_output_dims=hidden_dim_transient, + network_config={ + "otype": "FullyFusedMLP", + "activation": "ReLU", + "output_activation": "None", + "n_neurons": 64, + "n_hidden_layers": 1, + }, + ) + self.field_head_semantics_stuff = SemanticStuffFieldHead( + in_dim=self.mlp_semantics.n_output_dims, num_classes=num_semantics_stuff_classes + ) + self.field_head_semantics_thing = SemanticThingFieldHead( + in_dim=self.mlp_semantics.n_output_dims, num_classes=num_semantics_thing_classes + ) + self.mlp_head = tcnn.Network( n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim, n_output_dims=3, @@ -164,6 +218,8 @@ class TCNNNerfactoField(Field): return density, base_mlp_out def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None): + assert density_embedding is not None + outputs = {} if ray_samples.camera_indices is None: raise AttributeError("Camera indices are not provided.") camera_indices = ray_samples.camera_indices.squeeze() @@ -171,6 +227,7 @@ class TCNNNerfactoField(Field): directions_flat = directions.view(-1, 3) d = self.direction_encoding(directions_flat) + # appearance if self.training: embedded_appearance = self.embedding_appearance(camera_indices) else: @@ -183,21 +240,47 @@ class TCNNNerfactoField(Field): (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device ) - if density_embedding is None: - positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) - h = torch.cat([d, positions.view(-1, 3), embedded_appearance], dim=-1) - else: - h = torch.cat( + # transients + if self.use_transient_embedding and self.training: + embedded_transient = self.embedding_transient(camera_indices) + transient_input = torch.cat( [ - d, density_embedding.view(-1, self.geo_feat_dim), - embedded_appearance.view(-1, self.appearance_embedding_dim), + embedded_transient.view(-1, self.transient_embedding_dim), ], dim=-1, ) - rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions) + x = self.mlp_transient(transient_input).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions) + outputs[FieldHeadNames.UNCERTAINTY] = self.field_head_transient_uncertainty(x) + outputs[FieldHeadNames.TRANSIENT_RGB] = self.field_head_transient_rgb(x) + outputs[FieldHeadNames.TRANSIENT_DENSITY] = self.field_head_transient_density(x) - return {FieldHeadNames.RGB: rgb} + # semantics + if self.use_semantics: + density_embedding_copy = density_embedding.clone().detach() + semantics_input = torch.cat( + [ + density_embedding_copy.view(-1, self.geo_feat_dim), + ], + dim=-1, + ) + # print(semantics_input) + x = self.mlp_semantics(semantics_input).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions) + outputs[FieldHeadNames.SEMANTICS_STUFF] = self.field_head_semantics_stuff(x) + outputs[FieldHeadNames.SEMANTICS_THING] = self.field_head_semantics_thing(x) + + h = torch.cat( + [ + d, + density_embedding.view(-1, self.geo_feat_dim), + embedded_appearance.view(-1, self.appearance_embedding_dim), + ], + dim=-1, + ) + rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions) + outputs.update({FieldHeadNames.RGB: rgb}) + + return outputs class TorchNerfactoField(Field): diff --git a/nerfstudio/models/nerfw.py b/nerfstudio/models/nerfw.py deleted file mode 100644 index 5a03852df..000000000 --- a/nerfstudio/models/nerfw.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright 2022 The Nerfstudio Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -NeRF-W (NeRF in the wild) implementation. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Dict, Tuple, Type - -import torch -from torchmetrics import PeakSignalNoiseRatio -from torchmetrics.functional import structural_similarity_index_measure -from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity - -from nerfstudio.cameras.rays import RayBundle -from nerfstudio.configs.config_utils import to_immutable_dict -from nerfstudio.field_components.encodings import NeRFEncoding -from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.nerfw_field import VanillaNerfWField -from nerfstudio.fields.vanilla_nerf_field import NeRFField -from nerfstudio.model_components.losses import MSELoss -from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler -from nerfstudio.model_components.renderers import ( - AccumulationRenderer, - DepthRenderer, - RGBRenderer, - UncertaintyRenderer, -) -from nerfstudio.model_components.scene_colliders import AABBBoxCollider -from nerfstudio.models.base_model import Model, VanillaModelConfig -from nerfstudio.utils import colormaps, colors, misc - - -@dataclass -class NerfWModelConfig(VanillaModelConfig): - """NerfW model config""" - - _target: Type = field(default_factory=lambda: NerfWModel) - """target class to instantiate""" - loss_coefficients: Dict[str, float] = to_immutable_dict( - {"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0, "uncertainty_loss": 1.0, "density_loss": 0.01} - ) - """Loss specific weights.""" - num_coarse_samples: int = 64 - """Number of samples in coarse field evaluation.""" - num_importance_samples: int = 64 - """Number of samples in fine field evaluation.""" - uncertainty_min: float = 0.03 - """This is added to the end of the uncertainty - rendering operation. It's called 'beta_min' in other repos. - This avoids calling torch.log() on a zero value, which would be undefined. - """ - num_images: int = 10000 # TODO: don't hardcode this - """How many images exist in the dataset.""" - appearance_embedding_dim: int = 48 - """Dimension of appearance embedding.""" - transient_embedding_dim: int = 16 - """Dimension of transient embedding.""" - - -class NerfWModel(Model): - """NeRF-W model - - Args: - config: NerfW configuration to instantiate model - """ - - config: NerfWModelConfig - - def __init__( - self, - config: NerfWModelConfig, - **kwargs, - ) -> None: - """A NeRF-W model. - - Args: - ... - uncertainty_min (float, optional): This is added to the end of the uncertainty - rendering operation. It's called 'beta_min' in other repos. - This avoids calling torch.log() on a zero value, which would be undefined. - """ - self.field_coarse = None - self.field_fine = None - super().__init__(config=config, **kwargs) - - def populate_modules(self): - """Set the fields and modules""" - super().populate_modules() - - # setting up fields - position_encoding = NeRFEncoding( - in_dim=3, num_frequencies=10, min_freq_exp=0.0, max_freq_exp=8.0, include_input=True - ) - direction_encoding = NeRFEncoding( - in_dim=3, num_frequencies=4, min_freq_exp=0.0, max_freq_exp=4.0, include_input=True - ) - - self.field_coarse = NeRFField(position_encoding=position_encoding, direction_encoding=direction_encoding) - self.field_fine = VanillaNerfWField( - num_images=self.config.num_images, - position_encoding=position_encoding, - direction_encoding=direction_encoding, - appearance_embedding_dim=self.config.appearance_embedding_dim, - transient_embedding_dim=self.config.transient_embedding_dim, - ) - - # samplers - self.sampler_uniform = UniformSampler(num_samples=self.config.num_coarse_samples) - self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples) - - # renderers - self.renderer_rgb = RGBRenderer(background_color=colors.BLACK) - self.renderer_accumulation = AccumulationRenderer() - self.renderer_depth = DepthRenderer() - self.renderer_uncertainty = UncertaintyRenderer() - - # losses - self.rgb_loss = MSELoss() - - # metrics - self.psnr = PeakSignalNoiseRatio(data_range=1.0) - self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() - - # colliders - if self.config.enable_collider: - self.collider = AABBBoxCollider(scene_box=self.scene_box) - - def get_param_groups(self): - param_groups = {} - if self.field_coarse is None or self.field_fine is None: - raise ValueError("populate_fields() must be called before get_param_groups") - param_groups["fields"] = list(self.field_coarse.parameters()) + list(self.field_fine.parameters()) - return param_groups - - def get_outputs(self, ray_bundle: RayBundle): - - if ray_bundle.camera_indices is not None: - # TODO(ethan): remove this check - assert ( - torch.max(ray_bundle.camera_indices) < self.config.num_images - ), "num_images must be greater than the largest camera index" - - if self.field_coarse is None or self.field_fine is None: - raise ValueError("populate_fields() must be called before get_outputs") - - # uniform sampling - ray_samples_uniform = self.sampler_uniform(ray_bundle) - - # coarse field - field_outputs_coarse = self.field_coarse.forward(ray_samples_uniform) - weights_coarse = ray_samples_uniform.get_weights(field_outputs_coarse[FieldHeadNames.DENSITY]) - rgb_coarse = self.renderer_rgb( - rgb=field_outputs_coarse[FieldHeadNames.RGB], - weights=weights_coarse, - ) - depth_coarse = self.renderer_depth(weights_coarse, ray_samples_uniform) - - # pdf sampling - ray_samples_pdf = self.sampler_pdf(ray_bundle, ray_samples_uniform, weights_coarse) - - # fine field - field_outputs_fine = self.field_fine.forward(ray_samples_pdf) - - # fine weights - weights_fine = ray_samples_pdf.get_weights( - field_outputs_fine[FieldHeadNames.DENSITY] + field_outputs_fine[FieldHeadNames.TRANSIENT_DENSITY] - ) - weights_fine_static = ray_samples_pdf.get_weights(field_outputs_fine[FieldHeadNames.DENSITY]) - weights_fine_transient = ray_samples_pdf.get_weights(field_outputs_fine[FieldHeadNames.TRANSIENT_DENSITY]) - - # rgb - rgb_fine_static_component = self.renderer_rgb( - rgb=field_outputs_fine[FieldHeadNames.RGB], - weights=weights_fine, - ) - rgb_fine_transient_component = self.renderer_rgb( - rgb=field_outputs_fine[FieldHeadNames.TRANSIENT_RGB], - weights=weights_fine, - ) - rgb_fine = rgb_fine_static_component + rgb_fine_transient_component - rgb_fine_static = self.renderer_rgb( - rgb=field_outputs_fine[FieldHeadNames.RGB], - weights=weights_fine_static, - ) - - # density - density_transient = field_outputs_fine[FieldHeadNames.TRANSIENT_DENSITY] - - # depth - depth_fine = self.renderer_depth(weights_fine, ray_samples_pdf) - depth_fine_static = self.renderer_depth(weights_fine_static, ray_samples_pdf) - - # uncertainty - uncertainty = self.renderer_uncertainty(field_outputs_fine[FieldHeadNames.UNCERTAINTY], weights_fine_transient) - uncertainty += self.config.uncertainty_min - - outputs = { - "rgb_coarse": rgb_coarse, # (num_rays, 3) - "rgb_fine": rgb_fine, - "rgb_fine_static": rgb_fine_static, - "depth_coarse": depth_coarse, - "depth_fine": depth_fine, - "depth_fine_static": depth_fine_static, - "density_transient": density_transient, # (num_rays, num_samples, 1) - "uncertainty": uncertainty, # (num_rays, 1) - } - return outputs - - def get_loss_dict(self, outputs, batch, metrics_dict=None): - device = outputs["rgb_coarse"].device - image = batch["image"].to(device) - rgb_coarse = outputs["rgb_coarse"] - rgb_fine = outputs["rgb_fine"] - density_transient = outputs["density_transient"] - betas = outputs["uncertainty"] - rgb_loss_coarse = 0.5 * ((image - rgb_coarse) ** 2).sum(-1).mean() - rgb_loss_fine = 0.5 * (((image - rgb_fine) ** 2).sum(-1) / (betas[..., 0] ** 2)).mean() - uncertainty_loss = 0.5 * (3 + torch.log(betas)).mean() - density_loss = density_transient.mean() - - loss_dict = { - "rgb_loss_coarse": rgb_loss_coarse, - "rgb_loss_fine": rgb_loss_fine, - "uncertainty_loss": uncertainty_loss, - "density_loss": density_loss, - } - loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) - return loss_dict - - def get_image_metrics_and_images( - self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] - ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: - image = batch["image"].to(outputs["rgb_coarse"].device) - rgb_coarse = outputs["rgb_coarse"] - rgb_fine = outputs["rgb_fine"] - rgb_fine_static = outputs["rgb_fine_static"] - depth_coarse = outputs["depth_coarse"] - depth_fine = outputs["depth_fine"] - depth_fine_static = outputs["depth_fine_static"] - uncertainty = outputs["uncertainty"] - - depth_coarse = colormaps.apply_depth_colormap(depth_coarse) - depth_fine = colormaps.apply_depth_colormap(depth_fine) - depth_fine_static = colormaps.apply_depth_colormap(depth_fine_static) - uncertainty = colormaps.apply_depth_colormap(uncertainty) - - row0 = torch.cat([image, uncertainty, torch.ones_like(rgb_fine)], dim=-2) - row1 = torch.cat([rgb_fine, rgb_fine_static, rgb_coarse], dim=-2) - row2 = torch.cat([depth_fine, depth_fine_static, depth_coarse], dim=-2) - combined_image = torch.cat([row0, row1, row2], dim=-3) - - # this doesn't really make sense but do it anyway - fine_psnr = self.psnr(image, rgb_fine) - metrics_dict = { - "psnr": float(fine_psnr.item()), - } - images_dict = {"img": combined_image} - if "mask" in batch: - mask = batch["mask"].repeat(1, 1, 3) - images_dict["mask"] = mask - return metrics_dict, images_dict diff --git a/nerfstudio/models/semantic_nerf.py b/nerfstudio/models/semantic_nerf.py deleted file mode 100644 index 8fda7b358..000000000 --- a/nerfstudio/models/semantic_nerf.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2022 The Nerfstudio Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Semantic NeRF implementation. -""" -from __future__ import annotations - -from typing import Dict, Tuple - -import torch -from torch import nn - -from nerfstudio.cameras.rays import RayBundle -from nerfstudio.data.dataparsers.base_dataparser import Semantics -from nerfstudio.field_components.encodings import NeRFEncoding -from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.semantic_nerf_field import SemanticNerfField -from nerfstudio.fields.vanilla_nerf_field import NeRFField -from nerfstudio.model_components.renderers import SemanticRenderer -from nerfstudio.model_components.scene_colliders import AABBBoxCollider -from nerfstudio.models.base_model import ModelConfig -from nerfstudio.models.vanilla_nerf import NeRFModel -from nerfstudio.utils import misc - - -class SemanticNerfModel(NeRFModel): - """Semantic-NeRF model - - Args: - config: SemanticNeRF configuration to instantiate model - semantics: additional semantics data info - """ - - def __init__(self, config: ModelConfig, semantics: Semantics, **kwargs) -> None: - self.stuff_classes = semantics.stuff_classes - self.stuff_colors = semantics.stuff_colors - super().__init__(config=config, **kwargs) - - def populate_modules(self): - """Set the fields and modules""" - super().populate_modules() - - # setting up fields - position_encoding = NeRFEncoding( - in_dim=3, num_frequencies=10, min_freq_exp=0.0, max_freq_exp=8.0, include_input=True - ) - direction_encoding = NeRFEncoding( - in_dim=3, num_frequencies=4, min_freq_exp=0.0, max_freq_exp=4.0, include_input=True - ) - self.field_coarse = NeRFField(position_encoding=position_encoding, direction_encoding=direction_encoding) - - num_semantic_classes = len(self.stuff_classes) - self.field_fine = SemanticNerfField( - num_semantic_classes, position_encoding=position_encoding, direction_encoding=direction_encoding - ) - - # renderers - self.renderer_semantic = SemanticRenderer() - - # losses - self.cross_entropy_loss = nn.CrossEntropyLoss(reduction="mean") - - # colliders - if self.config.enable_collider: - self.collider = AABBBoxCollider(scene_box=self.scene_box) - - def get_outputs(self, ray_bundle: RayBundle): - # uniform sampling - ray_samples_uniform = self.sampler_uniform(ray_bundle) - - # coarse field - field_outputs_coarse = self.field_coarse.forward(ray_samples_uniform) - weights_coarse = ray_samples_uniform.get_weights(field_outputs_coarse[FieldHeadNames.DENSITY]) - rgb_coarse = self.renderer_rgb( - rgb=field_outputs_coarse[FieldHeadNames.RGB], - weights=weights_coarse, - ) - accumulation_coarse = self.renderer_accumulation(weights_coarse) - depth_coarse = self.renderer_depth(weights_coarse, ray_samples_uniform) - - # pdf sampling - ray_samples_pdf = self.sampler_pdf(ray_bundle, ray_samples_uniform, weights_coarse) - - # fine field - field_outputs_fine = self.field_fine.forward(ray_samples_pdf) - weights_fine = ray_samples_pdf.get_weights(field_outputs_fine[FieldHeadNames.DENSITY]) - rgb_fine = self.renderer_rgb( - rgb=field_outputs_fine[FieldHeadNames.RGB], - weights=weights_fine, - ) - accumulation_fine = self.renderer_accumulation(weights_fine) - depth_fine = self.renderer_depth(weights_fine, ray_samples_pdf) - semantic_fine = self.renderer_semantic(field_outputs_fine[FieldHeadNames.SEMANTICS], weights=weights_fine) - - outputs = { - "rgb_coarse": rgb_coarse, - "rgb_fine": rgb_fine, - "accumulation_coarse": accumulation_coarse, - "accumulation_fine": accumulation_fine, - "depth_coarse": depth_coarse, - "depth_fine": depth_fine, - "semantic_fine": semantic_fine, - } - return outputs - - def get_loss_dict(self, outputs, batch, metrics_dict=None): - image = batch["image"].to(outputs["rgb_coarse"].device) - rgb_loss_coarse = self.rgb_loss(image, outputs["rgb_coarse"]) - rgb_loss_fine = self.rgb_loss(image, outputs["rgb_fine"]) - semantic_logits = outputs["semantic_fine"] - semantic_classes = batch["semantics"][..., 0].long() - semantic_loss_fine = self.cross_entropy_loss(semantic_logits, semantic_classes) - loss_dict = { - "rgb_loss_coarse": rgb_loss_coarse, - "rgb_loss_fine": rgb_loss_fine, - "semantic_loss_fine": semantic_loss_fine, - } - loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) - return loss_dict - - def get_image_metrics_and_images( - self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] - ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: - metrics_dict, images_dict = super().get_image_metrics_and_images(outputs, batch) - semantic_logits = outputs["semantic_fine"] - semantic_labels = torch.argmax(torch.nn.functional.softmax(semantic_logits, dim=-1), dim=-1) # type: ignore - semantic_labels_image = self.stuff_colors[semantic_labels] - - mask = batch["mask"].repeat(1, 1, 3) - - images_dict["semantics"] = semantic_labels_image - images_dict["mask"] = mask - return metrics_dict, images_dict diff --git a/nerfstudio/models/semantic_nerfw.py b/nerfstudio/models/semantic_nerfw.py new file mode 100644 index 000000000..ac11c0b4d --- /dev/null +++ b/nerfstudio/models/semantic_nerfw.py @@ -0,0 +1,311 @@ +# Copyright 2022 The Plenoptix Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Semantic NeRF-W implementation which should be fast enough to view in the viewer. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Tuple, Type + +import numpy as np +import torch +from torch.nn import Parameter +from torchmetrics import PeakSignalNoiseRatio +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.data.dataparsers.base_dataparser import Semantics +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.fields.nerfacto_field import TCNNNerfactoField +from nerfstudio.model_components.losses import MSELoss, distortion_loss, interlevel_loss +from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + RGBRenderer, + SemanticRenderer, + UncertaintyRenderer, +) +from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.models.base_model import Model +from nerfstudio.models.nerfacto import NerfactoModelConfig +from nerfstudio.utils import colormaps, colors + + +@dataclass +class SemanticNerfWModelConfig(NerfactoModelConfig): + """Nerfacto Model Config""" + + _target: Type = field(default_factory=lambda: SemanticNerfWModel) + use_transient_embedding: bool = False + """Whether to use transient embedding.""" + + +class SemanticNerfWModel(Model): + """Nerfacto model + + Args: + config: Nerfacto configuration to instantiate model + """ + + config: SemanticNerfWModelConfig + + def __init__(self, config: SemanticNerfWModelConfig, semantics: Semantics, **kwargs) -> None: + self.semantics = semantics + super().__init__(config=config, **kwargs) + + def populate_modules(self): + """Set the fields and modules.""" + super().populate_modules() + + scene_contraction = SceneContraction(order=float("inf")) + + if self.config.use_transient_embedding: + raise ValueError("Transient embedding is not fully working for semantic nerf-w.") + + # Fields + self.field = TCNNNerfactoField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + num_images=self.num_train_data, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, + use_transient_embedding=self.config.use_transient_embedding, + use_semantics=True, + num_semantics_stuff_classes=len(self.semantics.stuff_classes), + num_semantics_thing_classes=len(self.semantics.thing_classes), + ) + + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + if self.config.use_same_proposal_network: + network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction) + self.proposal_networks.append(network) + self.density_fns = [network.density_fn for _ in range(self.config.num_proposal_network_iterations)] + else: + for _ in range(self.config.num_proposal_network_iterations): + network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction) + self.proposal_networks.append(network) + self.density_fns = [network.density_fn for network in self.proposal_networks] + + # Collider + self.collider = NearFarCollider(near_plane=self.config.near_plane, far_plane=self.config.far_plane) + + # Samplers + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_network_iterations, + single_jitter=self.config.use_single_jitter, + ) + + # renderers + background_color = None if self.config.randomize_background else colors.WHITE + self.renderer_rgb = RGBRenderer(background_color=background_color) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_uncertainty = UncertaintyRenderer() + self.renderer_semantics = SemanticRenderer() + + # losses + self.rgb_loss = MSELoss() + self.cross_entropy_loss = torch.nn.CrossEntropyLoss(reduction="mean") + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity() + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + param_groups = {} + param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) + param_groups["fields"] = list(self.field.parameters()) + return param_groups + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + callbacks = [] + if self.config.use_proposal_weight_anneal: + # anneal the weights of the proposal network before doing PDF sampling + N = self.config.proposal_weights_anneal_max_num_iters + + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / N, 0, 1) + bias = lambda x, b: (b * x) / ((b - 1) * x + 1) + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ) + ) + return callbacks + + def get_outputs(self, ray_bundle: RayBundle): + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + field_outputs = self.field(ray_samples) + + if self.training and self.config.use_transient_embedding: + density = field_outputs[FieldHeadNames.DENSITY] + field_outputs[FieldHeadNames.TRANSIENT_DENSITY] + weights = ray_samples.get_weights(density) + weights_static = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) + rgb_static_component = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + rgb_transient_component = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.TRANSIENT_RGB], weights=weights + ) + rgb = rgb_static_component + rgb_transient_component + else: + weights_static = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) + weights = weights_static + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + weights_list.append(weights_static) + ray_samples_list.append(ray_samples) + + depth = self.renderer_depth(weights=weights_static, ray_samples=ray_samples) + accumulation = self.renderer_accumulation(weights=weights_static) + + outputs = {"rgb": rgb, "accumulation": accumulation, "depth": depth} + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + for i in range(self.config.num_proposal_network_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + + # transients + if self.training and self.config.use_transient_embedding: + weights_transient = ray_samples.get_weights(field_outputs[FieldHeadNames.TRANSIENT_DENSITY]) + uncertainty = self.renderer_uncertainty(field_outputs[FieldHeadNames.UNCERTAINTY], weights_transient) + outputs["uncertainty"] = uncertainty + 0.03 # NOTE(ethan): this is the uncertainty min + outputs["density_transient"] = field_outputs[FieldHeadNames.TRANSIENT_DENSITY] + + # semantics + outputs["semantics_thing"] = self.renderer_semantics( + field_outputs[FieldHeadNames.SEMANTICS_THING], weights=weights_static.detach() + ) + outputs["semantics_stuff"] = self.renderer_semantics( + field_outputs[FieldHeadNames.SEMANTICS_STUFF], weights=weights_static.detach() + ) + + # semantics colormaps + semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics_stuff"], dim=-1), dim=-1) + outputs["semantics_colormap_stuff"] = self.semantics.stuff_colors[semantic_labels] + semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics_thing"], dim=-1), dim=-1) + outputs["semantics_colormap_thing"] = self.semantics.thing_colors[semantic_labels] + # TODO: adding these together is incorrect and produces weird results + outputs["semantics_colormap"] = outputs["semantics_colormap_stuff"] + outputs["semantics_colormap_thing"] + + return outputs + + def get_metrics_dict(self, outputs, batch): + metrics_dict = {} + image = batch["image"].to(self.device) + metrics_dict["psnr"] = self.psnr(outputs["rgb"], image) + metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"]) + return metrics_dict + + def get_loss_dict(self, outputs, batch, metrics_dict=None): + loss_dict = {} + image = batch["image"].to(self.device) + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + assert metrics_dict is not None and "distortion" in metrics_dict + loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] + + # transient loss + if self.training and self.config.use_transient_embedding: + betas = outputs["uncertainty"] + loss_dict["uncertainty_loss"] = 3 + torch.log(betas).mean() + loss_dict["density_loss"] = 0.01 * outputs["density_transient"].mean() + loss_dict["rgb_loss"] = (((image - outputs["rgb"]) ** 2).sum(-1) / (betas[..., 0] ** 2)).mean() + else: + loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"]) + + # semantic loss + loss_dict["semantics_stuff_loss"] = self.cross_entropy_loss( + outputs["semantics_stuff"], batch["semantics_stuff"][..., 0].long() + ) + loss_dict["semantics_thing_loss"] = self.cross_entropy_loss( + outputs["semantics_thing"], batch["semantics_thing"][..., 0].long() + ) + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + + image = batch["image"].to(self.device) + rgb = outputs["rgb"] + rgb = torch.clamp(rgb, min=0, max=1) + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + + combined_rgb = torch.cat([image, rgb], dim=1) + combined_acc = torch.cat([acc], dim=1) + combined_depth = torch.cat([depth], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + image = torch.moveaxis(image, -1, 0)[None, ...] + rgb = torch.moveaxis(rgb, -1, 0)[None, ...] + + psnr = self.psnr(image, rgb) + ssim = self.ssim(image, rgb) + lpips = self.lpips(image, rgb) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + + for i in range(self.config.num_proposal_network_iterations): + key = f"prop_depth_{i}" + prop_depth_i = colormaps.apply_depth_colormap( + outputs[key], + accumulation=outputs["accumulation"], + ) + images_dict[key] = prop_depth_i + + # semantics + semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics_stuff"], dim=-1), dim=-1) + images_dict["semantics_colormap_stuff"] = self.semantics.stuff_colors[semantic_labels] + semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics_thing"], dim=-1), dim=-1) + images_dict["semantics_colormap_thing"] = self.semantics.thing_colors[semantic_labels] + # TODO: adding these together is incorrect and produces weird results + images_dict["semantics_colormap"] = ( + images_dict["semantics_colormap_stuff"] + images_dict["semantics_colormap_thing"] + ) + + # valid mask + images_dict["mask"] = batch["mask"].repeat(1, 1, 3) + + return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index e92252cc6..29f8f2915 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -217,9 +217,12 @@ class VanillaPipeline(Pipeline): self.datamanager.to(device) # TODO(ethan): get rid of scene_bounds from the model assert self.datamanager.train_dataset is not None, "Missing input dataset" + self._model = config.model.setup( scene_box=self.datamanager.train_dataset.dataparser_outputs.scene_box, num_train_data=len(self.datamanager.train_dataset), + semantics=self.datamanager.train_dataset.dataparser_outputs.semantics, + device=device, ) self.model.to(device) diff --git a/tests/test_run_train.py b/tests/test_train.py similarity index 96% rename from tests/test_run_train.py rename to tests/test_train.py index 9b5e19dd6..0f8aabaa5 100644 --- a/tests/test_run_train.py +++ b/tests/test_train.py @@ -14,7 +14,7 @@ from nerfstudio.configs.model_configs import model_configs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.engine.trainer import train_loop -BLACKLIST = ["base", "semantic-nerf", "mipnerf-360", "instant-ngp", "nerfacto"] +BLACKLIST = ["base", "semantic-nerfw", "mipnerf-360", "instant-ngp", "nerfacto"] def set_reduced_config(config: Config):