diff --git a/.gitignore b/.gitignore index 6b2a94eba..85c2ca4e5 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,4 @@ bash/ cache/ renders/ package-lock.json +camera_paths/ \ No newline at end of file diff --git a/docs/guides/models/nerf.md b/docs/guides/models/nerf.md index 1744763fd..59bf4d2ee 100644 --- a/docs/guides/models/nerf.md +++ b/docs/guides/models/nerf.md @@ -73,7 +73,7 @@ NeRFs are a volumetric representation encoded into a neural network. They are no The associated NeRF fields can be instantiated with the following nerfstudio code (encoding described in next section): ```python -from nerfstudio.fields.nerf_field import NeRFField +from nerfstudio.fields.vanilla_nerf_field import NeRFField field_coarse = NeRFField(position_encoding=pos_enc, direction_encoding=dir_enc) field_fine = NeRFField(position_encoding=pos_enc, direction_encoding=dir_enc) @@ -84,7 +84,7 @@ field_fine = NeRFField(position_encoding=pos_enc, direction_encoding=dir_enc) An extra trick is necessary to make the neural network expressive enough to represent fine details in the scene. The input coordinates $(x,y,z,\theta,\phi)$ need to be encoded to a higher dimensional space prior to being input into the network. You can learn more about encodings [here](../model_components/visualize_encoders.ipynb). ```python -from nerfstudio.field_components.encoding import NeRFEncoding +from nerfstudio.field_components.encodings import NeRFEncoding pos_enc = NeRFEncoding( in_dim=3, num_frequencies=10, min_freq_exp=0.0, max_freq_exp=8.0, include_input=True @@ -142,7 +142,7 @@ How we sample points along rays in space is an important design decision. Variou Associated code: ```python -from nerfstudio.model_components.ray_sampler import PDFSampler, UniformSampler +from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler sampler_uniform = UniformSampler(num_samples=num_coarse_samples) ray_samples_uniform = sampler_uniform(ray_bundle) diff --git a/docs/reference/api/field_components/encodings.rst b/docs/reference/api/field_components/encodings.rst index a498fe66d..2fef31add 100644 --- a/docs/reference/api/field_components/encodings.rst +++ b/docs/reference/api/field_components/encodings.rst @@ -3,6 +3,6 @@ Encodings =================== -.. automodule:: nerfstudio.field_components.encoding +.. automodule:: nerfstudio.field_components.encodings :members: :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/field_components/field_losses.rst b/docs/reference/api/field_components/field_losses.rst deleted file mode 100644 index 07984cfdf..000000000 --- a/docs/reference/api/field_components/field_losses.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _field_losses: - -Field Losses -=================== - -.. automodule:: nerfstudio.field_components.field_losses - :members: - :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/field_components/index.rst b/docs/reference/api/field_components/index.rst index 096805081..e5f207a9b 100644 --- a/docs/reference/api/field_components/index.rst +++ b/docs/reference/api/field_components/index.rst @@ -10,6 +10,5 @@ TODO: High level description of field modules and how they connect together. encodings embeddings field_heads - field_losses mlp spatial_distortions \ No newline at end of file diff --git a/docs/reference/api/fields.rst b/docs/reference/api/fields.rst index febf65aac..4dea6f8e9 100644 --- a/docs/reference/api/fields.rst +++ b/docs/reference/api/fields.rst @@ -17,9 +17,9 @@ Instant NGP :members: :show-inheritance: -NeRF +Vanilla NeRF ---------------- -.. automodule:: nerfstudio.fields.nerf_field +.. automodule:: nerfstudio.fields.vanilla_nerf_field :members: :show-inheritance: diff --git a/docs/reference/api/model_components/index.rst b/docs/reference/api/model_components/index.rst index 67c5d56a7..fffc29d3b 100644 --- a/docs/reference/api/model_components/index.rst +++ b/docs/reference/api/model_components/index.rst @@ -1,12 +1,10 @@ .. _graph_modules: -Graph Modules +Model Components =================== -TODO: High level description of graph modules and how they are used. - .. toctree:: ray_sampler - ray_losses + losses renderers \ No newline at end of file diff --git a/docs/reference/api/model_components/losses.rst b/docs/reference/api/model_components/losses.rst new file mode 100644 index 000000000..8a6a15a6f --- /dev/null +++ b/docs/reference/api/model_components/losses.rst @@ -0,0 +1,8 @@ +.. _losses: + +Losses +=================== + +.. automodule:: nerfstudio.model_components.losses + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/model_components/ray_losses.rst b/docs/reference/api/model_components/ray_losses.rst deleted file mode 100644 index 0c174b9c3..000000000 --- a/docs/reference/api/model_components/ray_losses.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _ray_losses: - -Ray Losses -=================== - -.. automodule:: nerfstudio.model_components.ray_losses - :members: - :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/model_components/ray_sampler.rst b/docs/reference/api/model_components/ray_sampler.rst index f289ec1ee..821e85c5f 100644 --- a/docs/reference/api/model_components/ray_sampler.rst +++ b/docs/reference/api/model_components/ray_sampler.rst @@ -3,6 +3,6 @@ Ray Sampler =================== -.. automodule:: nerfstudio.model_components.ray_sampler +.. automodule:: nerfstudio.model_components.ray_samplers :members: :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/models.rst b/docs/reference/api/models.rst index 205185323..6428b8064 100644 --- a/docs/reference/api/models.rst +++ b/docs/reference/api/models.rst @@ -1,12 +1,12 @@ .. _graphs: -Graphs +Models ============ Base ---------------- -.. automodule:: nerfstudio.models.base +.. automodule:: nerfstudio.models.base_model :members: :show-inheritance: diff --git a/docs/reference/api/utils/colormaps.rst b/docs/reference/api/utils/colormaps.rst new file mode 100644 index 000000000..cf25c2aaf --- /dev/null +++ b/docs/reference/api/utils/colormaps.rst @@ -0,0 +1,8 @@ +.. _colormaps: + +Colormaps +---------------- + +.. automodule:: nerfstudio.utils.colormaps + :members: + :show-inheritance: diff --git a/docs/reference/api/utils/index.rst b/docs/reference/api/utils/index.rst index af803d7be..08c83d150 100644 --- a/docs/reference/api/utils/index.rst +++ b/docs/reference/api/utils/index.rst @@ -7,5 +7,5 @@ Utils colors math - visualization + colormaps tensor_dataclass \ No newline at end of file diff --git a/docs/reference/api/utils/visualization.rst b/docs/reference/api/utils/visualization.rst deleted file mode 100644 index e046e2772..000000000 --- a/docs/reference/api/utils/visualization.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _visualization: - -Visualization ----------------- - -.. automodule:: nerfstudio.utils.visualization - :members: - :show-inheritance: diff --git a/docs/tutorials/pipelines/creating_pipelines.ipynb b/docs/tutorials/pipelines/creating_pipelines.ipynb index f6104aa6c..368e8785e 100644 --- a/docs/tutorials/pipelines/creating_pipelines.ipynb +++ b/docs/tutorials/pipelines/creating_pipelines.ipynb @@ -120,7 +120,7 @@ "from nerfstudio.data.base import DataManager\n", "from nerfstudio.data.structs import DatasetInputs\n", "from nerfstudio.models.base import Model\n", - "from nerfstudio.pipelines.base import Pipeline\n", + "from nerfstudio.pipelines.base_pipeline import Pipeline\n", "from nerfstudio.models.instant_ngp import NGPModel\n", "from nerfstudio.cameras.rays import RayBundle" ] @@ -292,7 +292,7 @@ "pprint.pprint(pipeline_config)\n", "print(\"----------------------------------------------------\")\n", "\n", - "# from nerfstudio.pipelines.base import setup_pipeline\n", + "# from nerfstudio.pipelines.base_pipeline import setup_pipeline\n", "# pipeline = setup_pipeline(pipeline_config, device=\"cuda\")\n", "pipeline = pipeline_config.setup()" ] diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 87ae5aabb..1787b2b10 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -176,7 +176,7 @@ class ViewerConfig(PrintableConfig): from nerfstudio.engine.optimizers import OptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig -from nerfstudio.pipelines.base import VanillaPipelineConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig @dataclass diff --git a/nerfstudio/configs/model_configs.py b/nerfstudio/configs/model_configs.py index b58761723..f613709a1 100644 --- a/nerfstudio/configs/model_configs.py +++ b/nerfstudio/configs/model_configs.py @@ -29,7 +29,7 @@ from nerfstudio.data.dataparsers.friends_dataparser import FriendsDataParserConf from nerfstudio.data.dataparsers.mipnerf_dataparser import MipNerf360DataParserConfig from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig -from nerfstudio.models.base import VanillaModelConfig +from nerfstudio.models.base_model import VanillaModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.mipnerf_360 import MipNerf360Model @@ -38,7 +38,7 @@ from nerfstudio.models.nerfw import NerfWModelConfig from nerfstudio.models.semantic_nerf import SemanticNerfModel from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel -from nerfstudio.pipelines.base import VanillaPipelineConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig model_configs: Dict[str, Config] = {} diff --git a/nerfstudio/data/datamanagers.py b/nerfstudio/data/datamanagers.py index 3ecd96b04..6b4fdb453 100644 --- a/nerfstudio/data/datamanagers.py +++ b/nerfstudio/data/datamanagers.py @@ -45,8 +45,8 @@ from nerfstudio.data.utils.dataloaders import ( RandIndicesEvalDataloader, ) from nerfstudio.data.utils.datasets import InputDataset -from nerfstudio.model_components.ray_generator import RayGenerator -from nerfstudio.utils.callbacks import TrainingCallback, TrainingCallbackAttributes +from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes +from nerfstudio.model_components.ray_generators import RayGenerator from nerfstudio.utils.misc import IterableWrapper AnnotatedDataParserUnion = dcargs.extras.subcommand_type_from_defaults( diff --git a/nerfstudio/data/utils/datasets.py b/nerfstudio/data/utils/datasets.py index d19366346..6e471272e 100644 --- a/nerfstudio/data/utils/datasets.py +++ b/nerfstudio/data/utils/datasets.py @@ -27,7 +27,6 @@ from torch.utils.data import Dataset from torchtyping import TensorType from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs -from nerfstudio.utils.misc import is_not_none class InputDataset(Dataset): @@ -80,7 +79,6 @@ class InputDataset(Dataset): """ image = self.get_image(image_idx) data = {"image_idx": image_idx} - assert is_not_none(image) data["image"] = image for _, data_func_dict in self.dataset_inputs.additional_inputs.items(): assert "func" in data_func_dict, "Missing function to process data: specify `func` in `additional_inputs`" diff --git a/nerfstudio/utils/callbacks.py b/nerfstudio/engine/callbacks.py similarity index 100% rename from nerfstudio/utils/callbacks.py rename to nerfstudio/engine/callbacks.py diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 0c1992fdd..9bb986e8b 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -29,14 +29,14 @@ from rich import console from torch.cuda.amp.grad_scaler import GradScaler from nerfstudio.configs import base_config as cfg -from nerfstudio.engine.optimizers import Optimizers, setup_optimizers -from nerfstudio.pipelines.base import VanillaPipeline -from nerfstudio.utils import profiler, writer -from nerfstudio.utils.callbacks import ( +from nerfstudio.engine.callbacks import ( TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation, ) +from nerfstudio.engine.optimizers import Optimizers, setup_optimizers +from nerfstudio.pipelines.base_pipeline import VanillaPipeline +from nerfstudio.utils import profiler, writer from nerfstudio.utils.decorators import ( check_eval_enabled, check_main_thread, diff --git a/nerfstudio/field_components/__init__.py b/nerfstudio/field_components/__init__.py index 9189165d1..3489de62f 100644 --- a/nerfstudio/field_components/__init__.py +++ b/nerfstudio/field_components/__init__.py @@ -13,6 +13,6 @@ # limitations under the License. """init field modules""" -from .base import FieldModule -from .encoding import Encoding, ScalingAndOffset +from .base_field_component import FieldComponent +from .encodings import Encoding, ScalingAndOffset from .mlp import MLP diff --git a/nerfstudio/field_components/base.py b/nerfstudio/field_components/base_field_component.py similarity index 98% rename from nerfstudio/field_components/base.py rename to nerfstudio/field_components/base_field_component.py index d9fc655bd..40af9d8c5 100644 --- a/nerfstudio/field_components/base.py +++ b/nerfstudio/field_components/base_field_component.py @@ -22,7 +22,7 @@ from torch import nn from torchtyping import TensorType -class FieldModule(nn.Module): +class FieldComponent(nn.Module): """Field modules that can be combined to store and compute the fields. Args: diff --git a/nerfstudio/field_components/embedding.py b/nerfstudio/field_components/embedding.py index 8280a4ddf..6c2227ba3 100644 --- a/nerfstudio/field_components/embedding.py +++ b/nerfstudio/field_components/embedding.py @@ -20,10 +20,10 @@ Code for embeddings. import torch from torchtyping import TensorType -from nerfstudio.field_components.base import FieldModule +from nerfstudio.field_components.base_field_component import FieldComponent -class Embedding(FieldModule): +class Embedding(FieldComponent): """Index into embeddings. # TODO: add different types of initializations diff --git a/nerfstudio/field_components/encoding.py b/nerfstudio/field_components/encodings.py similarity index 99% rename from nerfstudio/field_components/encoding.py rename to nerfstudio/field_components/encodings.py index 3e33cae30..e37a25db5 100644 --- a/nerfstudio/field_components/encoding.py +++ b/nerfstudio/field_components/encodings.py @@ -25,9 +25,9 @@ import torch.nn.functional as F from torch import nn from torchtyping import TensorType -from nerfstudio.field_components.base import FieldModule -from nerfstudio.utils.comments import print_tcnn_speed_warning +from nerfstudio.field_components.base_field_component import FieldComponent from nerfstudio.utils.math import components_from_spherical_harmonics, expected_sin +from nerfstudio.utils.printing import print_tcnn_speed_warning try: import tinycudann as tcnn @@ -37,7 +37,7 @@ except ImportError: TCNN_EXISTS = False -class Encoding(FieldModule): +class Encoding(FieldComponent): """Encode an input tensor. Intended to be subclassed Args: diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index 51aa3d255..b8e356f99 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -21,7 +21,7 @@ from typing import Callable, Optional, Union from torch import nn from torchtyping import TensorType -from nerfstudio.field_components.base import FieldModule +from nerfstudio.field_components.base_field_component import FieldComponent class FieldHeadNames(Enum): @@ -36,7 +36,7 @@ class FieldHeadNames(Enum): SEMANTICS = "semantics" -class FieldHead(FieldModule): +class FieldHead(FieldComponent): """Base field output Args: diff --git a/nerfstudio/field_components/field_losses.py b/nerfstudio/field_components/field_losses.py deleted file mode 100644 index 02655775e..000000000 --- a/nerfstudio/field_components/field_losses.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/nerfstudio/field_components/mlp.py b/nerfstudio/field_components/mlp.py index 1d9ad9159..c5d89e16c 100644 --- a/nerfstudio/field_components/mlp.py +++ b/nerfstudio/field_components/mlp.py @@ -21,10 +21,10 @@ import torch from torch import nn from torchtyping import TensorType -from nerfstudio.field_components.base import FieldModule +from nerfstudio.field_components.base_field_component import FieldComponent -class MLP(FieldModule): +class MLP(FieldComponent): """Multilayer perceptron Args: diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index b3c0f6cc4..8536d398d 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -29,7 +29,7 @@ from nerfstudio.cameras.rays import RaySamples from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp from nerfstudio.field_components.embedding import Embedding -from nerfstudio.field_components.encoding import Encoding, HashEncoding, SHEncoding +from nerfstudio.field_components.encodings import Encoding, HashEncoding, SHEncoding from nerfstudio.field_components.field_heads import ( DensityFieldHead, FieldHead, diff --git a/nerfstudio/fields/nerfw_field.py b/nerfstudio/fields/nerfw_field.py index 99df63a02..1b189bed6 100644 --- a/nerfstudio/fields/nerfw_field.py +++ b/nerfstudio/fields/nerfw_field.py @@ -22,7 +22,7 @@ from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples from nerfstudio.field_components.embedding import Embedding -from nerfstudio.field_components.encoding import Encoding, Identity +from nerfstudio.field_components.encodings import Encoding, Identity from nerfstudio.field_components.field_heads import ( DensityFieldHead, FieldHeadNames, diff --git a/nerfstudio/fields/semantic_nerf_field.py b/nerfstudio/fields/semantic_nerf_field.py index b37b4f182..f897e29a3 100644 --- a/nerfstudio/fields/semantic_nerf_field.py +++ b/nerfstudio/fields/semantic_nerf_field.py @@ -22,7 +22,7 @@ from torch import nn from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples -from nerfstudio.field_components.encoding import Encoding, Identity +from nerfstudio.field_components.encodings import Encoding, Identity from nerfstudio.field_components.field_heads import ( DensityFieldHead, FieldHeadNames, diff --git a/nerfstudio/fields/nerf_field.py b/nerfstudio/fields/vanilla_nerf_field.py similarity index 98% rename from nerfstudio/fields/nerf_field.py rename to nerfstudio/fields/vanilla_nerf_field.py index 62dbf9492..228589857 100644 --- a/nerfstudio/fields/nerf_field.py +++ b/nerfstudio/fields/vanilla_nerf_field.py @@ -22,7 +22,7 @@ from torch import nn from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples -from nerfstudio.field_components.encoding import Encoding, Identity +from nerfstudio.field_components.encodings import Encoding, Identity from nerfstudio.field_components.field_heads import ( DensityFieldHead, FieldHead, diff --git a/nerfstudio/model_components/loss.py b/nerfstudio/model_components/losses.py similarity index 73% rename from nerfstudio/model_components/loss.py rename to nerfstudio/model_components/losses.py index a25c53abb..e5b28c38e 100644 --- a/nerfstudio/model_components/loss.py +++ b/nerfstudio/model_components/losses.py @@ -20,6 +20,8 @@ import torch from torch import nn from torchtyping import TensorType +from nerfstudio.cameras.rays import RaySamples + L1Loss = nn.L1Loss MSELoss = nn.MSELoss @@ -137,3 +139,45 @@ def distortion_loss(weights_list, ray_samples_list): w = weights_list[-1][..., 0] loss = torch.mean(lossfun_distortion(c, w)) return loss + + +def nerfstudio_distortion_loss( + ray_samples: RaySamples, + densities: TensorType["bs":..., "num_samples", 1] = None, + weights: TensorType["bs":..., "num_samples", 1] = None, +) -> TensorType["bs":..., 1]: + """Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss. + + .. math:: + + \\mathcal{L}(\\mathbf{s}, \\mathbf{w}) =\\iint\\limits_{-\\infty}^{\\,\\,\\,\\infty} + \\mathbf{w}_\\mathbf{s}(u)\\mathbf{w}_\\mathbf{s}(v)|u - v|\\,d_{u}\\,d_{v} + + where :math:`\\mathbf{w}_\\mathbf{s}(u)=\\sum_i w_i \\mathbb{1}_{[\\mathbf{s}_i, \\mathbf{s}_{i+1})}(u)` + is the weight at location :math:`u` between bin locations :math:`s_i` and :math:`s_{i+1}`. + + Args: + ray_samples: Ray samples to compute loss over + densities: Predicted sample densities + weights: Predicted weights from densities and sample locations + """ + if torch.is_tensor(densities): + assert not torch.is_tensor(weights), "Cannot use both densities and weights" + # Compute the weight at each sample location + weights = ray_samples.get_weights(densities) + if torch.is_tensor(weights): + assert not torch.is_tensor(densities), "Cannot use both densities and weights" + + starts = ray_samples.spacing_starts + ends = ray_samples.spacing_ends + + assert starts is not None and ends is not None, "Ray samples must have spacing starts and ends" + midpoints = (starts + ends) / 2.0 # (..., num_samples, 1) + + loss = ( + weights * weights[..., None, :, 0] * torch.abs(midpoints - midpoints[..., None, :, 0]) + ) # (..., num_samples, num_samples) + loss = torch.sum(loss, dim=(-1, -2))[..., None] # (..., num_samples) + loss = loss + 1 / 3.0 * torch.sum(weights**2 * (ends - starts), dim=-2) + + return loss diff --git a/nerfstudio/model_components/ray_generator.py b/nerfstudio/model_components/ray_generators.py similarity index 100% rename from nerfstudio/model_components/ray_generator.py rename to nerfstudio/model_components/ray_generators.py diff --git a/nerfstudio/model_components/ray_losses.py b/nerfstudio/model_components/ray_losses.py deleted file mode 100644 index 0a7068185..000000000 --- a/nerfstudio/model_components/ray_losses.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Collection of Ray Losses""" - -import torch -from torchtyping import TensorType - -from nerfstudio.cameras.rays import RaySamples - - -def distortion_loss( - ray_samples: RaySamples, - densities: TensorType["bs":..., "num_samples", 1] = None, - weights: TensorType["bs":..., "num_samples", 1] = None, -) -> TensorType["bs":..., 1]: - """Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss. - - .. math:: - - \\mathcal{L}(\\mathbf{s}, \\mathbf{w}) =\\iint\\limits_{-\\infty}^{\\,\\,\\,\\infty} - \\mathbf{w}_\\mathbf{s}(u)\\mathbf{w}_\\mathbf{s}(v)|u - v|\\,d_{u}\\,d_{v} - - where :math:`\\mathbf{w}_\\mathbf{s}(u)=\\sum_i w_i \\mathbb{1}_{[\\mathbf{s}_i, \\mathbf{s}_{i+1})}(u)` - is the weight at location :math:`u` between bin locations :math:`s_i` and :math:`s_{i+1}`. - - Args: - ray_samples: Ray samples to compute loss over - densities: Predicted sample densities - weights: Predicted weights from densities and sample locations - """ - if torch.is_tensor(densities): - assert not torch.is_tensor(weights), "Cannot use both densities and weights" - # Compute the weight at each sample location - weights = ray_samples.get_weights(densities) - if torch.is_tensor(weights): - assert not torch.is_tensor(densities), "Cannot use both densities and weights" - - starts = ray_samples.spacing_starts - ends = ray_samples.spacing_ends - - assert starts is not None and ends is not None, "Ray samples must have spacing starts and ends" - midpoints = (starts + ends) / 2.0 # (..., num_samples, 1) - - loss = ( - weights * weights[..., None, :, 0] * torch.abs(midpoints - midpoints[..., None, :, 0]) - ) # (..., num_samples, num_samples) - loss = torch.sum(loss, dim=(-1, -2))[..., None] # (..., num_samples) - loss = loss + 1 / 3.0 * torch.sum(weights**2 * (ends - starts), dim=-2) - - return loss diff --git a/nerfstudio/model_components/ray_sampler.py b/nerfstudio/model_components/ray_samplers.py similarity index 100% rename from nerfstudio/model_components/ray_sampler.py rename to nerfstudio/model_components/ray_samplers.py diff --git a/nerfstudio/models/base.py b/nerfstudio/models/base_model.py similarity index 99% rename from nerfstudio/models/base.py rename to nerfstudio/models/base_model.py index b8341206b..5652edfa0 100644 --- a/nerfstudio/models/base.py +++ b/nerfstudio/models/base_model.py @@ -31,8 +31,8 @@ from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.data.scene_box import SceneBox +from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.model_components.scene_colliders import NearFarCollider -from nerfstudio.utils.callbacks import TrainingCallback, TrainingCallbackAttributes # Model related configs diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index 9e31ffaad..6889f1a23 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -30,22 +30,22 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.fields.instant_ngp_field import TCNNInstantNGPField -from nerfstudio.model_components.loss import MSELoss -from nerfstudio.model_components.ray_sampler import VolumetricSampler +from nerfstudio.model_components.losses import MSELoss +from nerfstudio.model_components.ray_samplers import VolumetricSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base import Model, ModelConfig -from nerfstudio.utils import colors, visualization -from nerfstudio.utils.callbacks import ( - TrainingCallback, - TrainingCallbackAttributes, - TrainingCallbackLocation, -) +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors @dataclass @@ -223,12 +223,12 @@ class NGPModel(Model): image = batch["image"].to(self.device) rgb = outputs["rgb"] - acc = visualization.apply_colormap(outputs["accumulation"]) - depth = visualization.apply_depth_colormap( + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], ) - alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"]) + alive_ray_mask = colormaps.apply_colormap(outputs["alive_ray_mask"]) combined_rgb = torch.cat([image, rgb], dim=1) combined_acc = torch.cat([acc], dim=1) diff --git a/nerfstudio/models/mipnerf.py b/nerfstudio/models/mipnerf.py index 9180af324..80659cc73 100644 --- a/nerfstudio/models/mipnerf.py +++ b/nerfstudio/models/mipnerf.py @@ -26,18 +26,18 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle -from nerfstudio.field_components.encoding import NeRFEncoding +from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.nerf_field import NeRFField -from nerfstudio.model_components.loss import MSELoss -from nerfstudio.model_components.ray_sampler import PDFSampler, UniformSampler +from nerfstudio.fields.vanilla_nerf_field import NeRFField +from nerfstudio.model_components.losses import MSELoss +from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base import Model, ModelConfig -from nerfstudio.utils import colors, misc, visualization +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors, misc class MipNerfModel(Model): @@ -150,15 +150,15 @@ class MipNerfModel(Model): image = batch["image"].to(outputs["rgb_coarse"].device) rgb_coarse = outputs["rgb_coarse"] rgb_fine = outputs["rgb_fine"] - acc_coarse = visualization.apply_colormap(outputs["accumulation_coarse"]) - acc_fine = visualization.apply_colormap(outputs["accumulation_fine"]) - depth_coarse = visualization.apply_depth_colormap( + acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"]) + acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"]) + depth_coarse = colormaps.apply_depth_colormap( outputs["depth_coarse"], accumulation=outputs["accumulation_coarse"], near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"], ) - depth_fine = visualization.apply_depth_colormap( + depth_fine = colormaps.apply_depth_colormap( outputs["depth_fine"], accumulation=outputs["accumulation_fine"], near_plane=self.config.collider_params["near_plane"], diff --git a/nerfstudio/models/mipnerf_360.py b/nerfstudio/models/mipnerf_360.py index 8455ad1ee..6b99ed1eb 100644 --- a/nerfstudio/models/mipnerf_360.py +++ b/nerfstudio/models/mipnerf_360.py @@ -27,20 +27,19 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle -from nerfstudio.field_components.encoding import NeRFEncoding +from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction -from nerfstudio.fields.nerf_field import NeRFField -from nerfstudio.model_components.loss import MSELoss -from nerfstudio.model_components.ray_losses import distortion_loss -from nerfstudio.model_components.ray_sampler import PDFSampler, UniformSampler +from nerfstudio.fields.vanilla_nerf_field import NeRFField +from nerfstudio.model_components.losses import MSELoss, distortion_loss +from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base import Model, ModelConfig -from nerfstudio.utils import colors, misc, visualization +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors, misc class MipNerf360Model(Model): @@ -165,15 +164,15 @@ class MipNerf360Model(Model): image = batch["image"].to(outputs["rgb_coarse"].device) rgb_coarse = outputs["rgb_coarse"] rgb_fine = outputs["rgb_fine"] - acc_coarse = visualization.apply_colormap(outputs["accumulation_coarse"]) - acc_fine = visualization.apply_colormap(outputs["accumulation_fine"]) - depth_coarse = visualization.apply_depth_colormap( + acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"]) + acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"]) + depth_coarse = colormaps.apply_depth_colormap( outputs["depth_coarse"], accumulation=outputs["accumulation_coarse"], near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"], ) - depth_fine = visualization.apply_depth_colormap( + depth_fine = colormaps.apply_depth_colormap( outputs["depth_fine"], accumulation=outputs["accumulation_fine"], near_plane=self.config.collider_params["near_plane"], diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 43bde0238..b9cce1fba 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -29,25 +29,25 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField from nerfstudio.fields.nerfacto_field import TCNNNerfactoField -from nerfstudio.model_components.loss import MSELoss, distortion_loss, interlevel_loss -from nerfstudio.model_components.ray_sampler import ProposalNetworkSampler +from nerfstudio.model_components.losses import MSELoss, distortion_loss, interlevel_loss +from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) from nerfstudio.model_components.scene_colliders import NearFarCollider -from nerfstudio.models.base import Model, ModelConfig -from nerfstudio.utils import colors, visualization -from nerfstudio.utils.callbacks import ( - TrainingCallback, - TrainingCallbackAttributes, - TrainingCallbackLocation, -) +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors @dataclass @@ -224,8 +224,8 @@ class NerfactoModel(Model): image = batch["image"].to(self.device) rgb = outputs["rgb"] - acc = visualization.apply_colormap(outputs["accumulation"]) - depth = visualization.apply_depth_colormap( + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], ) @@ -250,7 +250,7 @@ class NerfactoModel(Model): for i in range(self.config.num_proposal_network_iterations): key = f"prop_depth_{i}" - prop_depth_i = visualization.apply_depth_colormap( + prop_depth_i = colormaps.apply_depth_colormap( outputs[key], accumulation=outputs["accumulation"], ) diff --git a/nerfstudio/models/nerfw.py b/nerfstudio/models/nerfw.py index caa60c91b..196301af3 100644 --- a/nerfstudio/models/nerfw.py +++ b/nerfstudio/models/nerfw.py @@ -28,12 +28,12 @@ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.config_utils import to_immutable_dict -from nerfstudio.field_components.encoding import NeRFEncoding +from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.nerf_field import NeRFField from nerfstudio.fields.nerfw_field import VanillaNerfWField -from nerfstudio.model_components.loss import MSELoss -from nerfstudio.model_components.ray_sampler import PDFSampler, UniformSampler +from nerfstudio.fields.vanilla_nerf_field import NeRFField +from nerfstudio.model_components.losses import MSELoss +from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, @@ -41,8 +41,8 @@ from nerfstudio.model_components.renderers import ( UncertaintyRenderer, ) from nerfstudio.model_components.scene_colliders import AABBBoxCollider -from nerfstudio.models.base import Model, VanillaModelConfig -from nerfstudio.utils import colors, misc, visualization +from nerfstudio.models.base_model import Model, VanillaModelConfig +from nerfstudio.utils import colormaps, colors, misc @dataclass @@ -255,10 +255,10 @@ class NerfWModel(Model): depth_fine_static = outputs["depth_fine_static"] uncertainty = outputs["uncertainty"] - depth_coarse = visualization.apply_depth_colormap(depth_coarse) - depth_fine = visualization.apply_depth_colormap(depth_fine) - depth_fine_static = visualization.apply_depth_colormap(depth_fine_static) - uncertainty = visualization.apply_depth_colormap(uncertainty) + depth_coarse = colormaps.apply_depth_colormap(depth_coarse) + depth_fine = colormaps.apply_depth_colormap(depth_fine) + depth_fine_static = colormaps.apply_depth_colormap(depth_fine_static) + uncertainty = colormaps.apply_depth_colormap(uncertainty) row0 = torch.cat([image, uncertainty, torch.ones_like(rgb_fine)], dim=-2) row1 = torch.cat([rgb_fine, rgb_fine_static, rgb_coarse], dim=-2) diff --git a/nerfstudio/models/semantic_nerf.py b/nerfstudio/models/semantic_nerf.py index d8c145b42..ffa8c22fc 100644 --- a/nerfstudio/models/semantic_nerf.py +++ b/nerfstudio/models/semantic_nerf.py @@ -24,13 +24,13 @@ from torch import nn from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.dataparsers.base_dataparser import Semantics -from nerfstudio.field_components.encoding import NeRFEncoding +from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.nerf_field import NeRFField from nerfstudio.fields.semantic_nerf_field import SemanticNerfField +from nerfstudio.fields.vanilla_nerf_field import NeRFField from nerfstudio.model_components.renderers import SemanticRenderer from nerfstudio.model_components.scene_colliders import AABBBoxCollider -from nerfstudio.models.base import ModelConfig +from nerfstudio.models.base_model import ModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel from nerfstudio.utils import misc diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index 56a994a7f..7c0999a6a 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -30,24 +30,24 @@ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.config_utils import to_immutable_dict +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.engine.optimizers import Optimizers -from nerfstudio.field_components.encoding import TensorVMEncoding +from nerfstudio.field_components.encodings import TensorVMEncoding from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.nerf_field import NeRFField -from nerfstudio.model_components.loss import L1Loss, MSELoss -from nerfstudio.model_components.ray_sampler import PDFSampler, UniformSampler +from nerfstudio.fields.vanilla_nerf_field import NeRFField +from nerfstudio.model_components.losses import L1Loss, MSELoss +from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base import Model, VanillaModelConfig -from nerfstudio.utils import colors, misc, visualization -from nerfstudio.utils.callbacks import ( - TrainingCallback, - TrainingCallbackAttributes, - TrainingCallbackLocation, -) +from nerfstudio.models.base_model import Model, VanillaModelConfig +from nerfstudio.utils import colormaps, colors, misc @dataclass @@ -215,8 +215,8 @@ class TensoRFModel(Model): ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: image = batch["image"].to(outputs["rgb"].device) rgb = outputs["rgb"] - acc = visualization.apply_colormap(outputs["accumulation"]) - depth = visualization.apply_depth_colormap( + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], near_plane=self.config.collider_params["near_plane"], diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py index 1b02e3e2d..6e4aab15b 100644 --- a/nerfstudio/models/vanilla_nerf.py +++ b/nerfstudio/models/vanilla_nerf.py @@ -27,18 +27,18 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle -from nerfstudio.field_components.encoding import NeRFEncoding +from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames -from nerfstudio.fields.nerf_field import NeRFField -from nerfstudio.model_components.loss import MSELoss -from nerfstudio.model_components.ray_sampler import PDFSampler, UniformSampler +from nerfstudio.fields.vanilla_nerf_field import NeRFField +from nerfstudio.model_components.losses import MSELoss +from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base import Model, ModelConfig -from nerfstudio.utils import colors, misc, visualization +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors, misc class NeRFModel(Model): @@ -159,15 +159,15 @@ class NeRFModel(Model): image = batch["image"].to(outputs["rgb_coarse"].device) rgb_coarse = outputs["rgb_coarse"] rgb_fine = outputs["rgb_fine"] - acc_coarse = visualization.apply_colormap(outputs["accumulation_coarse"]) - acc_fine = visualization.apply_colormap(outputs["accumulation_fine"]) - depth_coarse = visualization.apply_depth_colormap( + acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"]) + acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"]) + depth_coarse = colormaps.apply_depth_colormap( outputs["depth_coarse"], accumulation=outputs["accumulation_coarse"], near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"], ) - depth_fine = visualization.apply_depth_colormap( + depth_fine = colormaps.apply_depth_colormap( outputs["depth_fine"], accumulation=outputs["accumulation_fine"], near_plane=self.config.collider_params["near_plane"], diff --git a/nerfstudio/pipelines/base.py b/nerfstudio/pipelines/base_pipeline.py similarity index 99% rename from nerfstudio/pipelines/base.py rename to nerfstudio/pipelines/base_pipeline.py index d962f3f08..0992a866d 100644 --- a/nerfstudio/pipelines/base.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -42,9 +42,9 @@ from nerfstudio.data.datamanagers import ( VanillaDataManager, VanillaDataManagerConfig, ) -from nerfstudio.models.base import Model, ModelConfig +from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes +from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import profiler -from nerfstudio.utils.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.utils.misc import get_masked_dict diff --git a/nerfstudio/pipelines/dynamic_batch.py b/nerfstudio/pipelines/dynamic_batch.py index c76e6b7d0..3396412dd 100644 --- a/nerfstudio/pipelines/dynamic_batch.py +++ b/nerfstudio/pipelines/dynamic_batch.py @@ -22,7 +22,7 @@ from typing import Type import torch from nerfstudio.data.datamanagers import VanillaDataManager -from nerfstudio.pipelines.base import VanillaPipeline, VanillaPipelineConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipeline, VanillaPipelineConfig @dataclass diff --git a/nerfstudio/pipelines/pixel_nerf.py b/nerfstudio/pipelines/pixel_nerf.py deleted file mode 100644 index 774e9a5ad..000000000 --- a/nerfstudio/pipelines/pixel_nerf.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Pipeline for pixel nerf. -""" diff --git a/nerfstudio/utils/visualization.py b/nerfstudio/utils/colormaps.py similarity index 100% rename from nerfstudio/utils/visualization.py rename to nerfstudio/utils/colormaps.py diff --git a/nerfstudio/utils/io.py b/nerfstudio/utils/io.py index 0cf93579d..e46abeaad 100644 --- a/nerfstudio/utils/io.py +++ b/nerfstudio/utils/io.py @@ -15,10 +15,9 @@ """ Input/output utils. """ + import json -import pickle from pathlib import Path -from typing import Any def load_from_json(filename: Path): @@ -42,26 +41,3 @@ def write_to_json(filename: Path, content: dict): assert filename.suffix == ".json" with open(filename, "w", encoding="UTF-8") as file: json.dump(content, file) - - -def load_from_pkl(filename: Path): - """Load from a pickle file. - - Args: - filename: The filename to load from. - """ - assert filename.suffix == ".pkl" - with open(filename, "rb") as f: - return pickle.load(f) - - -def write_to_pkl(filename: Path, content: Any): - """Write to a pickle file. - - Args: - filename (str): The filename to write to. - content (Any): The data to write. - """ - assert filename.suffix == ".pkl" - with open(filename, "wb") as f: - pickle.dump(content, f) diff --git a/nerfstudio/utils/misc.py b/nerfstudio/utils/misc.py index 5abd850cd..6209e4783 100644 --- a/nerfstudio/utils/misc.py +++ b/nerfstudio/utils/misc.py @@ -16,9 +16,6 @@ Miscellaneous helper code. """ -import hashlib -import json -from math import floor, log from typing import Any, Callable, Dict, List, Optional, Union import torch @@ -59,11 +56,6 @@ def get_dict_to_cpu(stuff: Any): return stuff -def is_not_none(var: Any) -> bool: - """Return True if the variable var is None.""" - return not isinstance(var, type(None)) - - def get_masked_dict(d, mask): """Return a masked dictionary. TODO(ethan): add more asserts/checks so this doesn't have unpredictable behavior. @@ -78,19 +70,6 @@ def get_masked_dict(d, mask): return masked_dict -def get_hash_str_from_dict(dictionary: Dict[str, Any]) -> str: - """MD5 hash of a dictionary. Based on - https://www.doc.ic.ac.uk/~nuric/coding/how-to-hash-a-dictionary-in-python.html - - Args: - dictionary: dict to process - """ - dhash = hashlib.md5() - encoded = json.dumps(dictionary, sort_keys=True).encode() - dhash.update(encoded) - return dhash.hexdigest() - - class IterableWrapper: # pylint: disable=too-few-public-methods """A helper that will allow an instance of a class to return multiple kinds of iterables bound to different functions of that class. @@ -133,18 +112,6 @@ class IterableWrapper: # pylint: disable=too-few-public-methods return self -def human_format(num): - """Format a number in a more human readable way - - Args: - num: number to format - """ - units = ["", "K", "M", "B", "T", "P"] - k = 1000.0 - magnitude = int(floor(log(num, k))) - return f"{(num / k**magnitude):.2f} {units[magnitude]}" - - def scale_dict(dictionary: Dict[Any, Any], coefficients: Dict[str, float]) -> Dict[Any, Any]: """Scale a dictionary in-place given a coefficients dictionary. @@ -168,7 +135,7 @@ def step_check(step, step_size, run_at_zero=False) -> bool: return (run_at_zero or step != 0) and step % step_size == 0 -def _update_avg(prev_avg: float, new_val: float, step: int) -> float: +def update_avg(prev_avg: float, new_val: float, step: int) -> float: """helper to calculate the running average Args: diff --git a/nerfstudio/utils/plotly.py b/nerfstudio/utils/plotly_utils.py similarity index 100% rename from nerfstudio/utils/plotly.py rename to nerfstudio/utils/plotly_utils.py diff --git a/nerfstudio/utils/comments.py b/nerfstudio/utils/printing.py similarity index 80% rename from nerfstudio/utils/comments.py rename to nerfstudio/utils/printing.py index 8291de25c..0b4e63848 100644 --- a/nerfstudio/utils/comments.py +++ b/nerfstudio/utils/printing.py @@ -14,6 +14,8 @@ """A collection of common strings and print statements used throughout the codebase.""" +from math import floor, log + from rich.console import Console console = Console(width=120) @@ -29,3 +31,15 @@ def print_tcnn_speed_warning(method_name: str): ) console.print("[yellow]pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch") console.line() + + +def human_format(num): + """Format a number in a more human readable way + + Args: + num: number to format + """ + units = ["", "K", "M", "B", "T", "P"] + k = 1000.0 + magnitude = int(floor(log(num, k))) + return f"{(num / k**magnitude):.2f} {units[magnitude]}" diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py index d0cf71713..d781263bf 100644 --- a/nerfstudio/utils/writer.py +++ b/nerfstudio/utils/writer.py @@ -32,7 +32,7 @@ from torchtyping import TensorType from nerfstudio.configs import base_config as cfg from nerfstudio.utils.decorators import check_main_thread, decorate_all -from nerfstudio.utils.misc import human_format +from nerfstudio.utils.printing import human_format CONSOLE = console.Console() diff --git a/nerfstudio/viewer/app/src/modules/LandingModal/LandingModal.jsx b/nerfstudio/viewer/app/src/modules/LandingModal/LandingModal.jsx index 1372f6bd7..ce4001125 100644 --- a/nerfstudio/viewer/app/src/modules/LandingModal/LandingModal.jsx +++ b/nerfstudio/viewer/app/src/modules/LandingModal/LandingModal.jsx @@ -57,7 +57,7 @@ export default function LandingModel(props: LandingModalProps) { const handleOpen = () => setOpen(true); const handleClose = () => setOpen(false); - const text_intro = `At the top right is the Status Panel where you can connect to a WebSocket and see the train/eval status of your NeRF model. The WebSocket Port takes the form :.`; + const text_intro = `To use our viewer, you need to connect to the WebSocket server where your training job is running.`; const websocket_port = useSelector( (state) => state.websocketState.websocket_port, diff --git a/nerfstudio/viewer/server/subprocess.py b/nerfstudio/viewer/server/subprocess.py index 64e0134de..6d902ba28 100644 --- a/nerfstudio/viewer/server/subprocess.py +++ b/nerfstudio/viewer/server/subprocess.py @@ -84,7 +84,7 @@ def run_viewer_bridge_server_as_subprocess( "You likely have to modify --viewer.zmq-port and/or --viewer.websocket-port in the " "config to avoid conflicting ports.\n" ) - string += "Try modifying --viewer.zmq-port 6000 --viewer.websocket-port 7007\n" + string += "Try modifying --viewer.websocket-port 7007\n" CONSOLE.print(f"[bold red]{string}") cleanup(process) # This exists the entire program. sys.exit() will only kill the thread that this runs in. diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 9c3c24ecc..6a6446f04 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -32,8 +32,8 @@ from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs import base_config as cfg from nerfstudio.data.utils.datasets import InputDataset -from nerfstudio.models.base import Model -from nerfstudio.utils import profiler, visualization, writer +from nerfstudio.models.base_model import Model +from nerfstudio.utils import colormaps, profiler, writer from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.io import load_from_json, write_to_json from nerfstudio.utils.misc import get_dict_to_torch @@ -407,15 +407,13 @@ class ViewerState: if OutputTypes.ACCUMULATION in self.output_list else OutputTypes.ACCUMULATION_FINE ) - return visualization.apply_depth_colormap( - outputs[reformatted_output], accumulation=outputs[accumulation_str] - ) + return colormaps.apply_depth_colormap(outputs[reformatted_output], accumulation=outputs[accumulation_str]) # rendering accumulation outputs if self.prev_colormap_type == ColormapTypes.TURBO or ( self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.float ): - return visualization.apply_colormap(outputs[reformatted_output]) + return colormaps.apply_colormap(outputs[reformatted_output]) # rendering semantic outputs if self.prev_colormap_type == ColormapTypes.SEMANTIC or ( @@ -430,7 +428,7 @@ class ViewerState: if self.prev_colormap_type == ColormapTypes.BOOLEAN or ( self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.bool ): - return visualization.apply_boolean_colormap(outputs[reformatted_output]) + return colormaps.apply_boolean_colormap(outputs[reformatted_output]) raise NotImplementedError diff --git a/scripts/eval.py b/scripts/eval.py index 098740556..bceffdb0b 100755 --- a/scripts/eval.py +++ b/scripts/eval.py @@ -29,7 +29,7 @@ from typing_extensions import assert_never from nerfstudio.cameras.camera_paths import get_path_from_json, get_spiral_path from nerfstudio.cameras.cameras import Cameras from nerfstudio.configs import base_config as cfg -from nerfstudio.pipelines.base import Pipeline +from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils.rich_utils import ItersPerSecColumn console = Console(width=120) diff --git a/tests/field_components/test_encodings.py b/tests/field_components/test_encodings.py index fe79f7799..271adddc9 100644 --- a/tests/field_components/test_encodings.py +++ b/tests/field_components/test_encodings.py @@ -4,7 +4,7 @@ Encoding Tests import pytest import torch -from nerfstudio.field_components import encoding +from nerfstudio.field_components import encodings def test_scaling_and_offset(): @@ -14,7 +14,7 @@ def test_scaling_and_offset(): scaling = 2.0 offset = 4.5 - encoder = encoding.ScalingAndOffset(in_dim=in_dim, scaling=scaling, offset=offset) + encoder = encodings.ScalingAndOffset(in_dim=in_dim, scaling=scaling, offset=offset) assert encoder.get_out_dim() == in_dim encoded = encoder(in_tensor) @@ -22,7 +22,7 @@ def test_scaling_and_offset(): assert in_tensor * 6.5 == pytest.approx(encoded) with pytest.raises(ValueError): - encoding.ScalingAndOffset(in_dim=-1) + encodings.ScalingAndOffset(in_dim=-1) def test_nerf_encoder(): @@ -34,7 +34,7 @@ def test_nerf_encoder(): num_frequencies = 3 min_freq_exp = 0 max_freq_exp = 3 - encoder = encoding.NeRFEncoding( + encoder = encodings.NeRFEncoding( in_dim=in_dim, num_frequencies=num_frequencies, min_freq_exp=min_freq_exp, max_freq_exp=max_freq_exp ) assert encoder.get_out_dim() == out_dim @@ -62,7 +62,7 @@ def test_rff_encoder(): num_frequencies = 12 scale = 5 - encoder = encoding.RFFEncoding(in_dim=in_dim, num_frequencies=num_frequencies, scale=scale) + encoder = encodings.RFFEncoding(in_dim=in_dim, num_frequencies=num_frequencies, scale=scale) assert encoder.get_out_dim() == out_dim in_tensor = torch.ones((2, 3, in_dim)) @@ -83,7 +83,7 @@ def test_tensor_vm_encoder(): in_dim = 3 out_dim = 3 * num_components - encoder = encoding.TensorVMEncoding(num_components=num_components, resolution=resolution) + encoder = encodings.TensorVMEncoding(num_components=num_components, resolution=resolution) assert encoder.get_out_dim() == out_dim in_tensor = torch.ones((3, in_dim)) @@ -107,7 +107,7 @@ def test_tensor_cp_encoder(): in_dim = 3 out_dim = num_components - encoder = encoding.TensorCPEncoding(num_components=num_components, resolution=resolution) + encoder = encodings.TensorCPEncoding(num_components=num_components, resolution=resolution) assert encoder.get_out_dim() == out_dim in_tensor = torch.ones((3, in_dim)) @@ -128,9 +128,9 @@ def test_tensor_sh_encoder(): out_dim = levels**2 with pytest.raises(ValueError): - encoder = encoding.SHEncoding(levels=5) + encoder = encodings.SHEncoding(levels=5) - encoder = encoding.SHEncoding(levels=levels) + encoder = encodings.SHEncoding(levels=levels) assert encoder.get_out_dim() == out_dim in_tensor = torch.zeros((10, 3)) @@ -146,7 +146,7 @@ def test_tensor_hash_encoder(): features_per_level = 4 out_dim = num_levels * features_per_level - encoder = encoding.HashEncoding( + encoder = encodings.HashEncoding( num_levels=num_levels, features_per_level=features_per_level, log2_hashmap_size=5, @@ -159,7 +159,7 @@ def test_tensor_hash_encoder(): encoded = encoder(in_tensor) assert encoded.shape == (10, out_dim) - encoder = encoding.HashEncoding( + encoder = encodings.HashEncoding( num_levels=num_levels, features_per_level=features_per_level, log2_hashmap_size=5, diff --git a/tests/model_components/test_ray_sampler.py b/tests/model_components/test_ray_sampler.py index 577334421..55db613b9 100644 --- a/tests/model_components/test_ray_sampler.py +++ b/tests/model_components/test_ray_sampler.py @@ -5,7 +5,7 @@ Test samplers import torch from nerfstudio.cameras.rays import RayBundle -from nerfstudio.model_components.ray_sampler import ( +from nerfstudio.model_components.ray_samplers import ( LinearDisparitySampler, LogSampler, PDFSampler, diff --git a/tests/utils/test_visualization.py b/tests/utils/test_visualization.py index 39fbcc4d2..824722f2d 100644 --- a/tests/utils/test_visualization.py +++ b/tests/utils/test_visualization.py @@ -1,15 +1,15 @@ """ -Test visualizations +Test colormaps """ import torch -from nerfstudio.utils import visualization +from nerfstudio.utils import colormaps def test_apply_colormap(): """Test adding a colormap to data""" data = torch.rand((10, 20, 1)) - colored_data = visualization.apply_colormap(data) + colored_data = colormaps.apply_colormap(data) assert colored_data.shape == (10, 20, 3) assert torch.min(colored_data) >= 0 @@ -21,7 +21,7 @@ def test_apply_depth_colormap(): data = torch.rand((10, 20, 1)) accum = torch.rand((10, 20, 1)) accum = accum / torch.max(accum) - colored_data = visualization.apply_depth_colormap(depth=data, accumulation=accum) + colored_data = colormaps.apply_depth_colormap(depth=data, accumulation=accum) assert colored_data.shape == (10, 20, 3) assert torch.min(colored_data) >= 0 @@ -32,7 +32,7 @@ def test_apply_boolean_colormap(): """Test adding a colormap to boolean data""" data = torch.rand((10, 20, 1)) data = data > 0.5 - colored_data = visualization.apply_boolean_colormap(data) + colored_data = colormaps.apply_boolean_colormap(data) assert colored_data.shape == (10, 20, 3) assert torch.min(colored_data) >= 0