diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
new file mode 100644
index 000000000..7b46b3429
--- /dev/null
+++ b/.github/workflows/doc.yml
@@ -0,0 +1,34 @@
+name: Docs
+on:
+ push:
+ branches: [main]
+ pull_request:
+ branches: [main]
+ workflow_dispatch:
+
+permissions:
+ contents: write
+jobs:
+ docs:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-python@v3
+ with:
+ python-version: '3.9'
+ - name: Install dependencies
+ run: |
+ pip install .[docs]
+ - name: Sphinx build
+ # fail on warnings
+ run: |
+ sphinx-build docs _build -W --keep-going
+ - name: Deploy
+ uses: peaceiris/actions-gh-pages@v3
+ with:
+ publish_branch: gh-pages
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ publish_dir: _build/
+ force_orphan: true
+ cname: docs.nerf.studio
+ if: github.event_name != 'pull_request'
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
deleted file mode 100644
index d66aefd16..000000000
--- a/.readthedocs.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-# .readthedocs.yaml
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Set the version of Python and other tools you might need
-build:
- os: ubuntu-20.04
- tools:
- python: '3.9'
- # You can also specify other tool versions:
- # nodejs: "16"
- # rust: "1.55"
- # golang: "1.17"
-
-# Build documentation in the docs/ directory with Sphinx
-sphinx:
- fail_on_warning: true
- configuration: docs/conf.py
-
-# If using Sphinx, optionally build your docs in additional formats such as PDF
-# formats:
-# - pdf
-
-# Optionally declare the Python requirements required to build your docs
-python:
- install:
- # Equivalent to 'pip install .'
- - method: pip
- path: .
- # Equivalent to 'pip install .[docs]'
- - method: pip
- path: .
- extra_requirements:
- - docs
diff --git a/README.md b/README.md
index b6f05f9be..a853192ad 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
-
+
@@ -242,15 +242,15 @@ Using an existing dataset is great, but likely you want to use your own data! We
| Data | Capture Device | Requirements | `ns-process-data` Speed |
| ---------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------- | ----------------------- |
-| 📷 [Images](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
-| 📹 [Video](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
-| 🌎 [360 Data](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#360_data) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
-| 📱 [Polycam](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | 🐇 |
-| 📱 [KIRI Engine](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#kiri-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 |
-| 📱 [Record3D](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 |
-| 🖥 [Metashape](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 |
-| 🖥 [RealityCapture](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | 🐇 |
-| 🛠 [Custom](https://docs.nerf.studio/en/latest/quickstart/data_conventions.html) | Any | Camera Poses | 🐇 |
+| 📷 [Images](https://docs.nerf.studio/quickstart/custom_dataset.html#images-or-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
+| 📹 [Video](https://docs.nerf.studio/quickstart/custom_dataset.html#images-or-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
+| 🌎 [360 Data](https://docs.nerf.studio/quickstart/custom_dataset.html#data-equirectangular) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
+| 📱 [Polycam](https://docs.nerf.studio/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | 🐇 |
+| 📱 [KIRI Engine](https://docs.nerf.studio/quickstart/custom_dataset.html#kiri-engine-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 |
+| 📱 [Record3D](https://docs.nerf.studio/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 |
+| 🖥 [Metashape](https://docs.nerf.studio/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 |
+| 🖥 [RealityCapture](https://docs.nerf.studio/quickstart/custom_dataset.html#realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | 🐇 |
+| 🛠 [Custom](https://docs.nerf.studio/quickstart/data_conventions.html) | Any | Camera Poses | 🐇 |
## 5. Advanced Options
diff --git a/docs/Makefile b/docs/Makefile
index d4bb2cbb9..5f8322660 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -3,7 +3,7 @@
# You can set these variables from the command line, and also
# from the environment for the first two.
-SPHINXOPTS ?=
+SPHINXOPTS ?= -W --keep-going # build fail on warning
SPHINXBUILD ?= sphinx-build
SOURCEDIR = .
BUILDDIR = _build
diff --git a/docs/developer_guides/pipelines/datamanagers.md b/docs/developer_guides/pipelines/datamanagers.md
index 32c55b63e..78aec242d 100644
--- a/docs/developer_guides/pipelines/datamanagers.md
+++ b/docs/developer_guides/pipelines/datamanagers.md
@@ -62,8 +62,6 @@ class VanillaDataManagerConfig(InstantiateConfig):
"""number of rays per batch to use per eval iteration"""
eval_num_images_to_sample_from: int = -1
"""number of images to sample during eval iteration"""
- eval_image_indices: Optional[Tuple[int, ...]] = (0,)
- """specifies the image indices to use during eval; if None, uses all"""
camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig()
"""specifies the camera pose optimizer used during training"""
```
diff --git a/docs/developer_guides/viewer/index.md b/docs/developer_guides/viewer/index.md
index 93204a203..9df5c60d5 100644
--- a/docs/developer_guides/viewer/index.md
+++ b/docs/developer_guides/viewer/index.md
@@ -16,7 +16,7 @@ local_viewer
We thank the authors and contributors to the following repos, which we've started, used, and modified for our use-cases.
-- [Viser](https://github.com/brentyi/viser/tree/main/viser) - made by [Brent Yi](https://github.com/brentyi)
+- [Viser](https://github.com/brentyi/viser/) - made by [Brent Yi](https://github.com/brentyi)
- [meshcat-python](https://github.com/rdeits/meshcat-python) - made by [Robin Deits](https://github.com/rdeits)
- [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits)
- [ThreeJS](https://threejs.org/)
diff --git a/docs/make.bat b/docs/make.bat
index 32bb24529..008ca8c48 100644
--- a/docs/make.bat
+++ b/docs/make.bat
@@ -9,6 +9,7 @@ if "%SPHINXBUILD%" == "" (
)
set SOURCEDIR=.
set BUILDDIR=_build
+set SPHINXOPTS="-W --keep-going"
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md
index c82697936..723a97c91 100644
--- a/docs/quickstart/custom_dataset.md
+++ b/docs/quickstart/custom_dataset.md
@@ -57,10 +57,16 @@ There are many ways to install COLMAP, unfortunately it can sometimes be a bit f
::::::{tab-set}
:::::{tab-item} Linux
-We recommend trying `apt`:
+We recommend trying `conda`:
```
-sudo apt install colmap
+conda install -c conda-forge colmap
+```
+
+Check that COLMAP 3.8 with CUDA is successfully installed:
+
+```
+colmap -h
```
If that doesn't work, you can try VKPG:
diff --git a/docs/quickstart/data_conventions.md b/docs/quickstart/data_conventions.md
index 34324e30e..2e9e76eff 100644
--- a/docs/quickstart/data_conventions.md
+++ b/docs/quickstart/data_conventions.md
@@ -82,7 +82,7 @@ For a transform matrix, the first 3 columns are the +X, +Y, and +Z defining the
### Depth images
-To train with depth supervision, you can also provide a `depth_file_path` for each frame in your `transforms.json` and use one of the methods that support additional depth losses (e.g., depth-nerfacto). The depths are assumed to be 16-bit or 32-bit and to be in millimeters to remain consistent with [Polyform](https://github.com/PolyCam/polyform). You can adjust this scaling factor using the `depth_unit_scale_factor` parameter in `NerfstudioDataParserConfig`. Note that by default, we resize the depth images to match the shape of the RGB images.
+To train with depth supervision, you can also provide a `depth_file_path` for each frame in your `transforms.json` and use one of the methods that support additional depth losses (e.g., depth-nerfacto). The depths are assumed to be 16-bit or 32-bit and to be in millimeters to remain consistent with [Polyform](https://github.com/PolyCam/polyform). Zero-value in the depth image is treated as unknown depth. You can adjust this scaling factor using the `depth_unit_scale_factor` parameter in `NerfstudioDataParserConfig`. Note that by default, we resize the depth images to match the shape of the RGB images.
```json
{
diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py
index df8296ea1..7b7648884 100644
--- a/nerfstudio/cameras/camera_optimizers.py
+++ b/nerfstudio/cameras/camera_optimizers.py
@@ -23,19 +23,16 @@ from dataclasses import dataclass, field
from typing import Literal, Optional, Type, Union
import torch
-import tyro
from jaxtyping import Float, Int
from torch import Tensor, nn
from typing_extensions import assert_never
from nerfstudio.cameras.lie_groups import exp_map_SE3, exp_map_SO3xR3
+from nerfstudio.cameras.rays import RayBundle
from nerfstudio.configs.base_config import InstantiateConfig
-from nerfstudio.engine.optimizers import AdamOptimizerConfig, OptimizerConfig
-from nerfstudio.engine.schedulers import (
- ExponentialDecaySchedulerConfig,
- SchedulerConfig,
-)
from nerfstudio.utils import poses as pose_utils
+from nerfstudio.engine.optimizers import OptimizerConfig
+from nerfstudio.engine.schedulers import SchedulerConfig
@dataclass
@@ -47,21 +44,38 @@ class CameraOptimizerConfig(InstantiateConfig):
mode: Literal["off", "SO3xR3", "SE3"] = "off"
"""Pose optimization strategy to use. If enabled, we recommend SO3xR3."""
- position_noise_std: float = 0.0
- """Noise to add to initial positions. Useful for debugging."""
+ trans_l2_penalty: float = 1e-2
+ """L2 penalty on translation parameters."""
- orientation_noise_std: float = 0.0
- """Noise to add to initial orientations. Useful for debugging."""
+ rot_l2_penalty: float = 1e-3
+ """L2 penalty on rotation parameters."""
- optimizer: OptimizerConfig = field(default_factory=lambda: AdamOptimizerConfig(lr=6e-4, eps=1e-15))
- """ADAM parameters for camera optimization."""
+ optimizer: Optional[OptimizerConfig] = field(default=None)
+ """Deprecated, now specified inside the optimizers dict"""
- scheduler: SchedulerConfig = field(default_factory=lambda: ExponentialDecaySchedulerConfig(max_steps=10000))
- """Learning rate scheduler for camera optimizer.."""
+ scheduler: Optional[SchedulerConfig] = field(default=None)
+ """Deprecated, now specified inside the optimizers dict"""
- param_group: tyro.conf.Suppress[str] = "camera_opt"
- """Name of the parameter group used for pose optimization. Can be any string that doesn't conflict with other
- groups."""
+ def __post_init__(self):
+ if self.optimizer is not None:
+ import warnings
+ from nerfstudio.utils.rich_utils import CONSOLE
+
+ CONSOLE.print(
+ "\noptimizer is no longer specified in the CameraOptimizerConfig, it is now defined with the rest of the param groups inside the config file under the name 'camera_opt'\n",
+ style="bold yellow",
+ )
+ warnings.warn("above message coming from", FutureWarning, stacklevel=3)
+
+ if self.scheduler is not None:
+ import warnings
+ from nerfstudio.utils.rich_utils import CONSOLE
+
+ CONSOLE.print(
+ "\nscheduler is no longer specified in the CameraOptimizerConfig, it is now defined with the rest of the param groups inside the config file under the name 'camera_opt'\n",
+ style="bold yellow",
+ )
+ warnings.warn("above message coming from", FutureWarning, stacklevel=3)
class CameraOptimizer(nn.Module):
@@ -91,16 +105,6 @@ class CameraOptimizer(nn.Module):
else:
assert_never(self.config.mode)
- # Initialize pose noise; useful for debugging.
- if config.position_noise_std != 0.0 or config.orientation_noise_std != 0.0:
- assert config.position_noise_std >= 0.0 and config.orientation_noise_std >= 0.0
- std_vector = torch.tensor(
- [config.position_noise_std] * 3 + [config.orientation_noise_std] * 3, device=device
- )
- self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((num_cameras, 6), device=device), std_vector))
- else:
- self.pose_noise = None
-
def forward(
self,
indices: Int[Tensor, "camera_indices"],
@@ -125,13 +129,46 @@ class CameraOptimizer(nn.Module):
assert_never(self.config.mode)
# Detach non-trainable indices by setting to identity transform
if self.non_trainable_camera_indices is not None:
- outputs[0][self.non_trainable_camera_indices] = torch.eye(4, device=self.device)[:3, :4]
+ if self.non_trainable_camera_indices.device != self.pose_adjustment.device:
+ self.non_trainable_camera_indices = self.non_trainable_camera_indices.to(self.pose_adjustment.device)
+ outputs[0][self.non_trainable_camera_indices] = torch.eye(4, device=self.pose_adjustment.device)[:3, :4]
- # Apply initial pose noise.
- if self.pose_noise is not None:
- outputs.append(self.pose_noise[indices, :, :])
# Return: identity if no transforms are needed, otherwise multiply transforms together.
if len(outputs) == 0:
# Note that using repeat() instead of tile() here would result in unnecessary copies.
return torch.eye(4, device=self.device)[None, :3, :4].tile(indices.shape[0], 1, 1)
return functools.reduce(pose_utils.multiply, outputs)
+
+ def apply_to_raybundle(self, raybundle: RayBundle) -> None:
+ """Apply the pose correction to the raybundle"""
+ if self.config.mode != "off":
+ correction_matrices = self(raybundle.camera_indices.squeeze()) # type: ignore
+ raybundle.origins = raybundle.origins + correction_matrices[:, :3, 3]
+ raybundle.directions = torch.bmm(correction_matrices[:, :3, :3], raybundle.directions[..., None]).squeeze()
+
+ def get_loss_dict(self, loss_dict: dict) -> None:
+ """Add regularization"""
+ if self.config.mode != "off":
+ loss_dict["camera_opt_regularizer"] = (
+ self.pose_adjustment[:, :3].norm(dim=-1).mean() * self.config.trans_l2_penalty
+ + self.pose_adjustment[:, 3:].norm(dim=-1).mean() * self.config.rot_l2_penalty
+ )
+
+ def get_correction_matrices(self):
+ """Get optimized pose correction matrices"""
+ return self(torch.arange(0, self.num_cameras).long())
+
+ def get_metrics_dict(self, metrics_dict: dict) -> None:
+ """Get camera optimizer metrics"""
+ if self.config.mode != "off":
+ metrics_dict["camera_opt_translation"] = self.pose_adjustment[:, :3].norm()
+ metrics_dict["camera_opt_rotation"] = self.pose_adjustment[:, 3:].norm()
+
+ def get_param_groups(self, param_groups: dict) -> None:
+ """Get camera optimizer parameters"""
+ camera_opt_params = list(self.parameters())
+ if self.config.mode != "off":
+ assert len(camera_opt_params) > 0
+ param_groups["camera_opt"] = camera_opt_params
+ else:
+ assert len(camera_opt_params) == 0
diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py
index 42cc4399e..d4b3d4de6 100644
--- a/nerfstudio/configs/experiment_config.py
+++ b/nerfstudio/configs/experiment_config.py
@@ -83,7 +83,7 @@ class ExperimentConfig(InstantiateConfig):
def is_viewer_enabled(self) -> bool:
"""Checks if a viewer is enabled."""
- return ("viewer" == self.vis) | ("viewer+wandb" == self.vis) | ("viewer+tensorboard" == self.vis)
+ return self.vis in ("viewer", "viewer+wandb", "viewer+tensorboard", "viewer+comet")
def is_viewer_beta_enabled(self) -> bool:
"""Checks if a viewer beta is enabled."""
diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py
index 50ff17e88..dcfe5599c 100644
--- a/nerfstudio/configs/method_configs.py
+++ b/nerfstudio/configs/method_configs.py
@@ -27,24 +27,24 @@ from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig
from nerfstudio.configs.base_config import ViewerConfig
from nerfstudio.configs.external_methods import get_external_methods
from nerfstudio.data.datamanagers.base_datamanager import (
- VanillaDataManager, VanillaDataManagerConfig)
-from nerfstudio.data.datamanagers.full_images_datamanager import \
- FullImageDatamanagerConfig
-from nerfstudio.data.datamanagers.random_cameras_datamanager import \
- RandomCamerasDataManagerConfig
-from nerfstudio.data.dataparsers.blender_dataparser import \
- BlenderDataParserConfig
+ VanillaDataManager,
+ VanillaDataManagerConfig,
+)
+from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManagerConfig
+from nerfstudio.data.datamanagers.random_cameras_datamanager import (
+ RandomCamerasDataManagerConfig,
+)
+from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig
-from nerfstudio.data.dataparsers.instant_ngp_dataparser import \
- InstantNGPDataParserConfig
-from nerfstudio.data.dataparsers.nerfstudio_dataparser import \
- NerfstudioDataParserConfig
-from nerfstudio.data.dataparsers.phototourism_dataparser import \
- PhototourismDataParserConfig
-from nerfstudio.data.dataparsers.sdfstudio_dataparser import \
- SDFStudioDataParserConfig
-from nerfstudio.data.dataparsers.sitcoms3d_dataparser import \
- Sitcoms3DDataParserConfig
+from nerfstudio.data.dataparsers.instant_ngp_dataparser import (
+ InstantNGPDataParserConfig,
+)
+from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig
+from nerfstudio.data.dataparsers.phototourism_dataparser import (
+ PhototourismDataParserConfig,
+)
+from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
+from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig
from nerfstudio.data.datasets.depth_dataset import DepthDataset
from nerfstudio.data.datasets.sdf_dataset import SDFDataset
from nerfstudio.data.datasets.semantic_dataset import SemanticDataset
@@ -99,17 +99,15 @@ method_configs["nerfacto"] = TrainerConfig(
max_num_iterations=30000,
mixed_precision=True,
pipeline=VanillaPipelineConfig(
- datamanager=VanillaDataManagerConfig(
+ datamanager=ParallelDataManagerConfig(
dataparser=NerfstudioDataParserConfig(),
train_num_rays_per_batch=4096,
eval_num_rays_per_batch=4096,
- camera_optimizer=CameraOptimizerConfig(
- mode="SO3xR3",
- optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2),
- scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000),
- ),
),
- model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15),
+ model=NerfactoModelConfig(
+ eval_num_rays_per_chunk=1 << 15,
+ camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"),
+ ),
),
optimizers={
"proposal_networks": {
@@ -120,10 +118,15 @@ method_configs["nerfacto"] = TrainerConfig(
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000),
},
+ "camera_opt": {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000),
+ },
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
)
+
method_configs["nerfacto-big"] = TrainerConfig(
method_name="nerfacto",
steps_per_eval_batch=500,
@@ -131,14 +134,10 @@ method_configs["nerfacto-big"] = TrainerConfig(
max_num_iterations=100000,
mixed_precision=True,
pipeline=VanillaPipelineConfig(
- datamanager=VanillaDataManagerConfig(
+ datamanager=ParallelDataManagerConfig(
dataparser=NerfstudioDataParserConfig(),
train_num_rays_per_batch=8192,
eval_num_rays_per_batch=4096,
- camera_optimizer=CameraOptimizerConfig(
- mode="SO3xR3",
- optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3),
- ),
),
model=NerfactoModelConfig(
eval_num_rays_per_chunk=1 << 15,
@@ -150,6 +149,7 @@ method_configs["nerfacto-big"] = TrainerConfig(
max_res=4096,
proposal_weights_anneal_max_num_iters=5000,
log2_hashmap_size=21,
+ camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"),
),
),
optimizers={
@@ -161,10 +161,15 @@ method_configs["nerfacto-big"] = TrainerConfig(
"optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=50000),
},
+ "camera_opt": {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000),
+ },
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
)
+
method_configs["nerfacto-huge"] = TrainerConfig(
method_name="nerfacto",
steps_per_eval_batch=500,
@@ -172,15 +177,10 @@ method_configs["nerfacto-huge"] = TrainerConfig(
max_num_iterations=100000,
mixed_precision=True,
pipeline=VanillaPipelineConfig(
- datamanager=VanillaDataManagerConfig(
+ datamanager=ParallelDataManagerConfig(
dataparser=NerfstudioDataParserConfig(),
train_num_rays_per_batch=16384,
eval_num_rays_per_batch=4096,
- camera_optimizer=CameraOptimizerConfig(
- mode="SO3xR3",
- optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3),
- scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-5, max_steps=50000),
- ),
),
model=NerfactoModelConfig(
eval_num_rays_per_chunk=1 << 15,
@@ -196,6 +196,7 @@ method_configs["nerfacto-huge"] = TrainerConfig(
max_res=8192,
proposal_weights_anneal_max_num_iters=5000,
log2_hashmap_size=21,
+ camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"),
),
),
optimizers={
@@ -207,6 +208,10 @@ method_configs["nerfacto-huge"] = TrainerConfig(
"optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=50000),
},
+ "camera_opt": {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000),
+ },
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
@@ -225,11 +230,11 @@ method_configs["depth-nerfacto"] = TrainerConfig(
dataparser=NerfstudioDataParserConfig(),
train_num_rays_per_batch=4096,
eval_num_rays_per_batch=4096,
- camera_optimizer=CameraOptimizerConfig(
- mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2)
- ),
),
- model=DepthNerfactoModelConfig(eval_num_rays_per_chunk=1 << 15),
+ model=DepthNerfactoModelConfig(
+ eval_num_rays_per_chunk=1 << 15,
+ camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"),
+ ),
),
optimizers={
"proposal_networks": {
@@ -240,6 +245,10 @@ method_configs["depth-nerfacto"] = TrainerConfig(
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": None,
},
+ "camera_opt": {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000),
+ },
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
@@ -269,7 +278,6 @@ method_configs["instant-ngp"] = TrainerConfig(
vis="viewer",
)
-
method_configs["instant-ngp-bounded"] = TrainerConfig(
method_name="instant-ngp-bounded",
steps_per_eval_batch=500,
@@ -297,12 +305,12 @@ method_configs["instant-ngp-bounded"] = TrainerConfig(
viewer=ViewerConfig(num_rays_per_chunk=1 << 12),
vis="viewer",
)
-
-
+#
+#
method_configs["mipnerf"] = TrainerConfig(
method_name="mipnerf",
pipeline=VanillaPipelineConfig(
- datamanager=VanillaDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024),
+ datamanager=ParallelDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024),
model=VanillaModelConfig(
_target=MipNerfModel,
loss_coefficients={"rgb_loss_coarse": 0.1, "rgb_loss_fine": 1.0},
@@ -375,13 +383,14 @@ method_configs["tensorf"] = TrainerConfig(
max_num_iterations=30000,
mixed_precision=False,
pipeline=VanillaPipelineConfig(
- datamanager=VanillaDataManagerConfig(
+ datamanager=ParallelDataManagerConfig(
dataparser=BlenderDataParserConfig(),
train_num_rays_per_batch=4096,
eval_num_rays_per_batch=4096,
),
model=TensoRFModelConfig(
regularization="tv",
+ camera_optimizer=CameraOptimizerConfig(mode="off"),
),
),
optimizers={
@@ -393,6 +402,10 @@ method_configs["tensorf"] = TrainerConfig(
"optimizer": AdamOptimizerConfig(lr=0.02),
"scheduler": ExponentialDecaySchedulerConfig(lr_final=0.002, max_steps=30000),
},
+ "camera_opt": {
+ "optimizer": AdamOptimizerConfig(lr=1e-4, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-5, max_steps=5000),
+ },
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
@@ -431,16 +444,16 @@ method_configs["phototourism"] = TrainerConfig(
dataparser=PhototourismDataParserConfig(), # NOTE: one of the only differences with nerfacto
train_num_rays_per_batch=4096,
eval_num_rays_per_batch=4096,
- camera_optimizer=CameraOptimizerConfig(
- mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2)
- ),
# Large dataset, so using prior values from VariableResDataManager.
train_num_images_to_sample_from=40,
train_num_times_to_repeat_images=100,
eval_num_images_to_sample_from=40,
eval_num_times_to_repeat_images=100,
),
- model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15),
+ model=NerfactoModelConfig(
+ eval_num_rays_per_chunk=1 << 15,
+ camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"),
+ ),
),
optimizers={
"proposal_networks": {
@@ -451,6 +464,10 @@ method_configs["phototourism"] = TrainerConfig(
"optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
"scheduler": None,
},
+ "camera_opt": {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000),
+ },
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
@@ -515,9 +532,6 @@ method_configs["neus"] = TrainerConfig(
dataparser=SDFStudioDataParserConfig(),
train_num_rays_per_batch=1024,
eval_num_rays_per_batch=1024,
- camera_optimizer=CameraOptimizerConfig(
- mode="off", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2)
- ),
),
model=NeuSModelConfig(eval_num_rays_per_chunk=1024),
),
@@ -549,9 +563,6 @@ method_configs["neus-facto"] = TrainerConfig(
dataparser=SDFStudioDataParserConfig(),
train_num_rays_per_batch=2048,
eval_num_rays_per_batch=2048,
- camera_optimizer=CameraOptimizerConfig(
- mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2)
- ),
),
model=NeuSFactoModelConfig(
# proposal network allows for significantly smaller sdf/color network
diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py
index 34b540c7c..0c2abb6df 100644
--- a/nerfstudio/data/datamanagers/base_datamanager.py
+++ b/nerfstudio/data/datamanagers/base_datamanager.py
@@ -113,8 +113,6 @@ class DataManagerConfig(InstantiateConfig):
"""Target class to instantiate."""
data: Optional[Path] = None
"""Source of data, may not be used by all models."""
- camera_optimizer: tyro.conf.AvoidSubcommands[Optional[CameraOptimizerConfig]] = None
- """Specifies the camera pose optimizer used during training. Helpful if poses are noisy."""
masks_on_gpu: bool = False
"""Process masks on GPU for speed at the expense of memory, if True."""
images_on_gpu: bool = False
@@ -337,9 +335,6 @@ class VanillaDataManagerConfig(DataManagerConfig):
new images. If -1, never pick new images."""
eval_image_indices: Optional[Tuple[int, ...]] = (0,)
"""Specifies the image indices to use during eval; if None, uses all."""
- camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig()
- """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from
- Record3D."""
collate_fn: Callable[[Any], Any] = cast(Any, staticmethod(nerfstudio_collate))
"""Specifies the collate function to use for the train and eval dataloaders."""
camera_res_scale_factor: float = 1.0
@@ -348,9 +343,21 @@ class VanillaDataManagerConfig(DataManagerConfig):
"""
patch_size: int = 1
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
+ camera_optimizer: Optional[CameraOptimizerConfig] = field(default=None)
+ """Deprecated, has been moved to the model config."""
pixel_sampler: PixelSamplerConfig = PixelSamplerConfig()
"""Specifies the pixel sampler used to sample pixels from images."""
+ def __post_init__(self):
+ """Warn user of camera optimizer change."""
+ if self.camera_optimizer is not None:
+ import warnings
+
+ CONSOLE.print(
+ "\nCameraOptimizerConfig has been moved from the DataManager to the Model.\n", style="bold yellow"
+ )
+ warnings.warn("above message coming from", FutureWarning, stacklevel=3)
+
TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset)
@@ -488,13 +495,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]):
)
self.iter_train_image_dataloader = iter(self.train_image_dataloader)
self.train_pixel_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch)
- self.train_camera_optimizer = self.config.camera_optimizer.setup(
- num_cameras=self.train_dataset.cameras.size, device=self.device
- )
- self.train_ray_generator = RayGenerator(
- self.train_dataset.cameras.to(self.device),
- self.train_camera_optimizer,
- )
+ self.train_ray_generator = RayGenerator(self.train_dataset.cameras.to(self.device))
def setup_eval(self):
"""Sets up the data loader for evaluation"""
@@ -512,13 +513,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]):
)
self.iter_eval_image_dataloader = iter(self.eval_image_dataloader)
self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch)
- self.eval_camera_optimizer = self.config.camera_optimizer.setup(
- num_cameras=self.eval_dataset.cameras.size, device=self.device
- )
- self.eval_ray_generator = RayGenerator(
- self.eval_dataset.cameras.to(self.device),
- self.eval_camera_optimizer,
- )
+ self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device))
# for loading full images
self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader(
input_dataset=self.eval_dataset,
@@ -574,13 +569,4 @@ class VanillaDataManager(DataManager, Generic[TDataset]):
Returns:
A list of dictionaries containing the data manager's param groups.
"""
- param_groups = {}
-
- camera_opt_params = list(self.train_camera_optimizer.parameters())
- if self.config.camera_optimizer.mode != "off":
- assert len(camera_opt_params) > 0
- param_groups[self.config.camera_optimizer.param_group] = camera_opt_params
- else:
- assert len(camera_opt_params) == 0
-
- return param_groups
+ return {}
diff --git a/nerfstudio/data/datamanagers/parallel_datamanager.py b/nerfstudio/data/datamanagers/parallel_datamanager.py
new file mode 100644
index 000000000..7a2e5ac55
--- /dev/null
+++ b/nerfstudio/data/datamanagers/parallel_datamanager.py
@@ -0,0 +1,328 @@
+# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Parallel data manager that generates training data in multiple python processes.
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import queue
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import (
+ Dict,
+ Generic,
+ List,
+ Literal,
+ Optional,
+ Tuple,
+ Type,
+ Union,
+)
+
+import torch
+import torch.multiprocessing as mp
+from rich.progress import track
+from torch.nn import Parameter
+
+from nerfstudio.cameras.cameras import CameraType
+from nerfstudio.cameras.rays import RayBundle
+from nerfstudio.data.datamanagers.base_datamanager import (
+ DataManager,
+ VanillaDataManagerConfig,
+ TDataset,
+ variable_res_collate,
+)
+from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
+from nerfstudio.data.pixel_samplers import (
+ PixelSampler,
+ PixelSamplerConfig,
+ PatchPixelSamplerConfig,
+)
+from nerfstudio.data.utils.dataloaders import (
+ CacheDataloader,
+ FixedIndicesEvalDataloader,
+ RandIndicesEvalDataloader,
+)
+from nerfstudio.model_components.ray_generators import RayGenerator
+from nerfstudio.utils.rich_utils import CONSOLE
+
+
+@dataclass
+class ParallelDataManagerConfig(VanillaDataManagerConfig):
+ """Config for a `ParallelDataManager` which reads data in multiple processes"""
+
+ _target: Type = field(default_factory=lambda: ParallelDataManager)
+ """Target class to instantiate."""
+ num_processes: int = 1
+ """Number of processes to use for train data loading. More than 1 doesn't result in that much better performance"""
+ queue_size: int = 2
+ """Size of shared data queue containing generated ray bundles and batches.
+ If queue_size <= 0, the queue size is infinite."""
+ max_thread_workers: Optional[int] = None
+ """Maximum number of threads to use in thread pool executor. If None, use ThreadPool default."""
+
+
+class DataProcessor(mp.Process):
+ """Parallel dataset batch processor.
+
+ This class is responsible for generating ray bundles from an input dataset
+ in parallel python processes.
+
+ Args:
+ out_queue: the output queue for storing the processed data
+ config: configuration object for the parallel data manager
+ dataparser_outputs: outputs from the dataparser
+ dataset: input dataset
+ pixel_sampler: The pixel sampler for sampling rays
+ """
+
+ def __init__(
+ self,
+ out_queue: mp.Queue,
+ config: ParallelDataManagerConfig,
+ dataparser_outputs: DataparserOutputs,
+ dataset: TDataset,
+ pixel_sampler: PixelSampler,
+ ):
+ super().__init__()
+ self.daemon = True
+ self.out_queue = out_queue
+ self.config = config
+ self.dataparser_outputs = dataparser_outputs
+ self.dataset = dataset
+ self.exclude_batch_keys_from_device = self.dataset.exclude_batch_keys_from_device
+ self.pixel_sampler = pixel_sampler
+ self.ray_generator = RayGenerator(self.dataset.cameras)
+ self.cache_images()
+
+ def run(self):
+ """Append out queue in parallel with ray bundles and batches."""
+ while True:
+ batch = self.pixel_sampler.sample(self.img_data)
+ ray_indices = batch["indices"]
+ ray_bundle: RayBundle = self.ray_generator(ray_indices)
+ # check that GPUs are available
+ if torch.cuda.is_available():
+ ray_bundle = ray_bundle.pin_memory()
+ while True:
+ try:
+ self.out_queue.put_nowait((ray_bundle, batch))
+ break
+ except queue.Full:
+ time.sleep(0.0001)
+ except Exception:
+ CONSOLE.print_exception()
+ CONSOLE.print("[bold red]Error: Error occured in parallel datamanager queue.")
+
+ def cache_images(self):
+ """Caches all input images into a NxHxWx3 tensor."""
+ indices = range(len(self.dataset))
+ batch_list = []
+ results = []
+ with concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers) as executor:
+ for idx in indices:
+ res = executor.submit(self.dataset.__getitem__, idx)
+ results.append(res)
+ for res in track(results, description="Loading data batch", transient=False):
+ batch_list.append(res.result())
+ self.img_data = self.config.collate_fn(batch_list)
+
+
+class ParallelDataManager(DataManager, Generic[TDataset]):
+ """Data manager implementation for parallel dataloading.
+
+ Args:
+ config: the DataManagerConfig used to instantiate class
+ """
+
+ def __init__(
+ self,
+ config: ParallelDataManagerConfig,
+ device: Union[torch.device, str] = "cpu",
+ test_mode: Literal["test", "val", "inference"] = "val",
+ world_size: int = 1,
+ local_rank: int = 0,
+ **kwargs,
+ ):
+ self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", getattr(TDataset, "__default__"))
+ self.config = config
+ self.device = device
+ self.world_size = world_size
+ self.local_rank = local_rank
+ self.test_mode = test_mode
+ self.test_split = "test" if test_mode in ["test", "inference"] else "val"
+ self.dataparser_config = self.config.dataparser
+ if self.config.data is not None:
+ self.config.dataparser.data = Path(self.config.data)
+ else:
+ self.config.data = self.config.dataparser.data
+ self.dataparser = self.dataparser_config.setup()
+ if test_mode == "inference":
+ self.dataparser.downscale_factor = 1 # Avoid opening images
+ self.includes_time = self.dataparser.includes_time
+ self.train_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split="train")
+ self.eval_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split=self.test_split)
+ cameras = self.train_dataparser_outputs.cameras
+ if len(cameras) > 1:
+ for i in range(1, len(cameras)):
+ if cameras[0].width != cameras[i].width or cameras[0].height != cameras[i].height:
+ CONSOLE.print("Variable resolution, using variable_res_collate")
+ self.config.collate_fn = variable_res_collate
+ break
+ self.train_dataset = self.create_train_dataset()
+ self.eval_dataset = self.create_eval_dataset()
+ self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device
+ # Spawn is critical for not freezing the program (PyTorch compatability issue)
+ # check if spawn is already set
+ if mp.get_start_method(allow_none=True) is None:
+ mp.set_start_method("spawn")
+ super().__init__()
+
+ def create_train_dataset(self) -> TDataset:
+ """Sets up the data loaders for training."""
+ return self.dataset_type(
+ dataparser_outputs=self.train_dataparser_outputs,
+ scale_factor=self.config.camera_res_scale_factor,
+ )
+
+ def create_eval_dataset(self) -> TDataset:
+ """Sets up the data loaders for evaluation."""
+ return self.dataset_type(
+ dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split),
+ scale_factor=self.config.camera_res_scale_factor,
+ )
+
+ def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> PixelSampler:
+ """Infer pixel sampler to use."""
+ if self.config.patch_size > 1 and type(self.config.pixel_sampler) is PixelSamplerConfig:
+ return PatchPixelSamplerConfig().setup(
+ patch_size=self.config.patch_size, num_rays_per_batch=num_rays_per_batch
+ )
+ is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all()
+ if is_equirectangular.any():
+ CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.")
+ return self.config.pixel_sampler.setup(
+ is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch
+ )
+
+ def setup_train(self):
+ """Sets up parallel python data processes for training."""
+ assert self.train_dataset is not None
+ self.train_pixel_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) # type: ignore
+ self.data_queue = mp.Manager().Queue(maxsize=self.config.queue_size)
+ self.data_procs = [
+ DataProcessor(
+ out_queue=self.data_queue, # type: ignore
+ config=self.config,
+ dataparser_outputs=self.train_dataparser_outputs,
+ dataset=self.train_dataset,
+ pixel_sampler=self.train_pixel_sampler,
+ )
+ for i in range(self.config.num_processes)
+ ]
+ for proc in self.data_procs:
+ proc.start()
+ print("Started threads")
+
+ # Prime the executor with the first batch
+ self.train_executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers)
+ self.train_batch_fut = self.train_executor.submit(self.data_queue.get)
+
+ def setup_eval(self):
+ """Sets up the data loader for evaluation."""
+ assert self.eval_dataset is not None
+ CONSOLE.print("Setting up evaluation dataset...")
+ self.eval_image_dataloader = CacheDataloader(
+ self.eval_dataset,
+ num_images_to_sample_from=self.config.eval_num_images_to_sample_from,
+ num_times_to_repeat_images=self.config.eval_num_times_to_repeat_images,
+ device=self.device,
+ num_workers=self.world_size * 4,
+ pin_memory=True,
+ collate_fn=self.config.collate_fn,
+ exclude_batch_keys_from_device=self.exclude_batch_keys_from_device,
+ )
+ self.iter_eval_image_dataloader = iter(self.eval_image_dataloader)
+ self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch) # type: ignore
+ self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device))
+ # for loading full images
+ self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader(
+ input_dataset=self.eval_dataset,
+ device=self.device,
+ num_workers=self.world_size * 4,
+ )
+ self.eval_dataloader = RandIndicesEvalDataloader(
+ input_dataset=self.eval_dataset,
+ device=self.device,
+ num_workers=self.world_size * 4,
+ )
+
+ def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
+ """Returns the next batch of data from the parallel training processes."""
+ self.train_count += 1
+
+ # Fetch the next batch in an executor to parallelize the queue get() operation
+ # with the train step
+ bundle, batch = self.train_batch_fut.result()
+ self.train_batch_fut = self.train_executor.submit(self.data_queue.get)
+ ray_bundle = bundle.to(self.device)
+ return ray_bundle, batch
+
+ def next_eval(self, step: int) -> Tuple[RayBundle, Dict]:
+ """Returns the next batch of data from the eval dataloader."""
+ self.eval_count += 1
+ image_batch = next(self.iter_eval_image_dataloader)
+ assert self.eval_pixel_sampler is not None
+ assert isinstance(image_batch, dict)
+ batch = self.eval_pixel_sampler.sample(image_batch)
+ ray_indices = batch["indices"]
+ ray_bundle = self.eval_ray_generator(ray_indices)
+ return ray_bundle, batch
+
+ def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]:
+ """Retrieve the next eval image."""
+ for camera_ray_bundle, batch in self.eval_dataloader:
+ assert camera_ray_bundle.camera_indices is not None
+ image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0])
+ return image_idx, camera_ray_bundle, batch
+ raise ValueError("No more eval images")
+
+ def get_train_rays_per_batch(self) -> int:
+ """Returns the number of rays per batch for training."""
+ return self.config.train_num_rays_per_batch
+
+ def get_eval_rays_per_batch(self) -> int:
+ """Returns the number of rays per batch for evaluation."""
+ return self.config.eval_num_rays_per_batch
+
+ def get_datapath(self) -> Path:
+ """Returns the path to the data. This is used to determine where to save camera paths."""
+ return self.config.dataparser.data
+
+ def get_param_groups(self) -> Dict[str, List[Parameter]]:
+ """Get the param groups for the data manager.
+ Returns:
+ A list of dictionaries containing the data manager's param groups.
+ """
+ return {}
+
+ def __del__(self):
+ """Clean up the parallel data processes."""
+ if hasattr(self, "data_procs"):
+ for proc in self.data_procs:
+ proc.terminate()
+ proc.join()
diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py
index 27ff0c55c..02ddca293 100644
--- a/nerfstudio/data/dataparsers/colmap_dataparser.py
+++ b/nerfstudio/data/dataparsers/colmap_dataparser.py
@@ -147,11 +147,11 @@ class ColmapDataParser(DataParser):
frame.update(cameras[im_data.camera_id])
if self.config.masks_path is not None:
frame["mask_path"] = (
- (self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),
+ (self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix()
)
if self.config.depths_path is not None:
frame["depth_path"] = (
- (self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),
+ (self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix()
)
frames.append(frame)
if camera_model is not None:
diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py
index 463d08294..9234a5d42 100644
--- a/nerfstudio/data/pixel_samplers.py
+++ b/nerfstudio/data/pixel_samplers.py
@@ -98,7 +98,7 @@ class PixelSampler:
chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size)
indices = nonzero_indices[chosen_indices]
else:
- indices = torch.floor(
+ indices = (
torch.rand((batch_size, 3), device=device)
* torch.tensor([num_images, image_height, image_width], device=device)
).long()
@@ -172,13 +172,11 @@ class PixelSampler:
collated_batch = {
key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and value is not None
}
-
assert collated_batch["image"].shape[0] == num_rays_per_batch
# Needed to correct the random indices to their actual camera idx locations.
indices[:, 0] = batch["image_idx"][c]
collated_batch["indices"] = indices # with the abs camera indices
-
if keep_full_image:
collated_batch["full_image"] = batch["image"]
@@ -399,10 +397,11 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
mask: Optional[Tensor] = None,
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
+ rays_to_sample = self.rays_to_sample
if isinstance(mask, Tensor):
m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius)
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
- chosen_indices = random.sample(range(len(nonzero_indices)), k=self.rays_to_sample)
+ chosen_indices = random.sample(range(len(nonzero_indices)), k=rays_to_sample)
indices = nonzero_indices[chosen_indices]
else:
rays_to_sample = self.rays_to_sample
@@ -418,12 +417,12 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device)
indices = torch.concat((ns, hs, ws), dim=1)
- pair_indices = torch.hstack(
- (
- torch.zeros(rays_to_sample, 1, device=device, dtype=torch.long),
- torch.randint(-self.radius, self.radius, (rays_to_sample, 2), device=device, dtype=torch.long),
- )
+ pair_indices = torch.hstack(
+ (
+ torch.zeros(rays_to_sample, 1, device=device, dtype=torch.long),
+ torch.randint(-self.radius, self.radius, (rays_to_sample, 2), device=device, dtype=torch.long),
)
- pair_indices += indices
- indices = torch.hstack((indices, pair_indices)).view(rays_to_sample * 2, 3)
+ )
+ pair_indices += indices
+ indices = torch.hstack((indices, pair_indices)).view(rays_to_sample * 2, 3)
return indices
diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py
index b80b77bcb..9b8d48dba 100644
--- a/nerfstudio/engine/optimizers.py
+++ b/nerfstudio/engine/optimizers.py
@@ -84,6 +84,25 @@ class Optimizers:
self.schedulers = {}
self.parameters = {}
for param_group_name, params in param_groups.items():
+ # For deprecation, catch the camera_opt param group and fix it nicely
+ if param_group_name == "camera_opt" and "camera_opt" not in config:
+ from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig
+ from nerfstudio.utils.rich_utils import CONSOLE
+
+ CONSOLE.print(
+ "\nThe 'camera_opt' param group should be assigned an optimizer in the config. Assigning default optimizers for now. This will be removed in a future release.\n",
+ style="bold yellow",
+ )
+
+ config["camera_opt"] = {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=30000),
+ }
+ # Print some nice warning messages if the user forgot to specify an optimizer
+ if param_group_name not in config:
+ raise RuntimeError(
+ f"""Optimizer config for '{param_group_name}' not found in config file. Make sure you specify an optimizer for each parameter group. Provided configs were: {config.keys()}"""
+ )
lr_init = config[param_group_name]["optimizer"].lr
self.optimizers[param_group_name] = config[param_group_name]["optimizer"].setup(params=params)
self.parameters[param_group_name] = params
diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py
index 6ce7862ed..c7035b7d2 100644
--- a/nerfstudio/engine/trainer.py
+++ b/nerfstudio/engine/trainer.py
@@ -220,13 +220,6 @@ class Trainer:
"""
optimizer_config = self.config.optimizers.copy()
param_groups = self.pipeline.get_param_groups()
- camera_optimizer_config = self.config.pipeline.datamanager.camera_optimizer
- if camera_optimizer_config is not None and camera_optimizer_config.mode != "off":
- assert camera_optimizer_config.param_group not in optimizer_config
- optimizer_config[camera_optimizer_config.param_group] = {
- "optimizer": camera_optimizer_config.optimizer,
- "scheduler": camera_optimizer_config.scheduler,
- }
return Optimizers(optimizer_config, param_groups)
def train(self) -> None:
diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py
index 437ee805f..ee2aac93c 100644
--- a/nerfstudio/field_components/encodings.py
+++ b/nerfstudio/field_components/encodings.py
@@ -27,9 +27,13 @@ from jaxtyping import Float, Int, Shaped
from torch import Tensor, nn
from nerfstudio.field_components.base_field_component import FieldComponent
-from nerfstudio.utils.math import components_from_spherical_harmonics, expected_sin
+from nerfstudio.utils.external import TCNN_EXISTS, tcnn
+from nerfstudio.utils.math import (
+ components_from_spherical_harmonics,
+ expected_sin,
+ generate_polyhedron_basis,
+)
from nerfstudio.utils.printing import print_tcnn_speed_warning
-from nerfstudio.utils.external import tcnn, TCNN_EXISTS
class Encoding(FieldComponent):
@@ -153,7 +157,7 @@ class NeRFEncoding(Encoding):
Output values will be between -1 and 1
"""
scaled_in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
- freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies).to(in_tensor.device)
+ freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies, device=in_tensor.device)
scaled_inputs = scaled_in_tensor[..., None] * freqs # [..., "input_dim", "num_scales"]
scaled_inputs = scaled_inputs.view(*scaled_inputs.shape[:-2], -1) # [..., "input_dim" * "num_scales"]
@@ -178,34 +182,40 @@ class NeRFEncoding(Encoding):
return self.pytorch_fwd(in_tensor, covs)
-class RFFEncoding(Encoding):
- """Random Fourier Feature encoding. Supports integrated encodings.
+class FFEncoding(Encoding):
+ """Fourier Feature encoding. Supports integrated encodings.
Args:
in_dim: Input dimension of tensor
- num_frequencies: Number of encoding frequencies
- scale: Std of Gaussian to sample frequencies. Must be greater than zero
+ basis: Basis matrix from which to construct the Fourier features.
+ num_frequencies: Number of encoded frequencies per axis
+ min_freq_exp: Minimum frequency exponent
+ max_freq_exp: Maximum frequency exponent
include_input: Append the input coordinate to the encoding
"""
- def __init__(self, in_dim: int, num_frequencies: int, scale: float, include_input: bool = False) -> None:
+ def __init__(
+ self,
+ in_dim: int,
+ basis: Float[Tensor, "M N"],
+ num_frequencies: int,
+ min_freq_exp: float,
+ max_freq_exp: float,
+ include_input: bool = False,
+ ) -> None:
super().__init__(in_dim)
-
self.num_frequencies = num_frequencies
- if not scale > 0:
- raise ValueError("RFF encoding scale should be greater than zero")
- self.scale = scale
- if self.in_dim is None:
- raise ValueError("Input dimension has not been set")
- b_matrix = torch.normal(mean=0, std=self.scale, size=(self.in_dim, self.num_frequencies))
- self.register_buffer(name="b_matrix", tensor=b_matrix)
+ self.min_freq = min_freq_exp
+ self.max_freq = max_freq_exp
+ self.register_buffer(name="b_matrix", tensor=basis)
self.include_input = include_input
def get_out_dim(self) -> int:
- out_dim = self.num_frequencies * 2
+ if self.in_dim is None:
+ raise ValueError("Input dimension has not been set")
+ assert isinstance(self.b_matrix, Tensor)
+ out_dim = self.b_matrix.shape[1] * self.num_frequencies * 2
if self.include_input:
- if self.in_dim is None:
- raise ValueError("Input dimension has not been set")
out_dim += self.in_dim
return out_dim
@@ -214,7 +224,7 @@ class RFFEncoding(Encoding):
in_tensor: Float[Tensor, "*bs input_dim"],
covs: Optional[Float[Tensor, "*bs input_dim input_dim"]] = None,
) -> Float[Tensor, "*bs output_dim"]:
- """Calculates RFF encoding. If covariances are provided the encodings will be integrated as proposed
+ """Calculates FF encoding. If covariances are provided the encodings will be integrated as proposed
in mip-NeRF.
Args:
@@ -226,11 +236,16 @@ class RFFEncoding(Encoding):
"""
scaled_in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
scaled_inputs = scaled_in_tensor @ self.b_matrix # [..., "num_frequencies"]
+ freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies, device=in_tensor.device)
+ scaled_inputs = scaled_inputs[..., None] * freqs # [..., "input_dim", "num_scales"]
+ scaled_inputs = scaled_inputs.view(*scaled_inputs.shape[:-2], -1) # [..., "input_dim" * "num_scales"]
if covs is None:
encoded_inputs = torch.sin(torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1))
else:
input_var = torch.sum((covs @ self.b_matrix) * self.b_matrix, -2)
+ input_var = input_var[..., :, None] * freqs[None, :] ** 2
+ input_var = input_var.reshape((*input_var.shape[:-2], -1))
encoded_inputs = expected_sin(
torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1), torch.cat(2 * [input_var], dim=-1)
)
@@ -241,6 +256,49 @@ class RFFEncoding(Encoding):
return encoded_inputs
+class RFFEncoding(FFEncoding):
+ """Random Fourier Feature encoding. Supports integrated encodings.
+
+ Args:
+ in_dim: Input dimension of tensor
+ num_frequencies: Number of encoding frequencies
+ scale: Std of Gaussian to sample frequencies. Must be greater than zero
+ include_input: Append the input coordinate to the encoding
+ """
+
+ def __init__(self, in_dim: int, num_frequencies: int, scale: float, include_input: bool = False) -> None:
+ if not scale > 0:
+ raise ValueError("RFF encoding scale should be greater than zero")
+
+ b_matrix = torch.normal(mean=0, std=scale, size=(in_dim, num_frequencies))
+ super().__init__(in_dim, b_matrix, 1, 0.0, 0.0, include_input)
+
+
+class PolyhedronFFEncoding(FFEncoding):
+ """Fourier Feature encoding using polyhedron basis as proposed by mip-NeRF360. Supports integrated encodings.
+
+ Args:
+ num_frequencies: Number of encoded frequencies per axis
+ min_freq_exp: Minimum frequency exponent
+ max_freq_exp: Maximum frequency exponent
+ basis_shape: Shape of polyhedron basis. Either "octahedron" or "icosahedron"
+ basis_subdivisions: Number of times to tesselate the polyhedron.
+ include_input: Append the input coordinate to the encoding
+ """
+
+ def __init__(
+ self,
+ num_frequencies: int,
+ min_freq_exp: float,
+ max_freq_exp: float,
+ basis_shape: Literal["octahedron", "icosahedron"] = "octahedron",
+ basis_subdivisions: int = 1,
+ include_input: bool = False,
+ ) -> None:
+ basis_t = generate_polyhedron_basis(basis_shape, basis_subdivisions).T
+ super().__init__(3, basis_t, num_frequencies, min_freq_exp, max_freq_exp, include_input)
+
+
class HashEncoding(Encoding):
"""Hash encoding
@@ -424,9 +482,8 @@ class TensorCPEncoding(Encoding):
resolution: Target resolution.
"""
- self.line_coef.data = F.interpolate(
- self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True
- )
+ line_coef = F.interpolate(self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True)
+ self.line_coef = torch.nn.Parameter(line_coef)
self.resolution = resolution
diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py
index 34c6346fc..f215a0045 100644
--- a/nerfstudio/fields/nerfacto_field.py
+++ b/nerfstudio/fields/nerfacto_field.py
@@ -116,6 +116,7 @@ class NerfactoField(Field):
self.use_pred_normals = use_pred_normals
self.pass_semantic_gradients = pass_semantic_gradients
self.base_res = base_res
+ self.step = 0
self.direction_encoding = SHEncoding(
levels=4,
diff --git a/nerfstudio/model_components/ray_generators.py b/nerfstudio/model_components/ray_generators.py
index 49754548d..bad32258d 100644
--- a/nerfstudio/model_components/ray_generators.py
+++ b/nerfstudio/model_components/ray_generators.py
@@ -18,7 +18,6 @@ Ray generator.
from jaxtyping import Int
from torch import Tensor, nn
-from nerfstudio.cameras.camera_optimizers import CameraOptimizer
from nerfstudio.cameras.cameras import Cameras
from nerfstudio.cameras.rays import RayBundle
@@ -34,10 +33,9 @@ class RayGenerator(nn.Module):
image_coords: Tensor
- def __init__(self, cameras: Cameras, pose_optimizer: CameraOptimizer) -> None:
+ def __init__(self, cameras: Cameras) -> None:
super().__init__()
self.cameras = cameras
- self.pose_optimizer = pose_optimizer
self.register_buffer("image_coords", cameras.get_image_coords(), persistent=False)
def forward(self, ray_indices: Int[Tensor, "num_rays 3"]) -> RayBundle:
@@ -51,11 +49,8 @@ class RayGenerator(nn.Module):
x = ray_indices[:, 2] # col indices
coords = self.image_coords[y, x]
- camera_opt_to_camera = self.pose_optimizer(c)
-
ray_bundle = self.cameras.generate_rays(
camera_indices=c.unsqueeze(-1),
coords=coords,
- camera_opt_to_camera=camera_opt_to_camera,
)
return ray_bundle
diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py
index e18c680ae..7a2052b63 100644
--- a/nerfstudio/model_components/ray_samplers.py
+++ b/nerfstudio/model_components/ray_samplers.py
@@ -530,6 +530,7 @@ class ProposalNetworkSampler(Sampler):
single_jitter: Use a same random jitter for all samples along a ray.
update_sched: A function that takes the iteration number of steps between updates.
initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set.
+ pdf_sampler: PDFSampler to use after the first iteration. Uses PDFSampler if not set.
"""
def __init__(
@@ -540,6 +541,7 @@ class ProposalNetworkSampler(Sampler):
single_jitter: bool = False,
update_sched: Callable = lambda x: 1,
initial_sampler: Optional[Sampler] = None,
+ pdf_sampler: Optional[PDFSampler] = None,
) -> None:
super().__init__()
self.num_proposal_samples_per_ray = num_proposal_samples_per_ray
@@ -554,7 +556,10 @@ class ProposalNetworkSampler(Sampler):
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
else:
self.initial_sampler = initial_sampler
- self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter)
+ if pdf_sampler is None:
+ self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter)
+ else:
+ self.pdf_sampler = pdf_sampler
self._anneal = 1.0
self._steps_since_update = 0
diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py
index 1122d5c20..df8eed9b6 100644
--- a/nerfstudio/models/nerfacto.py
+++ b/nerfstudio/models/nerfacto.py
@@ -28,6 +28,7 @@ from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image import PeakSignalNoiseRatio
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
+from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig
from nerfstudio.cameras.rays import RayBundle, RaySamples
from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation
from nerfstudio.field_components.field_heads import FieldHeadNames
@@ -126,6 +127,8 @@ class NerfactoModelConfig(ModelConfig):
"""Which implementation to use for the model."""
appearance_embed_dim: int = 32
"""Dimension of the appearance embedding."""
+ camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="SO3xR3")
+ """Config of the camera optimizer to use"""
class NerfactoModel(Model):
@@ -165,6 +168,9 @@ class NerfactoModel(Model):
implementation=self.config.implementation,
)
+ self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup(
+ num_cameras=self.num_train_data, device="cpu"
+ )
self.density_fns = []
num_prop_nets = self.config.num_proposal_iterations
# Build the proposal network(s)
@@ -229,7 +235,7 @@ class NerfactoModel(Model):
# losses
self.rgb_loss = MSELoss()
-
+ self.step = 0
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
@@ -240,6 +246,7 @@ class NerfactoModel(Model):
param_groups = {}
param_groups["proposal_networks"] = list(self.proposal_networks.parameters())
param_groups["fields"] = list(self.field.parameters())
+ self.camera_optimizer.get_param_groups(param_groups=param_groups)
return param_groups
def get_training_callbacks(
@@ -252,6 +259,7 @@ class NerfactoModel(Model):
def set_anneal(step):
# https://arxiv.org/pdf/2111.12077.pdf eq. 18
+ self.step = step
train_frac = np.clip(step / N, 0, 1)
self.step = step
@@ -278,6 +286,9 @@ class NerfactoModel(Model):
return callbacks
def get_outputs(self, ray_bundle: RayBundle):
+ # apply the camera optimizer pose tweaks
+ if self.training:
+ self.camera_optimizer.apply_to_raybundle(ray_bundle)
ray_samples: RaySamples
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals)
@@ -324,7 +335,6 @@ class NerfactoModel(Model):
for i in range(self.config.num_proposal_iterations):
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
-
return outputs
def get_metrics_dict(self, outputs, batch):
@@ -336,6 +346,8 @@ class NerfactoModel(Model):
if self.training:
metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"])
+
+ self.camera_optimizer.get_metrics_dict(metrics_dict)
return metrics_dict
def get_loss_dict(self, outputs, batch, metrics_dict=None):
@@ -364,6 +376,8 @@ class NerfactoModel(Model):
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
outputs["rendered_pred_normal_loss"]
)
+ # Add loss from camera optimizer
+ self.camera_optimizer.get_loss_dict(loss_dict)
return loss_dict
def get_image_metrics_and_images(
diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py
index a0380d42e..9cc9c7ccf 100644
--- a/nerfstudio/models/tensorf.py
+++ b/nerfstudio/models/tensorf.py
@@ -53,6 +53,7 @@ from nerfstudio.model_components.renderers import (
from nerfstudio.model_components.scene_colliders import AABBBoxCollider
from nerfstudio.models.base_model import Model, ModelConfig
from nerfstudio.utils import colormaps, colors, misc
+from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig
@dataclass
@@ -89,6 +90,8 @@ class TensoRFModelConfig(ModelConfig):
tensorf_encoding: Literal["triplane", "vm", "cp"] = "vm"
regularization: Literal["none", "l1", "tv"] = "l1"
"""Regularization method used in tensorf paper"""
+ camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="SO3xR3")
+ """Config of the camera optimizer to use"""
background_color: Literal["random", "last_sample", "black", "white"] = "white"
"""Whether to randomize the background color."""
@@ -256,6 +259,11 @@ class TensoRFModel(Model):
if self.config.tensorf_encoding == "cp" and self.config.regularization == "tv":
raise RuntimeError("TV reg not supported for CP decomposition")
+ # (optional) camera optimizer
+ self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup(
+ num_cameras=self.num_train_data, device="cpu"
+ )
+
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
@@ -267,11 +275,14 @@ class TensoRFModel(Model):
param_groups["encodings"] = list(self.field.color_encoding.parameters()) + list(
self.field.density_encoding.parameters()
)
+ self.camera_optimizer.get_param_groups(param_groups=param_groups)
return param_groups
def get_outputs(self, ray_bundle: RayBundle):
# uniform sampling
+ if self.training:
+ self.camera_optimizer.apply_to_raybundle(ray_bundle)
ray_samples_uniform = self.sampler_uniform(ray_bundle)
dens = self.field.get_density(ray_samples_uniform)
weights = ray_samples_uniform.get_weights(dens)
@@ -334,6 +345,8 @@ class TensoRFModel(Model):
else:
raise ValueError(f"Regularization {self.config.regularization} not supported")
+ self.camera_optimizer.get_loss_dict(loss_dict)
+
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
return loss_dict
@@ -367,5 +380,7 @@ class TensoRFModel(Model):
"ssim": float(ssim.item()),
"lpips": float(lpips.item()),
}
+ self.camera_optimizer.get_metrics_dict(metrics_dict)
+
images_dict = {"img": combined_rgb, "accumulation": acc, "depth": depth}
return metrics_dict, images_dict
diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py
index 1b802380a..85cc7e528 100644
--- a/nerfstudio/pipelines/base_pipeline.py
+++ b/nerfstudio/pipelines/base_pipeline.py
@@ -37,11 +37,13 @@ from torch.nn.parallel import DistributedDataParallel as DDP
from nerfstudio.cameras.cameras import Cameras
from nerfstudio.configs import base_config as cfg
-from nerfstudio.data.datamanagers.base_datamanager import (DataManager,
- DataManagerConfig,
- VanillaDataManager)
-from nerfstudio.engine.callbacks import (TrainingCallback,
- TrainingCallbackAttributes)
+from nerfstudio.data.datamanagers.base_datamanager import (
+ DataManager,
+ DataManagerConfig,
+ VanillaDataManager,
+)
+from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager
+from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes
from nerfstudio.models.base_model import Model, ModelConfig
from nerfstudio.utils import profiler
@@ -298,18 +300,6 @@ class VanillaPipeline(Pipeline):
ray_bundle, batch = self.datamanager.next_train(step)
model_outputs = self._model(ray_bundle) # train distributed data parallel model if world_size > 1
metrics_dict = self.model.get_metrics_dict(model_outputs, batch)
-
- if self.config.datamanager.camera_optimizer is not None:
- camera_opt_param_group = self.config.datamanager.camera_optimizer.param_group
- if camera_opt_param_group in self.datamanager.get_param_groups():
- # Report the camera optimization metrics
- metrics_dict["camera_opt_translation"] = (
- self.datamanager.get_param_groups()[camera_opt_param_group][0].data[:, :3].norm()
- )
- metrics_dict["camera_opt_rotation"] = (
- self.datamanager.get_param_groups()[camera_opt_param_group][0].data[:, 3:].norm()
- )
-
loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict)
return model_outputs, loss_dict, metrics_dict
@@ -377,9 +367,13 @@ class VanillaPipeline(Pipeline):
"""
self.eval()
metrics_dict_list = []
+<<<<<<< HEAD
# TODO (jake-austin): Can we remove this assert statement? Are all the needed functions
# abstract functions in the upper-most DataManager class?
assert isinstance(self.datamanager, VanillaDataManager)
+=======
+ assert isinstance(self.datamanager, (VanillaDataManager, ParallelDataManager))
+>>>>>>> 8e2a0651d0dc9bb9d6de5d4a934d0a195ad5032d
num_images = len(self.datamanager.fixed_indices_eval_dataloader)
with Progress(
TextColumn("[progress.description]{task.description}"),
diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py
index 2e4fc68ca..b104597ab 100644
--- a/nerfstudio/scripts/exporter.py
+++ b/nerfstudio/scripts/exporter.py
@@ -34,6 +34,7 @@ from typing_extensions import Annotated, Literal
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager
+from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager
from nerfstudio.data.scene_box import OrientedBox
from nerfstudio.exporter import texture_utils, tsdf_utils
from nerfstudio.exporter.exporter_utils import (
@@ -137,7 +138,7 @@ class ExportPointCloud(Exporter):
validate_pipeline(self.normal_method, self.normal_output_name, pipeline)
# Increase the batchsize to speed up the evaluation.
- assert isinstance(pipeline.datamanager, VanillaDataManager)
+ assert isinstance(pipeline.datamanager, (VanillaDataManager, ParallelDataManager))
assert pipeline.datamanager.train_pixel_sampler is not None
pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch
@@ -307,7 +308,7 @@ class ExportPoissonMesh(Exporter):
validate_pipeline(self.normal_method, self.normal_output_name, pipeline)
# Increase the batchsize to speed up the evaluation.
- assert isinstance(pipeline.datamanager, VanillaDataManager)
+ assert isinstance(pipeline.datamanager, (VanillaDataManager, ParallelDataManager))
assert pipeline.datamanager.train_pixel_sampler is not None
pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch
diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py
index 65e98764d..3ce172389 100644
--- a/nerfstudio/scripts/render.py
+++ b/nerfstudio/scripts/render.py
@@ -23,7 +23,8 @@ import os
import struct
import shutil
import sys
-from contextlib import ExitStack
+import gzip
+from contextlib import ExitStack, contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Union
@@ -52,8 +53,11 @@ from nerfstudio.cameras.camera_paths import (
get_path_from_json,
get_spiral_path,
)
-from nerfstudio.cameras.cameras import Cameras, CameraType
-from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager
+from nerfstudio.cameras.cameras import Cameras, CameraType, RayBundle
+from nerfstudio.data.datasets.base_dataset import Dataset
+from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig
+from nerfstudio.data.utils.dataloaders import FixedIndicesEvalDataloader
+from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.data.scene_box import OrientedBox
from nerfstudio.model_components import renderers
from nerfstudio.pipelines.base_pipeline import Pipeline
@@ -585,11 +589,209 @@ class SpiralRender(BaseRender):
)
+@contextmanager
+def _disable_datamanager_setup(cls):
+ """
+ Disables setup_train or setup_eval for faster initialization.
+ """
+ old_setup_train = getattr(cls, "setup_train")
+ old_setup_eval = getattr(cls, "setup_eval")
+ setattr(cls, "setup_train", lambda *args, **kwargs: None)
+ setattr(cls, "setup_eval", lambda *args, **kwargs: None)
+ yield cls
+ setattr(cls, "setup_train", old_setup_train)
+ setattr(cls, "setup_eval", old_setup_eval)
+
+
+@dataclass
+class DatasetRender(BaseRender):
+ """Render all images in the dataset."""
+
+ output_path: Path = Path("renders")
+ """Path to output video file."""
+ data: Optional[Path] = None
+ """Override path to the dataset."""
+ downscale_factor: Optional[float] = None
+ """Scaling factor to apply to the camera image resolution."""
+ split: Literal["train", "val", "test", "train+test"] = "test"
+ """Split to render."""
+ rendered_output_names: Optional[List[str]] = field(default_factory=lambda: None)
+ """Name of the renderer outputs to use. rgb, depth, raw-depth, gt-rgb etc. By default all outputs are rendered."""
+
+ def main(self):
+ config: TrainerConfig
+
+ def update_config(config: TrainerConfig) -> TrainerConfig:
+ data_manager_config = config.pipeline.datamanager
+ assert isinstance(data_manager_config, VanillaDataManagerConfig)
+ data_manager_config.eval_num_images_to_sample_from = -1
+ data_manager_config.eval_num_times_to_repeat_images = -1
+ data_manager_config.train_num_images_to_sample_from = -1
+ data_manager_config.train_num_times_to_repeat_images = -1
+ if self.data is not None:
+ data_manager_config.data = self.data
+ if self.downscale_factor is not None:
+ assert hasattr(data_manager_config.dataparser, "downscale_factor")
+ setattr(data_manager_config.dataparser, "downscale_factor", self.downscale_factor)
+ return config
+
+ config, pipeline, _, _ = eval_setup(
+ self.load_config,
+ eval_num_rays_per_chunk=self.eval_num_rays_per_chunk,
+ test_mode="inference",
+ update_config_callback=update_config,
+ )
+ data_manager_config = config.pipeline.datamanager
+ assert isinstance(data_manager_config, VanillaDataManagerConfig)
+
+ for split in self.split.split("+"):
+ datamanager: VanillaDataManager
+ dataset: Dataset
+ if split == "train":
+ with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access
+ datamanager = data_manager_config.setup(test_mode="test", device=pipeline.device)
+
+ dataset = datamanager.train_dataset
+ dataparser_outputs = getattr(dataset, "_dataparser_outputs", datamanager.train_dataparser_outputs)
+ else:
+ with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access
+ datamanager = data_manager_config.setup(test_mode=split, device=pipeline.device)
+
+ dataset = datamanager.eval_dataset
+ dataparser_outputs = getattr(dataset, "_dataparser_outputs", None)
+ if dataparser_outputs is None:
+ dataparser_outputs = datamanager.dataparser.get_dataparser_outputs(split=datamanager.test_split)
+ dataloader = FixedIndicesEvalDataloader(
+ input_dataset=dataset,
+ device=datamanager.device,
+ num_workers=datamanager.world_size * 4,
+ )
+ images_root = Path(os.path.commonpath(dataparser_outputs.image_filenames))
+ with Progress(
+ TextColumn(f":movie_camera: Rendering split {split} :movie_camera:"),
+ BarColumn(),
+ TaskProgressColumn(
+ text_format="[progress.percentage]{task.completed}/{task.total:>.0f}({task.percentage:>3.1f}%)",
+ show_speed=True,
+ ),
+ ItersPerSecColumn(suffix="fps"),
+ TimeRemainingColumn(elapsed_when_finished=False, compact=False),
+ TimeElapsedColumn(),
+ ) as progress:
+ for camera_idx, (ray_bundle, batch) in enumerate(progress.track(dataloader, total=len(dataset))):
+ ray_bundle: RayBundle
+ with torch.no_grad():
+ outputs = pipeline.model.get_outputs_for_camera_ray_bundle(ray_bundle)
+
+ gt_batch = batch.copy()
+ gt_batch["rgb"] = gt_batch.pop("image")
+ all_outputs = (
+ list(outputs.keys())
+ + [f"raw-{x}" for x in outputs.keys()]
+ + [f"gt-{x}" for x in gt_batch.keys()]
+ + [f"raw-gt-{x}" for x in gt_batch.keys()]
+ )
+ rendered_output_names = self.rendered_output_names
+ if rendered_output_names is None:
+ rendered_output_names = ["gt-rgb"] + list(outputs.keys())
+ for rendered_output_name in rendered_output_names:
+ if rendered_output_name not in all_outputs:
+ CONSOLE.rule("Error", style="red")
+ CONSOLE.print(
+ f"Could not find {rendered_output_name} in the model outputs", justify="center"
+ )
+ CONSOLE.print(
+ f"Please set --rendered-output-name to one of: {all_outputs}", justify="center"
+ )
+ sys.exit(1)
+
+ is_raw = False
+ is_depth = rendered_output_name.find("depth") != -1
+ image_name = f"{camera_idx:05d}"
+
+ # Try to get the original filename
+ image_name = (
+ dataparser_outputs.image_filenames[camera_idx].with_suffix("").relative_to(images_root)
+ )
+
+ output_path = self.output_path / split / rendered_output_name / image_name
+ output_path.parent.mkdir(exist_ok=True, parents=True)
+
+ output_name = rendered_output_name
+ if output_name.startswith("raw-"):
+ output_name = output_name[4:]
+ is_raw = True
+ if output_name.startswith("gt-"):
+ output_name = output_name[3:]
+ output_image = gt_batch[output_name]
+ else:
+ output_image = outputs[output_name]
+ if is_depth:
+ # Divide by the dataparser scale factor
+ output_image.div_(dataparser_outputs.dataparser_scale)
+ else:
+ if output_name.startswith("gt-"):
+ output_name = output_name[3:]
+ output_image = gt_batch[output_name]
+ else:
+ output_image = outputs[output_name]
+ del output_name
+
+ # Map to color spaces / numpy
+ if is_raw:
+ output_image = output_image.cpu().numpy()
+ elif is_depth:
+ output_image = (
+ colormaps.apply_depth_colormap(
+ output_image,
+ accumulation=outputs["accumulation"],
+ near_plane=self.depth_near_plane,
+ far_plane=self.depth_far_plane,
+ colormap_options=self.colormap_options,
+ )
+ .cpu()
+ .numpy()
+ )
+ else:
+ output_image = (
+ colormaps.apply_colormap(
+ image=output_image,
+ colormap_options=self.colormap_options,
+ )
+ .cpu()
+ .numpy()
+ )
+
+ # Save to file
+ if is_raw:
+ with gzip.open(output_path.with_suffix(".npy.gz"), "wb") as f:
+ np.save(f, output_image)
+ elif self.image_format == "png":
+ media.write_image(output_path.with_suffix(".png"), output_image, fmt="png")
+ elif self.image_format == "jpeg":
+ media.write_image(
+ output_path.with_suffix(".jpg"), output_image, fmt="jpeg", quality=self.jpeg_quality
+ )
+ else:
+ raise ValueError(f"Unknown image format {self.image_format}")
+
+ table = Table(
+ title=None,
+ show_header=False,
+ box=box.MINIMAL,
+ title_style=style.Style(bold=True),
+ )
+ for split in self.split.split("+"):
+ table.add_row(f"Outputs {split}", str(self.output_path / split))
+ CONSOLE.print(Panel(table, title="[bold][green]:tada: Render on split {} Complete :tada:[/bold]", expand=False))
+
+
Commands = tyro.conf.FlagConversionOff[
Union[
Annotated[RenderCameraPath, tyro.conf.subcommand(name="camera-path")],
Annotated[RenderInterpolated, tyro.conf.subcommand(name="interpolate")],
Annotated[SpiralRender, tyro.conf.subcommand(name="spiral")],
+ Annotated[DatasetRender, tyro.conf.subcommand(name="dataset")],
]
]
diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py
index 13948678b..0b7306791 100644
--- a/nerfstudio/utils/eval_utils.py
+++ b/nerfstudio/utils/eval_utils.py
@@ -20,13 +20,12 @@ from __future__ import annotations
import os
import sys
from pathlib import Path
-from typing import Literal, Optional, Tuple
+from typing import Literal, Optional, Tuple, Callable
import torch
import yaml
from nerfstudio.configs.method_configs import all_methods
-from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig
from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils.rich_utils import CONSOLE
@@ -69,6 +68,7 @@ def eval_setup(
config_path: Path,
eval_num_rays_per_chunk: Optional[int] = None,
test_mode: Literal["test", "val", "inference"] = "test",
+ update_config_callback: Optional[Callable[[TrainerConfig], TrainerConfig]] = None,
) -> Tuple[TrainerConfig, Pipeline, Path, int]:
"""Shared setup for loading a saved pipeline for evaluation.
@@ -79,6 +79,7 @@ def eval_setup(
'val': loads train/val datasets into memory
'test': loads train/test dataset into memory
'inference': does not load any dataset into memory
+ update_config_callback: Callback to update the config before loading the pipeline
Returns:
@@ -92,11 +93,12 @@ def eval_setup(
if eval_num_rays_per_chunk:
config.pipeline.model.eval_num_rays_per_chunk = eval_num_rays_per_chunk
+ if update_config_callback is not None:
+ config = update_config_callback(config)
+
# load checkpoints from wherever they were saved
# TODO: expose the ability to choose an arbitrary checkpoint
config.load_dir = config.get_checkpoint_dir()
- if isinstance(config.pipeline.datamanager, VanillaDataManagerConfig):
- config.pipeline.datamanager.eval_image_indices = None
# setup pipeline (which includes the DataManager)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py
index 8567a8b28..0ba9e6a51 100644
--- a/nerfstudio/utils/math.py
+++ b/nerfstudio/utils/math.py
@@ -14,6 +14,8 @@
""" Math Helper Functions """
+import itertools
+import math
from dataclasses import dataclass
from typing import Literal, Tuple
@@ -195,7 +197,6 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor:
Returns:
torch.Tensor: The expected value of sin.
"""
-
return torch.exp(-0.5 * x_vars) * torch.sin(x_means)
@@ -360,4 +361,160 @@ def normalized_depth_scale_and_shift(
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
return scale, shift
- return scale, shift
+
+
+def columnwise_squared_l2_distance(
+ x: Float[Tensor, "*M N"],
+ y: Float[Tensor, "*M N"],
+) -> Float[Tensor, "N N"]:
+ """Compute the squared Euclidean distance between all pairs of columns.
+ Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py
+
+ Args:
+ x: tensor of floats, with shape [M, N].
+ y: tensor of floats, with shape [M, N].
+ Returns:
+ sq_dist: tensor of floats, with shape [N, N].
+ """
+ # Use the fact that ||x - y||^2 == ||x||^2 + ||y||^2 - 2 x^T y.
+ sq_norm_x = torch.sum(x**2, 0)
+ sq_norm_y = torch.sum(y**2, 0)
+ sq_dist = sq_norm_x[:, None] + sq_norm_y[None, :] - 2 * x.T @ y
+ return sq_dist
+
+
+def _compute_tesselation_weights(v: int) -> Tensor:
+ """Tesselate the vertices of a triangle by a factor of `v`.
+ Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py
+
+ Args:
+ v: int, the factor of the tesselation (v==1 is a no-op to the triangle).
+
+ Returns:
+ weights: tesselated weights.
+ """
+ if v < 1:
+ raise ValueError(f"v {v} must be >= 1")
+ int_weights = []
+ for i in range(v + 1):
+ for j in range(v + 1 - i):
+ int_weights.append((i, j, v - (i + j)))
+ int_weights = torch.FloatTensor(int_weights)
+ weights = int_weights / v # Barycentric weights.
+ return weights
+
+
+def _tesselate_geodesic(
+ vertices: Float[Tensor, "N 3"], faces: Float[Tensor, "M 3"], v: int, eps: float = 1e-4
+) -> Tensor:
+ """Tesselate the vertices of a geodesic polyhedron.
+
+ Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py
+
+ Args:
+ vertices: tensor of floats, the vertex coordinates of the geodesic.
+ faces: tensor of ints, the indices of the vertices of base_verts that
+ constitute eachface of the polyhedra.
+ v: int, the factor of the tesselation (v==1 is a no-op).
+ eps: float, a small value used to determine if two vertices are the same.
+
+ Returns:
+ verts: a tensor of floats, the coordinates of the tesselated vertices.
+ """
+ tri_weights = _compute_tesselation_weights(v)
+
+ verts = []
+ for face in faces:
+ new_verts = torch.matmul(tri_weights, vertices[face, :])
+ new_verts /= torch.sqrt(torch.sum(new_verts**2, 1, keepdim=True))
+ verts.append(new_verts)
+ verts = torch.concatenate(verts, 0)
+
+ sq_dist = columnwise_squared_l2_distance(verts.T, verts.T)
+ assignment = torch.tensor([torch.min(torch.argwhere(d <= eps)) for d in sq_dist])
+ unique = torch.unique(assignment)
+ verts = verts[unique, :]
+ return verts
+
+
+def generate_polyhedron_basis(
+ basis_shape: Literal["icosahedron", "octahedron"],
+ angular_tesselation: int,
+ remove_symmetries: bool = True,
+ eps: float = 1e-4,
+) -> Tensor:
+ """Generates a 3D basis by tesselating a geometric polyhedron.
+ Basis is used to construct Fourier features for positional encoding.
+ See Mip-Nerf360 paper: https://arxiv.org/abs/2111.12077
+ Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py
+
+ Args:
+ base_shape: string, the name of the starting polyhedron, must be either
+ 'icosahedron' or 'octahedron'.
+ angular_tesselation: int, the number of times to tesselate the polyhedron,
+ must be >= 1 (a value of 1 is a no-op to the polyhedron).
+ remove_symmetries: bool, if True then remove the symmetric basis columns,
+ which is usually a good idea because otherwise projections onto the basis
+ will have redundant negative copies of each other.
+ eps: float, a small number used to determine symmetries.
+
+ Returns:
+ basis: a matrix with shape [3, n].
+ """
+ if basis_shape == "icosahedron":
+ a = (math.sqrt(5) + 1) / 2
+ verts = torch.FloatTensor(
+ [
+ (-1, 0, a),
+ (1, 0, a),
+ (-1, 0, -a),
+ (1, 0, -a),
+ (0, a, 1),
+ (0, a, -1),
+ (0, -a, 1),
+ (0, -a, -1),
+ (a, 1, 0),
+ (-a, 1, 0),
+ (a, -1, 0),
+ (-a, -1, 0),
+ ]
+ ) / math.sqrt(a + 2)
+ faces = torch.tensor(
+ [
+ (0, 4, 1),
+ (0, 9, 4),
+ (9, 5, 4),
+ (4, 5, 8),
+ (4, 8, 1),
+ (8, 10, 1),
+ (8, 3, 10),
+ (5, 3, 8),
+ (5, 2, 3),
+ (2, 7, 3),
+ (7, 10, 3),
+ (7, 6, 10),
+ (7, 11, 6),
+ (11, 0, 6),
+ (0, 1, 6),
+ (6, 1, 10),
+ (9, 0, 11),
+ (9, 11, 2),
+ (9, 2, 5),
+ (7, 2, 11),
+ ]
+ )
+ verts = _tesselate_geodesic(verts, faces, angular_tesselation)
+ elif basis_shape == "octahedron":
+ verts = torch.FloatTensor([(0, 0, -1), (0, 0, 1), (0, -1, 0), (0, 1, 0), (-1, 0, 0), (1, 0, 0)])
+ corners = torch.FloatTensor(list(itertools.product([-1, 1], repeat=3)))
+ pairs = torch.argwhere(columnwise_squared_l2_distance(corners.T, verts.T) == 2)
+ faces, _ = torch.sort(torch.reshape(pairs[:, 1], [3, -1]).T, 1)
+ verts = _tesselate_geodesic(verts, faces, angular_tesselation)
+
+ if remove_symmetries:
+ # Remove elements of `verts` that are reflections of each other.
+ match = columnwise_squared_l2_distance(verts.T, -verts.T) < eps
+ verts = verts[torch.any(torch.triu(match), 1), :]
+
+ basis = verts.flip(-1)
+ return basis
diff --git a/nerfstudio/utils/tensor_dataclass.py b/nerfstudio/utils/tensor_dataclass.py
index 0ebe7b5ce..a2b8d1dad 100644
--- a/nerfstudio/utils/tensor_dataclass.py
+++ b/nerfstudio/utils/tensor_dataclass.py
@@ -261,6 +261,14 @@ class TensorDataclass:
"""
return self._apply_fn_to_fields(lambda x: x.to(device))
+ def pin_memory(self: TensorDataclassT) -> TensorDataclassT:
+ """Pins the tensor dataclass memory
+
+ Returns:
+ TensorDataclass: A new TensorDataclass with the same data but pinned.
+ """
+ return self._apply_fn_to_fields(lambda x: x.pin_memory())
+
def _apply_fn_to_fields(
self: TensorDataclassT,
fn: Callable,
diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py
index b76c2dd53..79da0fd1f 100644
--- a/nerfstudio/viewer_beta/viewer.py
+++ b/nerfstudio/viewer_beta/viewer.py
@@ -161,22 +161,40 @@ class Viewer:
with tabs.add_tab("Export", viser.Icon.PACKAGE_EXPORT):
populate_export_tab(self.viser_server, self.control_panel, config_path)
- def nested_folder_install(folder_labels: List[str], element: ViewerElement):
+ # Keep track of the pointers to generated GUI folders, because each generated folder holds a unique ID.
+ viewer_gui_folders = dict()
+
+ def nested_folder_install(folder_labels: List[str], prev_labels: List[str], element: ViewerElement):
if len(folder_labels) == 0:
element.install(self.viser_server)
# also rewire the hook to rerender
prev_cb = element.cb_hook
element.cb_hook = lambda element: [prev_cb(element), self._interrupt_render(element)]
else:
- with self.viser_server.add_gui_folder(folder_labels[0]):
- nested_folder_install(folder_labels[1:], element)
+ # recursively create folders
+ # If the folder name is "Custom Elements/a/b", then:
+ # in the beginning: folder_path will be
+ # "/".join([] + ["Custom Elements"]) --> "Custom Elements"
+ # later, folder_path will be
+ # "/".join(["Custom Elements"] + ["a"]) --> "Custom Elements/a"
+ # "/".join(["Custom Elements", "a"] + ["b"]) --> "Custom Elements/a/b"
+ # --> the element will be installed in the folder "Custom Elements/a/b"
+ #
+ # Note that the gui_folder is created only when the folder is not in viewer_gui_folders,
+ # and we use the folder_path as the key to check if the folder is already created.
+ # Otherwise, use the existing folder as context manager.
+ folder_path = "/".join(prev_labels + [folder_labels[0]])
+ if folder_path not in viewer_gui_folders:
+ viewer_gui_folders[folder_path] = self.viser_server.add_gui_folder(folder_labels[0])
+ with viewer_gui_folders[folder_path]:
+ nested_folder_install(folder_labels[1:], prev_labels + [folder_labels[0]], element)
with control_tab:
self.viewer_elements = []
self.viewer_elements.extend(parse_object(pipeline, ViewerElement, "Custom Elements"))
for param_path, element in self.viewer_elements:
folder_labels = param_path.split("/")[:-1]
- nested_folder_install(folder_labels, element)
+ nested_folder_install(folder_labels, [], element)
# scrape the trainer/pipeline for any ViewerControl objects to initialize them
self.viewer_controls: List[ViewerControl] = [
diff --git a/tests/data/configs/test_config1.yml b/tests/data/configs/test_config1.yml
index 3cf014760..67ce18cda 100644
--- a/tests/data/configs/test_config1.yml
+++ b/tests/data/configs/test_config1.yml
@@ -111,8 +111,6 @@ pipeline: !!python/object:nerfstudio.pipelines.base_pipeline.VanillaPipelineConf
- data
- varun
- cheezit-scaled
- eval_image_indices: !!python/tuple
- - 0
eval_num_images_to_sample_from: -1
eval_num_rays_per_batch: 4096
eval_num_times_to_repeat_images: -1
diff --git a/tests/data/configs/test_config2.yml b/tests/data/configs/test_config2.yml
index c1ccda379..90737789f 100644
--- a/tests/data/configs/test_config2.yml
+++ b/tests/data/configs/test_config2.yml
@@ -105,8 +105,6 @@ pipeline: !!python/object:nerfstudio.pipelines.base_pipeline.VanillaPipelineConf
- data
- varun
- cheezit-scaled
- eval_image_indices: !!python/tuple
- - 0
eval_num_images_to_sample_from: -1
eval_num_rays_per_batch: 4096
eval_num_times_to_repeat_images: -1