diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 000000000..84f99bf49
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,13 @@
+{
+ "name": "NerfStudio Dev Container",
+ "dockerComposeFile": "docker-compose.yml",
+ "service": "nerfstudio",
+ "workspaceFolder": "/workspace",
+ "shutdownAction": "stopCompose",
+ "extensions": [
+ "mutantdino.resourcemonitor",
+ "ms-azuretools.vscode-docker",
+ "nvidia.nsight-vscode-edition",
+ "ms-python.python"
+ ]
+}
\ No newline at end of file
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
new file mode 100644
index 000000000..6376422cf
--- /dev/null
+++ b/.devcontainer/docker-compose.yml
@@ -0,0 +1,28 @@
+version: "3.8"
+
+services:
+ nerfstudio:
+ shm_size: '12gb'
+ image: dromni/nerfstudio:0.1.17
+ # build:
+ # context: ..
+ # dockerfile: ./Dockerfile
+ stdin_open: true
+ tty: true
+ environment:
+ NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
+ DISPLAY: $DISPLAY
+ ports:
+ - "7007:7007"
+ volumes:
+ - ../:/workspace/
+ - /tmp/.X11-unix:/tmp/.X11-unix
+ working_dir: /workspace/
+ command: /bin/bash
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ count: 1
+ capabilities: [ gpu ]
\ No newline at end of file
diff --git a/README.md b/README.md
index ca2bd48b4..0b0d8592c 100644
--- a/README.md
+++ b/README.md
@@ -95,13 +95,25 @@ python -m pip install --upgrade pip
### Dependencies
-Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
+Install pytorch with CUDA (this repo has been tested with CUDA 11.3 and CUDA 11.7) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
+
+For CUDA 11.3:
```bash
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
+For CUDA 11.7:
+
+```bash
+pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117
+pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
+```
+
+See [Dependencies](https://github.com/nerfstudio-project/nerfstudio/blob/main/docs/quickstart/installation.md#dependencies)
+in the Installation documentation for more.
+
### Installing nerfstudio
Easy option:
@@ -290,16 +302,18 @@ We provide the following support structures to make life easier for getting star
# Citation
+You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
+
If you use this library or find the documentation useful for your research, please consider citing:
```
-@misc{nerfstudio,
- title={Nerfstudio: A Framework for Neural Radiance Field Development},
- author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi,
- Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi,
- Abhik Ahuja, David McAllister, Angjoo Kanazawa},
- year={2022},
- url={https://github.com/nerfstudio-project/nerfstudio},
+@article{nerfstudio,
+ author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi,
+ Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin,
+ Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo},
+ title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development},
+ journal = {arXiv preprint arXiv:2302.04264},
+ year = {2023},
}
```
diff --git a/colab/demo.ipynb b/colab/demo.ipynb
index e92f8b04d..d1c58ad6c 100644
--- a/colab/demo.ipynb
+++ b/colab/demo.ipynb
@@ -292,9 +292,9 @@
"if os.path.exists(f\"data/nerfstudio/{scene}/transforms.json\"):\n",
" !ns-train nerfacto --viewer.websocket-port 7007 nerfstudio-data --data data/nerfstudio/$scene --downscale-factor 4\n",
"else:\n",
- " import IPython\n",
- " display(IPython.display.HTML('
Error: Data processing did not complete
'))\n",
- " display(IPython.display.HTML('Please re-run `Downloading and Processing Data`, or view the FAQ for more info.
'))"
+ " from IPython.core.display import display, HTML\n",
+ " display(HTML('Error: Data processing did not complete
'))\n",
+ " display(HTML('Please re-run `Downloading and Processing Data`, or view the FAQ for more info.
'))"
]
},
{
@@ -383,4 +383,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
diff --git a/docs/developer_guides/pipelines/datamanagers.md b/docs/developer_guides/pipelines/datamanagers.md
index bbe3413d9..32c55b63e 100644
--- a/docs/developer_guides/pipelines/datamanagers.md
+++ b/docs/developer_guides/pipelines/datamanagers.md
@@ -87,7 +87,7 @@ def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
You can see our code for more details.
-```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/datamanagers.py
+```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/data/datamanagers/base_datamanager.py
:color: primary
:outline:
See the code!
diff --git a/docs/index.md b/docs/index.md
index 53d45784d..88e4c90ae 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -155,16 +155,18 @@ We'll be constantly growing this list! So make sure to check back in to see our
## Citation
+You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
+
If you use this library or find the documentation useful for your research, please consider citing:
```none
-@misc{nerfstudio,
- title={Nerfstudio: A Framework for Neural Radiance Field Development},
- author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi,
- Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi,
- Abhik Ahuja, David McAllister, Angjoo Kanazawa},
- year={2022},
- url={https://github.com/nerfstudio-project/nerfstudio},
+@article{nerfstudio,
+ author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi,
+ Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin,
+ Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo},
+ title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development},
+ journal = {arXiv preprint arXiv:2302.04264},
+ year = {2023},
}
```
diff --git a/docs/quickstart/first_nerf.md b/docs/quickstart/first_nerf.md
index e7071c329..73e40a9d8 100644
--- a/docs/quickstart/first_nerf.md
+++ b/docs/quickstart/first_nerf.md
@@ -30,7 +30,7 @@ Navigating to the link at the end of the terminal will load the webviewer. If yo
- All data configurations must go at the end. In this case, `nerfstudio-data` and all of its corresponding configurations come at the end after the model and viewer specification.
:::
-## Resume from checkpoint / visualize existing run
+## Resume from checkpoint
It is possible to load a pretrained model by running
@@ -40,6 +40,14 @@ ns-train nerfacto --data data/nerfstudio/poster --load-dir {outputs/.../nerfstud
This will automatically start training. If you do not want it to train, add `--viewer.start-train False` to your training command.
+## Visualize existing run
+
+Given a pretrained model checkpoint, you can start the viewer by running
+
+```bash
+ns-viewer --load-config {outputs/.../config.yml}
+```
+
## Exporting Results
Once you have a NeRF model you can either render out a video or export a point cloud.
diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md
index 97c63aac8..20ab8859c 100644
--- a/docs/quickstart/installation.md
+++ b/docs/quickstart/installation.md
@@ -17,12 +17,38 @@ python -m pip install --upgrade pip
## Dependencies
-Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
+(pytorch)=
+### pytorch
+
+Either pytorch 1.12.1 (with CUDA up to 11.3) or 1.13.1 (with CUDA 11.6/7/8) can be used.
+
+- To install 1.12.1 with CUDA 11.3:
```bash
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
-pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
+```
+- To install 1.13.1 with CUDA 11.7:
+
+Note that if a pytorch version prior to 1.13 is installed,
+it should be uninstalled first to avoid upgrade issues (e.g. with functorch)
+
+```bash
+pip uninstall torch torchvision functorch
+```
+
+Install pytorch 1.13.1 with CUDA (this repo has been tested with CUDA 11.7.1) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
+
+```bash
+pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117
+```
+
+### tinycudann
+
+After pytorch, install the torch bindings for tinycudann:
+
+```bash
+pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
## Installing nerfstudio
@@ -134,9 +160,8 @@ While installing tiny-cuda, you run into: `The detected CUDA version mismatches
**Solution**:
-```
-pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
-```
+Reinstall pytorch with the correct CUDA version.
+See [pytorch](pytorch) under Dependencies, above.
diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py
index 61c489edf..a43504694 100644
--- a/nerfstudio/cameras/camera_paths.py
+++ b/nerfstudio/cameras/camera_paths.py
@@ -100,12 +100,16 @@ def get_spiral_path(
new_c2ws.append(c2wh[:3, :4])
new_c2ws = torch.stack(new_c2ws, dim=0)
+ times = None
+ if camera.times is not None:
+ times = torch.linspace(0, 1, steps)[:, None]
return Cameras(
fx=camera.fx[0],
fy=camera.fy[0],
cx=camera.cx[0],
cy=camera.cy[0],
camera_to_worlds=new_c2ws,
+ times=times,
)
diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py
index d7d228bae..80b7f6487 100644
--- a/nerfstudio/cameras/cameras.py
+++ b/nerfstudio/cameras/cameras.py
@@ -230,7 +230,7 @@ class Cameras(TensorDataclass):
c_x_y: cx or cy for when h_w == None
"""
if isinstance(h_w, int):
- h_w = torch.Tensor([h_w]).to(torch.int64).to(self.device)
+ h_w = torch.as_tensor([h_w]).to(torch.int64).to(self.device)
elif isinstance(h_w, torch.Tensor):
assert not torch.is_floating_point(h_w), f"height and width tensor must be of type int, not: {h_w.dtype}"
h_w = h_w.to(torch.int64).to(self.device)
@@ -238,7 +238,7 @@ class Cameras(TensorDataclass):
h_w = h_w.unsqueeze(-1)
# assert torch.all(h_w == h_w.view(-1)[0]), "Batched cameras of different h, w will be allowed in the future."
elif h_w is None:
- h_w = torch.Tensor((c_x_y * 2).to(torch.int64).to(self.device))
+ h_w = torch.as_tensor((c_x_y * 2)).to(torch.int64).to(self.device)
else:
raise ValueError("Height must be an int, tensor, or None, received: " + str(type(h_w)))
return h_w
diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py
index d032c4d48..1c8c3e08d 100644
--- a/nerfstudio/data/datamanagers/base_datamanager.py
+++ b/nerfstudio/data/datamanagers/base_datamanager.py
@@ -49,6 +49,7 @@ from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserCo
from nerfstudio.data.dataparsers.phototourism_dataparser import (
PhototourismDataParserConfig,
)
+from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler
from nerfstudio.data.utils.dataloaders import (
@@ -75,6 +76,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of
"dnerf-data": DNeRFDataParserConfig(),
"phototourism-data": PhototourismDataParserConfig(),
"dycheck-data": DycheckDataParserConfig(),
+ "sdfstudio-data": SDFStudioDataParserConfig(),
},
prefix_names=False, # Omit prefixes in subcommands themselves.
)
diff --git a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py
index d09aa86f9..2fdc2df85 100644
--- a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py
+++ b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py
@@ -123,6 +123,10 @@ class InstantNGP(DataParser):
w, h = meta["w"], meta["h"]
+ camera_type = CameraType.PERSPECTIVE
+ if meta.get("is_fisheye", False):
+ camera_type = CameraType.FISHEYE
+
cameras = Cameras(
fx=float(fl_x),
fy=float(fl_y),
@@ -132,7 +136,7 @@ class InstantNGP(DataParser):
height=int(h),
width=int(w),
camera_to_worlds=camera_to_world,
- camera_type=CameraType.PERSPECTIVE,
+ camera_type=camera_type,
)
# TODO(ethan): add alpha background color
diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py
new file mode 100644
index 000000000..f25c8f3e7
--- /dev/null
+++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py
@@ -0,0 +1,158 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Datapaser for sdfstudio formatted data"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Type
+
+import torch
+from rich.console import Console
+
+from nerfstudio.cameras import camera_utils
+from nerfstudio.cameras.cameras import Cameras, CameraType
+from nerfstudio.data.dataparsers.base_dataparser import (
+ DataParser,
+ DataParserConfig,
+ DataparserOutputs,
+)
+from nerfstudio.data.scene_box import SceneBox
+from nerfstudio.utils.io import load_from_json
+
+CONSOLE = Console()
+
+
+@dataclass
+class SDFStudioDataParserConfig(DataParserConfig):
+ """Scene dataset parser config"""
+
+ _target: Type = field(default_factory=lambda: SDFStudio)
+ """target class to instantiate"""
+ data: Path = Path("data/DTU/scan65")
+ """Directory specifying location of data."""
+ include_mono_prior: bool = False
+ """whether or not to load monocular depth and normal """
+ include_foreground_mask: bool = False
+ """whether or not to load foreground mask"""
+ downscale_factor: int = 1
+ scene_scale: float = 2.0
+ """
+ Sets the bounding cube to have edge length of this size.
+ The longest dimension of the Friends axis-aligned bbox will be scaled to this value.
+ """
+ skip_every_for_val_split: int = 1
+ """sub sampling validation images"""
+ auto_orient: bool = False
+
+
+@dataclass
+class SDFStudio(DataParser):
+ """SDFStudio Dataset"""
+
+ config: SDFStudioDataParserConfig
+
+ def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements
+ # load meta data
+ meta = load_from_json(self.config.data / "meta_data.json")
+
+ indices = list(range(len(meta["frames"])))
+ # subsample to avoid out-of-memory for validation set
+ if split != "train" and self.config.skip_every_for_val_split >= 1:
+ indices = indices[:: self.config.skip_every_for_val_split]
+
+ image_filenames = []
+ depth_filenames = []
+ normal_filenames = []
+ transform = None
+ fx = []
+ fy = []
+ cx = []
+ cy = []
+ camera_to_worlds = []
+ for i, frame in enumerate(meta["frames"]):
+ if i not in indices:
+ continue
+
+ image_filename = self.config.data / frame["rgb_path"]
+ depth_filename = self.config.data / frame["mono_depth_path"]
+ normal_filename = self.config.data / frame["mono_normal_path"]
+
+ intrinsics = torch.tensor(frame["intrinsics"])
+ camtoworld = torch.tensor(frame["camtoworld"])
+
+ # append data
+ image_filenames.append(image_filename)
+ depth_filenames.append(depth_filename)
+ normal_filenames.append(normal_filename)
+ fx.append(intrinsics[0, 0])
+ fy.append(intrinsics[1, 1])
+ cx.append(intrinsics[0, 2])
+ cy.append(intrinsics[1, 2])
+ camera_to_worlds.append(camtoworld)
+
+ fx = torch.stack(fx)
+ fy = torch.stack(fy)
+ cx = torch.stack(cx)
+ cy = torch.stack(cy)
+ camera_to_worlds = torch.stack(camera_to_worlds)
+
+ # Convert from COLMAP's/OPENCV's camera coordinate system to nerfstudio
+ camera_to_worlds[:, 0:3, 1:3] *= -1
+
+ if self.config.auto_orient:
+ camera_to_worlds, transform = camera_utils.auto_orient_and_center_poses(
+ camera_to_worlds,
+ method="up",
+ center_poses=False,
+ )
+
+ # scene box from meta data
+ meta_scene_box = meta["scene_box"]
+ aabb = torch.tensor(meta_scene_box["aabb"], dtype=torch.float32)
+ scene_box = SceneBox(
+ aabb=aabb,
+ )
+
+ height, width = meta["height"], meta["width"]
+ cameras = Cameras(
+ fx=fx,
+ fy=fy,
+ cx=cx,
+ cy=cy,
+ height=height,
+ width=width,
+ camera_to_worlds=camera_to_worlds[:, :3, :4],
+ camera_type=CameraType.PERSPECTIVE,
+ )
+
+ # TODO supports downsample
+ # cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor)
+
+ assert meta["has_mono_prior"] == self.config.include_mono_prior, f"no mono prior in {self.config.data}"
+
+ dataparser_outputs = DataparserOutputs(
+ image_filenames=image_filenames,
+ cameras=cameras,
+ scene_box=scene_box,
+ metadata={
+ "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None,
+ "normal_filenames": normal_filenames if len(normal_filenames) > 0 else None,
+ "transform": transform,
+ "camera_to_worlds": camera_to_worlds if len(camera_to_worlds) > 0 else None,
+ "include_mono_prior": self.config.include_mono_prior,
+ },
+ )
+ return dataparser_outputs
diff --git a/nerfstudio/data/datasets/sdf_dataset.py b/nerfstudio/data/datasets/sdf_dataset.py
new file mode 100644
index 000000000..48e0cff6d
--- /dev/null
+++ b/nerfstudio/data/datasets/sdf_dataset.py
@@ -0,0 +1,97 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+SDFStudio dataset.
+"""
+
+from pathlib import Path
+from typing import Dict
+
+import numpy as np
+import torch
+
+from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
+from nerfstudio.data.datasets.base_dataset import InputDataset
+
+
+class SDFDataset(InputDataset):
+ """Dataset that returns images and depths.
+
+ Args:
+ dataparser_outputs: description of where and how to read input images.
+ scale_factor: The scaling factor for the dataparser outputs.
+ """
+
+ def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
+ super().__init__(dataparser_outputs, scale_factor)
+
+ # can be none if monoprior not included
+ self.depth_filenames = self.metadata["depth_filenames"]
+ self.normal_filenames = self.metadata["normal_filenames"]
+ self.camera_to_worlds = self.metadata["camera_to_worlds"]
+ # can be none if auto orient not enabled in dataparser
+ self.transform = self.metadata["normal_filenames"]
+ self.include_mono_prior = self.metadata["include_mono_prior"]
+
+ def get_metadata(self, data: Dict) -> Dict:
+ # TODO supports foreground_masks
+ metadata = {}
+ if self.include_mono_prior:
+ depth_filepath = self.depth_filenames[data["image_idx"]]
+ normal_filepath = self.normal_filenames[data["image_idx"]]
+ camtoworld = self.camera_to_worlds[data["image_idx"]]
+
+ # Scale depth images to meter units and also by scaling applied to cameras
+ depth_image, normal_image = self.get_depths_and_normals(
+ depth_filepath=depth_filepath, normal_filename=normal_filepath, camtoworld=camtoworld
+ )
+ metadata["depth_image"] = depth_image
+ metadata["normal_image"] = normal_image
+
+ return metadata
+
+ def get_depths_and_normals(self, depth_filepath: Path, normal_filename: Path, camtoworld: np.ndarray):
+ """function to process additional depths and normal information
+ Args:
+ depth_filepath: path to depth file
+ normal_filename: path to normal file
+ camtoworld: camera to world transformation matrix
+ """
+
+ # load mono depth
+ depth = np.load(depth_filepath)
+ depth = torch.from_numpy(depth).float()
+
+ # load mono normal
+ normal = np.load(normal_filename)
+
+ # transform normal to world coordinate system
+ normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here
+ normal = torch.from_numpy(normal).float()
+
+ rot = camtoworld[:3, :3]
+
+ normal_map = normal.reshape(3, -1)
+ normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0)
+
+ normal_map = rot @ normal_map
+ normal = normal_map.permute(1, 0).reshape(*normal.shape[1:], 3)
+
+ if self.transform is not None:
+ h, w, _ = normal.shape
+ normal = self.transform[:3, :3] @ normal.reshape(-1, 3).permute(1, 0)
+ normal = normal.permute(1, 0).reshape(h, w, 3)
+
+ return depth, normal
diff --git a/nerfstudio/field_components/temporal_grid.py b/nerfstudio/field_components/temporal_grid.py
index 6bccc110a..3bbe274b7 100644
--- a/nerfstudio/field_components/temporal_grid.py
+++ b/nerfstudio/field_components/temporal_grid.py
@@ -116,7 +116,6 @@ class TemporalGridEncodeFunc(Function):
@staticmethod
@custom_bwd
def backward(ctx, grad):
-
inputs, temporal_row_index, embeddings, offsets, dy_dx = ctx.saved_tensors
B, D, grid_channel, C, L, S, H, gridtype = ctx.dims
align_corners = ctx.align_corners
@@ -336,7 +335,7 @@ class TemporalGridEncoder(nn.Module):
"""
outputs = TemporalGridEncodeFunc.apply(
xyz,
- self.get_temporal_index(time[:, 0]),
+ self.get_temporal_index(time[:, 0].float()),
self.embeddings,
self.offsets,
self.per_level_scale,
diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py
index a91c3b49f..fec3aa588 100644
--- a/nerfstudio/generative/stable_diffusion.py
+++ b/nerfstudio/generative/stable_diffusion.py
@@ -19,7 +19,7 @@
import sys
from dataclasses import dataclass
from pathlib import Path
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Union
import appdirs
import mediapy
@@ -29,6 +29,8 @@ import torch.nn.functional as F
import tyro
from rich.console import Console
from torch import nn
+from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.cuda.amp.grad_scaler import GradScaler
from torchtyping import TensorType
CONSOLE = Console(width=120)
@@ -58,6 +60,26 @@ class UNet2DConditionOutput:
sample: torch.FloatTensor
+class _SDSGradient(torch.autograd.Function): # pylint: disable=abstract-method
+ """Custom gradient function for SDS loss. Since it is already computed, we can just return it."""
+
+ @staticmethod
+ @custom_fwd
+ def forward(ctx, input_tensor, gt_grad): # pylint: disable=arguments-differ
+ del input_tensor
+ ctx.save_for_backward(gt_grad)
+ # Return magniture of gradient, not the actual loss.
+ return torch.mean(gt_grad**2) ** 0.5
+
+ @staticmethod
+ @custom_bwd
+ def backward(ctx, grad): # pylint: disable=arguments-differ
+ del grad
+ (gt_grad,) = ctx.saved_tensors
+ batch_size = len(gt_grad)
+ return gt_grad / batch_size, None
+
+
class StableDiffusion(nn.Module):
"""Stable Diffusion implementation
Args:
@@ -162,14 +184,16 @@ class StableDiffusion(nn.Module):
text_embeddings: TensorType["N", "max_length", "embed_dim"],
image: TensorType["BS", 3, "H", "W"],
guidance_scale: float = 100.0,
- ) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]:
+ grad_scaler: Optional[GradScaler] = None,
+ ) -> torch.Tensor:
"""Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/)
Args:
text_embeddings: Text embeddings
image: Rendered image
guidance_scale: How much to weigh the guidance
+ grad_scaler: Grad scaler
Returns:
- A tuple of Loss, latent, and gradient
+ The loss
"""
image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear")
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
@@ -186,7 +210,7 @@ class StableDiffusion(nn.Module):
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
- noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+ noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# w(t), sigma_t^2
w = 1 - self.alphas[t]
@@ -194,9 +218,11 @@ class StableDiffusion(nn.Module):
grad = w * (noise_pred - noise)
grad = torch.nan_to_num(grad)
- sds_mse = torch.mean(torch.nan_to_num(torch.square(noise_pred - noise)))
+ if grad_scaler is not None:
+ latents = grad_scaler.scale(latents)
+ loss = _SDSGradient.apply(latents, grad)
- return sds_mse, latents, grad
+ return loss
def produce_latents(
self,
@@ -240,7 +266,7 @@ class StableDiffusion(nn.Module):
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
- noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+ noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore
diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py
index 353dd7db9..419a8a1f4 100644
--- a/nerfstudio/model_components/losses.py
+++ b/nerfstudio/model_components/losses.py
@@ -20,8 +20,10 @@ from enum import Enum
import torch
from torch import nn
from torchtyping import TensorType
+from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
+from nerfstudio.utils.math import masked_reduction
L1Loss = nn.L1Loss
MSELoss = nn.MSELoss
@@ -308,3 +310,218 @@ def depth_loss(
return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma)
raise NotImplementedError("Provided depth loss type not implemented.")
+
+
+def monosdf_normal_loss(
+ normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
+) -> TensorType[0]:
+ """
+ Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
+ Enforces consistency between the volume rendered normal and the predicted monocular normal.
+ With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf
+ Args:
+ normal_pred: volume rendered normal
+ normal_gt: monocular normal
+ """
+ normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1)
+ normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1)
+ l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean()
+ cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean()
+ return l1 + cos
+
+
+class MiDaSMSELoss(nn.Module):
+ """
+ data term from MiDaS paper
+ """
+
+ def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
+ super().__init__()
+
+ self.reduction_type = reduction_type
+ # reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
+ self.mse_loss = MSELoss(reduction="none")
+
+ def forward(
+ self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
+ ) -> TensorType[0]:
+ """
+ Args:
+ prediction: predicted depth map
+ target: ground truth depth map
+ mask: mask of valid pixels
+ Returns:
+ mse loss based on reduction function
+ """
+ summed_mask = torch.sum(mask, (1, 2))
+ image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
+ image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
+
+ return image_loss
+
+
+# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py
+class GradientLoss(nn.Module):
+ """
+ multiscale, scale-invariant gradient matching term to the disparity space.
+ This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
+ More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
+ """
+
+ def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
+ """
+ Args:
+ scales: number of scales to use
+ reduction_type: either "batch" or "image"
+ """
+ super().__init__()
+ self.reduction_type = reduction_type
+ self.__scales = scales
+
+ def forward(
+ self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
+ ) -> TensorType[0]:
+ """
+ Args:
+ prediction: predicted depth map
+ target: ground truth depth map
+ mask: mask of valid pixels
+ Returns:
+ gradient loss based on reduction function
+ """
+ total = 0
+
+ for scale in range(self.__scales):
+ step = pow(2, scale)
+
+ grad_loss = self.gradient_loss(
+ prediction[:, ::step, ::step],
+ target[:, ::step, ::step],
+ mask[:, ::step, ::step],
+ )
+ total += grad_loss
+
+ return total
+
+ def gradient_loss(
+ self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
+ ) -> TensorType[0]:
+ """
+ multiscale, scale-invariant gradient matching term to the disparity space.
+ This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
+ More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
+ Args:
+ prediction: predicted depth map
+ target: ground truth depth map
+ reduction: reduction function, either reduction_batch_based or reduction_image_based
+ Returns:
+ gradient loss based on reduction function
+ """
+ summed_mask = torch.sum(mask, (1, 2))
+ diff = prediction - target
+ diff = torch.mul(mask, diff)
+
+ grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1])
+ mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1])
+ grad_x = torch.mul(mask_x, grad_x)
+
+ grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :])
+ mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :])
+ grad_y = torch.mul(mask_y, grad_y)
+
+ image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2))
+ image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
+
+ return image_loss
+
+
+class ScaleAndShiftInvariantLoss(nn.Module):
+ """
+ Scale and shift invariant loss as described in
+ "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer"
+ https://arxiv.org/pdf/1907.01341.pdf
+ """
+
+ def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
+ """
+ Args:
+ alpha: weight of the regularization term
+ scales: number of scales to use
+ reduction_type: either "batch" or "image"
+ """
+ super().__init__()
+ self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type)
+ self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type)
+ self.__alpha = alpha
+
+ self.__prediction_ssi = None
+
+ @classmethod
+ def compute_scale_and_shift(
+ cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
+ ):
+ """
+ More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
+ This function computes scale/shift required to normalizes predicted depth map,
+ to allow for using normalized depth maps as input from monocular depth estimation networks.
+ These networks are trained such that they predict normalized depth maps.
+
+ Solves for scale/shift using a least squares approach with a closed form solution:
+ Based on:
+ https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
+ Args:
+ prediction: predicted depth map
+ target: ground truth depth map
+ mask: mask of valid pixels
+ Returns:
+ scale and shift for depth prediction
+ """
+ # system matrix: A = [[a_00, a_01], [a_10, a_11]]
+ a_00 = torch.sum(mask * prediction * prediction, (1, 2))
+ a_01 = torch.sum(mask * prediction, (1, 2))
+ a_11 = torch.sum(mask, (1, 2))
+
+ # right hand side: b = [b_0, b_1]
+ b_0 = torch.sum(mask * prediction * target, (1, 2))
+ b_1 = torch.sum(mask * target, (1, 2))
+
+ # solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
+ scale = torch.zeros_like(b_0)
+ shift = torch.zeros_like(b_1)
+
+ det = a_00 * a_11 - a_01 * a_01
+ valid = det.nonzero()
+
+ scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
+ shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
+
+ return scale, shift
+
+ def forward(
+ self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
+ ) -> TensorType[0]:
+ """
+ Args:
+ prediction: predicted depth map (unnormalized)
+ target: ground truth depth map (normalized)
+ mask: mask of valid pixels
+ Returns:
+ scale and shift invariant loss
+ """
+ scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
+ self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
+
+ total = self.__data_loss(self.__prediction_ssi, target, mask)
+ if self.__alpha > 0:
+ total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask)
+
+ return total
+
+ def __get_prediction_ssi(self):
+ """
+ scale and shift invariant prediction
+ from https://arxiv.org/pdf/1907.01341.pdf equation 1
+ """
+ return self.__prediction_ssi
+
+ prediction_ssi = property(__get_prediction_ssi)
diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py
index 5df52d2fa..ef0079ffd 100644
--- a/nerfstudio/model_components/ray_samplers.py
+++ b/nerfstudio/model_components/ray_samplers.py
@@ -509,7 +509,16 @@ class VolumetricSampler(Sampler):
class ProposalNetworkSampler(Sampler):
- """Sampler that uses a proposal network to generate samples."""
+ """Sampler that uses a proposal network to generate samples.
+
+ Args:
+ num_proposal_samples_per_ray: Number of samples to generate per ray for each proposal step.
+ num_nerf_samples_per_ray: Number of samples to generate per ray for the NERF model.
+ num_proposal_network_iterations: Number of proposal network iterations to run.
+ single_jitter: Use a same random jitter for all samples along a ray.
+ update_sched: A function that takes the iteration number of steps between updates.
+ initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set.
+ """
def __init__(
self,
@@ -518,6 +527,7 @@ class ProposalNetworkSampler(Sampler):
num_proposal_network_iterations: int = 2,
single_jitter: bool = False,
update_sched: Callable = lambda x: 1,
+ initial_sampler: Optional[Sampler] = None,
) -> None:
super().__init__()
self.num_proposal_samples_per_ray = num_proposal_samples_per_ray
@@ -528,7 +538,10 @@ class ProposalNetworkSampler(Sampler):
raise ValueError("num_proposal_network_iterations must be >= 1")
# samplers
- self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
+ if initial_sampler is None:
+ self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
+ else:
+ self.initial_sampler = initial_sampler
self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter)
self._anneal = 1.0
diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py
index 736cd0870..9c0579031 100644
--- a/nerfstudio/models/instant_ngp.py
+++ b/nerfstudio/models/instant_ngp.py
@@ -137,7 +137,7 @@ class NGPModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
diff --git a/nerfstudio/models/mipnerf.py b/nerfstudio/models/mipnerf.py
index b8c3bcc4f..31a337d02 100644
--- a/nerfstudio/models/mipnerf.py
+++ b/nerfstudio/models/mipnerf.py
@@ -86,7 +86,7 @@ class MipNerfModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
@@ -173,8 +173,8 @@ class MipNerfModel(Model):
image = torch.moveaxis(image, -1, 0)[None, ...]
rgb_coarse = torch.moveaxis(rgb_coarse, -1, 0)[None, ...]
rgb_fine = torch.moveaxis(rgb_fine, -1, 0)[None, ...]
- rgb_coarse = torch.clip(rgb_coarse, min=-1, max=1)
- rgb_fine = torch.clip(rgb_fine, min=-1, max=1)
+ rgb_coarse = torch.clip(rgb_coarse, min=0, max=1)
+ rgb_fine = torch.clip(rgb_fine, min=0, max=1)
coarse_psnr = self.psnr(image, rgb_coarse)
fine_psnr = self.psnr(image, rgb_fine)
diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py
index 7442bfe8b..4aa58d468 100644
--- a/nerfstudio/models/nerfacto.py
+++ b/nerfstudio/models/nerfacto.py
@@ -197,7 +197,7 @@ class NerfactoModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
diff --git a/nerfstudio/models/nerfplayer_nerfacto.py b/nerfstudio/models/nerfplayer_nerfacto.py
index f35a748b1..3811aba67 100644
--- a/nerfstudio/models/nerfplayer_nerfacto.py
+++ b/nerfstudio/models/nerfplayer_nerfacto.py
@@ -168,7 +168,7 @@ class NerfplayerNerfactoModel(NerfactoModel):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
self.temporal_distortion = True # for viewer
def get_outputs(self, ray_bundle: RayBundle):
diff --git a/nerfstudio/models/nerfplayer_ngp.py b/nerfstudio/models/nerfplayer_ngp.py
index 39507a538..878694468 100644
--- a/nerfstudio/models/nerfplayer_ngp.py
+++ b/nerfstudio/models/nerfplayer_ngp.py
@@ -145,7 +145,7 @@ class NerfplayerNGPModel(NGPModel):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
self.temporal_distortion = True # for viewer
def get_outputs(self, ray_bundle: RayBundle):
diff --git a/nerfstudio/models/semantic_nerfw.py b/nerfstudio/models/semantic_nerfw.py
index 941716dc8..22957d268 100644
--- a/nerfstudio/models/semantic_nerfw.py
+++ b/nerfstudio/models/semantic_nerfw.py
@@ -137,7 +137,7 @@ class SemanticNerfWModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py
index 60fec9706..d5af1579d 100644
--- a/nerfstudio/models/tensorf.py
+++ b/nerfstudio/models/tensorf.py
@@ -201,7 +201,7 @@ class TensoRFModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
# colliders
if self.config.enable_collider:
diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py
index 3e731524f..53e838891 100644
--- a/nerfstudio/models/vanilla_nerf.py
+++ b/nerfstudio/models/vanilla_nerf.py
@@ -118,7 +118,7 @@ class NeRFModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
- self.lpips = LearnedPerceptualImagePatchSimilarity()
+ self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
if getattr(self.config, "enable_temporal_distortion", False):
params = self.config.temporal_distortion_params
diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py
index cee6108a9..b0a0aaec4 100644
--- a/nerfstudio/utils/math.py
+++ b/nerfstudio/utils/math.py
@@ -19,6 +19,7 @@ from typing import Tuple
import torch
from torchtyping import TensorType
+from typing_extensions import Literal
_USE_NERFACC = True
@@ -285,3 +286,28 @@ def safe_normalize(
Normalized vectors.
"""
return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps)
+
+
+def masked_reduction(
+ input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
+):
+ """
+ Whether to consolidate the input_tensor across the batch or across the image
+ Args:
+ input_tensor: input tensor
+ mask: mask tensor
+ reduction_type: either "batch" or "image"
+ Returns:
+ input_tensor: reduced input_tensor
+ """
+ if reduction_type == "batch":
+ # avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0)
+ divisor = torch.sum(mask)
+ input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor
+ elif reduction_type == "image":
+ # avoid division by 0 (if M = sum(mask) = 0: image_loss = 0)
+ valid = mask.nonzero()
+
+ input_tensor[valid] = input_tensor[valid] / mask[valid]
+ input_tensor = torch.mean(input_tensor)
+ return input_tensor
diff --git a/pyproject.toml b/pyproject.toml
index ba57c01d4..8901aa282 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "nerfstudio"
-version = "0.1.16"
+version = "0.1.17"
description = "All-in-one repository for state-of-the-art NeRFs"
readme = "README.md"
license = { text="Apache 2.0"}
@@ -22,7 +22,7 @@ dependencies = [
"tyro>=0.3.31",
"gdown>=4.6.0",
"ninja>=1.10",
- "functorch==0.2.1",
+ "functorch>=0.2.1",
"h5py>=2.9.0",
"imageio>=2.21.1",
'importlib-metadata>=6.0.0; python_version < "3.10"',
@@ -32,7 +32,7 @@ dependencies = [
"mediapy>=1.1.0",
"msgpack>=1.0.4",
"msgpack_numpy>=0.4.8",
- "nerfacc==0.3.4",
+ "nerfacc==0.3.3",
"open3d>=0.16.0",
"opencv-python==4.6.0.66",
"Pillow>=9.3.0",
@@ -45,7 +45,7 @@ dependencies = [
"rich>=12.5.1",
"scikit-image>=0.19.3",
"tensorboard==2.9.0",
- "torch==1.12.1",
+ "torch>=1.12.1",
"torchmetrics[image]>=0.9.3",
"torchtyping>=0.1.4",
"torchvision>=0.13.0",
@@ -100,6 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint"
ns-process-data = "scripts.process_data:entrypoint"
ns-download-data = "scripts.downloads.download_data:entrypoint"
ns-train = "scripts.train:entrypoint"
+ns-viewer = "scripts.viewer.run_viewer:entrypoint"
ns-eval = "scripts.eval:entrypoint"
ns-render = "scripts.render:entrypoint"
ns-export = "scripts.exporter:entrypoint"
diff --git a/scripts/completions/install.py b/scripts/completions/install.py
index 636ed1979..98e67fea2 100755
--- a/scripts/completions/install.py
+++ b/scripts/completions/install.py
@@ -10,27 +10,29 @@ import shutil
import stat
import subprocess
import sys
-from typing import List, Union
+from typing import List, Optional, Union
import tyro
from rich.console import Console
from rich.prompt import Confirm
from typing_extensions import Literal, assert_never
+from typing_extensions import get_args as typing_get_args
+
+if sys.version_info < (3, 10):
+ import importlib_metadata
+else:
+ from importlib import metadata as importlib_metadata
ConfigureMode = Literal["install", "uninstall"]
ShellType = Literal["zsh", "bash"]
CONSOLE = Console(width=120)
+HEADER_LINE = "# Source nerfstudio autocompletions."
-ENTRYPOINTS = [
- "ns-install-cli",
- "ns-process-data",
- "ns-download-data",
- "ns-train",
- "ns-eval",
- "ns-render",
- "ns-dev-test",
-]
+
+def _get_all_entry_points() -> List[str]:
+ entry_points = importlib_metadata.distribution("nerfstudio").entry_points
+ return [x.name for x in entry_points]
def _check_tyro_cli(script_path: pathlib.Path) -> bool:
@@ -127,6 +129,62 @@ def _exclamation() -> str:
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
+def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str:
+ if shell is None:
+ # Install the universal script
+ result_script = []
+ for shell_type in typing_get_args(ShellType):
+ result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
+ result_script.append(_get_deactivate_script(commands, shell_type, add_header=False))
+ result_script.append("fi")
+ source_lines = "\n".join(result_script)
+
+ elif shell == "zsh":
+ source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands])
+ elif shell == "bash":
+ source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands])
+ else:
+ assert_never(shell)
+
+ if add_header:
+ source_lines = f"\n{HEADER_LINE}\n{source_lines}"
+ return source_lines
+
+
+def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str:
+ if shell is None:
+ # Install the universal script
+ result_script = []
+ for shell_type in typing_get_args(ShellType):
+ result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
+ result_script.append(_get_source_script(completions_dir, shell_type, add_header=False))
+ result_script.append("fi")
+ source_lines = "\n".join(result_script)
+
+ elif shell == "zsh":
+ source_lines = "\n".join(
+ [
+ "if ! command -v compdef &> /dev/null; then",
+ " autoload -Uz compinit",
+ " compinit",
+ "fi",
+ f"source {completions_dir / 'setup.zsh'}",
+ ]
+ )
+ elif shell == "bash":
+ source_lines = "\n".join(
+ [
+ f"source {completions_dir / 'setup.bash'}",
+ ]
+ )
+ else:
+ assert_never(shell)
+
+ if add_header:
+ source_lines = f"\n{HEADER_LINE}\n{source_lines}"
+ return source_lines
+
+
def _update_rc(
completions_dir: pathlib.Path,
mode: ConfigureMode,
@@ -141,36 +199,13 @@ def _update_rc(
"""
# Install or uninstall `source_line`.
- header_line = "# Source nerfstudio autocompletions."
- if shell == "zsh":
- source_lines = "\n".join(
- [
- "",
- header_line,
- "if ! command -v compdef &> /dev/null; then",
- " autoload -Uz compinit",
- " compinit",
- "fi",
- f"source {completions_dir / 'setup.zsh'}",
- ]
- )
- elif shell == "bash":
- source_lines = "\n".join(
- [
- "",
- header_line,
- f"source {completions_dir / 'setup.bash'}",
- ]
- )
- else:
- assert_never(shell)
-
+ source_lines = _get_source_script(completions_dir, shell)
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
# Always try to uninstall previous completions.
rc_source = rc_path.read_text()
- while header_line in rc_source:
- before_install, _, after_install = rc_source.partition(header_line)
+ while HEADER_LINE in rc_source:
+ before_install, _, after_install = rc_source.partition(HEADER_LINE)
source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n")
assert source_file.endswith(f"/completions/setup.{shell}")
rc_source = before_install + after_install
@@ -189,6 +224,108 @@ def _update_rc(
assert mode == "uninstall"
+def _update_conda_scripts(
+ commands: List[str],
+ completions_dir: pathlib.Path,
+ mode: ConfigureMode,
+) -> None:
+ """Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts.
+
+ Args:
+ completions_dir: Path to location of this script.
+ mode: Install or uninstall completions.
+ """
+
+ # Install or uninstall `source_line`.
+ activate_source_lines = _get_source_script(completions_dir, None)
+ deactivate_source_lines = _get_deactivate_script(commands, None)
+
+ conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
+ activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh"
+ deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh"
+ if mode == "uninstall":
+ if activate_path.exists():
+ os.remove(activate_path)
+ if deactivate_path.exists():
+ os.remove(deactivate_path)
+ CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.")
+ elif mode == "install":
+ # Install completions.
+ activate_path.parent.mkdir(exist_ok=True, parents=True)
+ deactivate_path.parent.mkdir(exist_ok=True, parents=True)
+ with activate_path.open("w+", encoding="utf8") as f:
+ f.write(activate_source_lines)
+ with deactivate_path.open("w+", encoding="utf8") as f:
+ f.write(deactivate_source_lines)
+ CONSOLE.log(
+ f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment"
+ " to try them out."
+ )
+ else:
+ assert_never(mode)
+
+
+def _get_conda_path() -> Optional[pathlib.Path]:
+ """
+ Returns the path to the conda environment if
+ the nerfstudio package is installed in one.
+ """
+ conda_path = None
+ if "CONDA_PREFIX" in os.environ:
+ # Conda is active, we will check if the Nerfstudio is installed in the conda env.
+ distribution = importlib_metadata.distribution("nerfstudio")
+ if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]):
+ conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
+ return conda_path
+
+
+def _generate_completions_files(
+ completions_dir: pathlib.Path,
+ scripts_dir: pathlib.Path,
+ shells_supported: List[ShellType],
+ shells_found: List[ShellType],
+) -> None:
+ # Set to True to install completions for scripts as well.
+ include_scripts = False
+
+ # Find tyro CLIs.
+ script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
+ script_names = tuple(p.name for p in script_paths)
+ assert len(set(script_names)) == len(script_names)
+
+ # Get existing completion files.
+ existing_completions = set()
+ for shell in shells_supported:
+ target_dir = completions_dir / shell
+ if target_dir.exists():
+ existing_completions |= set(target_dir.glob("*"))
+
+ # Get all entry_points.
+ entry_points = _get_all_entry_points()
+
+ # Run generation jobs.
+ concurrent_executor = concurrent.futures.ThreadPoolExecutor()
+ with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
+ completion_paths = list(
+ concurrent_executor.map(
+ lambda path_or_entrypoint_and_shell: _generate_completion(
+ path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
+ ),
+ itertools.product(script_paths + entry_points, shells_found),
+ )
+ )
+
+ # Delete obsolete completion files.
+ for unexpected_path in set(p.absolute() for p in existing_completions) - set(
+ p.absolute() for p in completion_paths
+ ):
+ if unexpected_path.is_dir():
+ shutil.rmtree(unexpected_path)
+ elif unexpected_path.exists():
+ unexpected_path.unlink()
+ CONSOLE.log(f":broom: Deleted {unexpected_path}.")
+
+
def main(mode: ConfigureMode = "install") -> None:
"""Main script.
@@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None:
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
return
+ # Get conda path if in conda environment.
+ conda_path = _get_conda_path()
+
# Try to locate the user's bashrc or zshrc.
- shells_supported: List[ShellType] = ["zsh", "bash"]
- shells_found: List[ShellType] = []
- for shell in shells_supported:
- rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
- if not rc_path.exists():
- CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
- else:
- CONSOLE.log(f":mag: Found {rc_path.name}!")
- shells_found.append(shell)
+ shells_supported: List[ShellType] = list(typing_get_args(ShellType))
+ if conda_path is not None:
+ # Running in conda; we have to support all shells.
+ shells_found = shells_supported
+ CONSOLE.log(f":mag: Detected conda environment {conda_path}!")
+ else:
+ shells_found: List[ShellType] = []
+ for shell in shells_supported:
+ rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
+ if not rc_path.exists():
+ CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
+ else:
+ CONSOLE.log(f":mag: Found {rc_path.name}!")
+ shells_found.append(shell)
# Get scripts/ directory.
completions_dir = pathlib.Path(__file__).absolute().parent
@@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None:
else:
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
elif mode == "install":
- # Set to True to install completions for scripts as well.
- include_scripts = False
-
- # Find tyro CLIs.
- script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
- script_names = tuple(p.name for p in script_paths)
- assert len(set(script_names)) == len(script_names)
-
- # Get existing completion files.
- existing_completions = set()
- for shell in shells_supported:
- target_dir = completions_dir / shell
- if target_dir.exists():
- existing_completions |= set(target_dir.glob("*"))
-
- # Run generation jobs.
- concurrent_executor = concurrent.futures.ThreadPoolExecutor()
- with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
- completion_paths = list(
- concurrent_executor.map(
- lambda path_or_entrypoint_and_shell: _generate_completion(
- path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
- ),
- itertools.product(script_paths + ENTRYPOINTS, shells_found),
- )
- )
-
- # Delete obsolete completion files.
- for unexpected_path in set(p.absolute() for p in existing_completions) - set(
- p.absolute() for p in completion_paths
- ):
- if unexpected_path.is_dir():
- shutil.rmtree(unexpected_path)
- elif unexpected_path.exists():
- unexpected_path.unlink()
- CONSOLE.log(f":broom: Deleted {unexpected_path}.")
+ _generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found)
else:
assert_never(mode)
- # Install or uninstall from bashrc/zshrc.
- for shell in shells_found:
- _update_rc(completions_dir, mode, shell)
+ if conda_path is not None:
+ # In conda environment we add the completitions activation scripts.
+ commands = _get_all_entry_points()
+ _update_conda_scripts(commands, completions_dir, mode)
+ else:
+ # Install or uninstall from bashrc/zshrc.
+ for shell in shells_found:
+ _update_rc(completions_dir, mode, shell)
CONSOLE.print("[bold]All done![/bold]")
diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py
index 89c77391f..b6775a339 100755
--- a/scripts/downloads/download_data.py
+++ b/scripts/downloads/download_data.py
@@ -287,6 +287,75 @@ class PhototourismDownload(DatasetDownload):
os.remove(download_path)
+# credit to https://autonomousvision.github.io/sdfstudio/
+# pylint: disable=line-too-long
+sdfstudio_downloads = {
+ "sdfstudio-demo-data": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/sdfstudio-demo-data.tar",
+ "dtu": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/DTU.tar",
+ "replica": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Replica.tar",
+ "scannet": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/scannet.tar",
+ "tanks-and-temple": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/tnt_advanced.tar",
+ "tanks-and-temple-highres": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/highresTNT.tar",
+ "heritage": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Heritage-Recon.tar",
+ "neural-rgbd-data": "http://kaldir.vc.in.tum.de/neural_rgbd/neural_rgbd_data.zip",
+ "all": None,
+}
+
+SDFstudioCaptureName = tyro.extras.literal_type_from_choices(sdfstudio_downloads.keys())
+
+
+@dataclass
+class SDFstudioDemoDownload(DatasetDownload):
+ """Download the sdfstudio dataset."""
+
+ dataset_name: SDFstudioCaptureName = "sdfstudio-demo-data"
+
+ def download(self, save_dir: Path):
+ """Download the D-NeRF dataset (https://github.com/albertpumarola/D-NeRF)."""
+ # TODO: give this code the same structure as download_nerfstudio
+
+ if self.dataset_name == "all":
+ for dataset_name in sdfstudio_downloads:
+ if dataset_name != "all":
+ SDFstudioDemoDownload(dataset_name=dataset_name).download(save_dir)
+ return
+
+ assert (
+ self.dataset_name in sdfstudio_downloads
+ ), f"Capture name {self.dataset_name} not found in {sdfstudio_downloads.keys()}"
+
+ url = sdfstudio_downloads[self.dataset_name]
+
+ target_path = str(save_dir / self.dataset_name)
+ os.makedirs(target_path, exist_ok=True)
+
+ file_format = url[-4:]
+
+ download_path = Path(f"{target_path}{file_format}")
+ tmp_path = str(save_dir / ".temp")
+ shutil.rmtree(tmp_path, ignore_errors=True)
+ os.makedirs(tmp_path, exist_ok=True)
+
+ os.system(f"curl -L {url} > {download_path}")
+ if file_format == ".tar":
+ with tarfile.open(download_path, "r") as tar_ref:
+ tar_ref.extractall(str(tmp_path))
+ elif file_format == ".zip":
+ with zipfile.ZipFile(download_path, "r") as zip_ref:
+ zip_ref.extractall(str(target_path))
+ return
+ else:
+ raise NotImplementedError
+
+ inner_folders = os.listdir(tmp_path)
+ assert len(inner_folders) == 1, "There is more than one folder inside this zip file."
+ folder = os.path.join(tmp_path, inner_folders[0])
+ shutil.rmtree(target_path)
+ shutil.move(folder, target_path)
+ shutil.rmtree(tmp_path)
+ os.remove(download_path)
+
+
Commands = Union[
Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")],
Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")],
@@ -294,6 +363,7 @@ Commands = Union[
Annotated[Record3dDownload, tyro.conf.subcommand(name="record3d")],
Annotated[DNerfDownload, tyro.conf.subcommand(name="dnerf")],
Annotated[PhototourismDownload, tyro.conf.subcommand(name="phototourism")],
+ Annotated[SDFstudioDemoDownload, tyro.conf.subcommand(name="sdfstudio")],
]
diff --git a/scripts/process_data.py b/scripts/process_data.py
index f6f9a9a7f..752e1d15c 100755
--- a/scripts/process_data.py
+++ b/scripts/process_data.py
@@ -737,7 +737,7 @@ class ProcessRealityCapture:
"""Process RealityCapture data into a nerfstudio dataset.
This script assumes that cameras have been aligned using RealityCapture. After alignment, it is necessary to
- export the camera poses as a `.csv` file.
+ export the camera poses as a `.csv` file using the `Internal/External camera parameters` option.
This script does the following:
diff --git a/scripts/viewer/__init__.py b/scripts/viewer/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scripts/viewer/run_viewer.py b/scripts/viewer/run_viewer.py
new file mode 100644
index 000000000..9a96c6703
--- /dev/null
+++ b/scripts/viewer/run_viewer.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+"""
+Starts viewer in eval mode.
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+
+import tyro
+from rich.console import Console
+
+from nerfstudio.configs.base_config import ViewerConfig
+from nerfstudio.engine.trainer import TrainerConfig
+from nerfstudio.pipelines.base_pipeline import Pipeline
+from nerfstudio.utils import writer
+from nerfstudio.utils.eval_utils import eval_setup
+from nerfstudio.utils.writer import EventName, TimeWriter
+from nerfstudio.viewer.server import viewer_utils
+
+CONSOLE = Console(width=120)
+
+
+@dataclass
+class ViewerConfigWithoutNumRays(ViewerConfig):
+ """Configuration for viewer instantiation"""
+
+ num_rays_per_chunk: tyro.conf.Suppress[int] = -1
+ start_train: tyro.conf.Suppress[bool] = False
+
+ def as_viewer_config(self):
+ """Converts the instance to ViewerConfig"""
+ return ViewerConfig(**{x.name: getattr(self, x.name) for x in fields(self)})
+
+
+@dataclass
+class RunViewer:
+ """Load a checkpoint and start the viewer."""
+
+ load_config: Path
+ """Path to config YAML file."""
+ viewer: ViewerConfigWithoutNumRays = field(default_factory=ViewerConfigWithoutNumRays)
+ """Viewer configuration"""
+
+ def main(self) -> None:
+ """Main function."""
+ config, pipeline, _ = eval_setup(
+ self.load_config,
+ eval_num_rays_per_chunk=None,
+ test_mode="test",
+ )
+ num_rays_per_chunk = config.viewer.num_rays_per_chunk
+ assert self.viewer.num_rays_per_chunk == -1
+ config.vis = "viewer"
+ config.viewer = self.viewer.as_viewer_config()
+ config.viewer.num_rays_per_chunk = num_rays_per_chunk
+
+ self._start_viewer(config, pipeline)
+
+ def _start_viewer(self, config: TrainerConfig, pipeline):
+ base_dir = config.get_base_dir()
+ viewer_log_path = base_dir / config.viewer.relative_log_filename
+ viewer_state, banner_messages = viewer_utils.setup_viewer(
+ config.viewer, log_filename=viewer_log_path, datapath=config.pipeline.datamanager.dataparser.data
+ )
+
+ # We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
+ config.logging.local_writer.enable = False
+ writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages)
+
+ assert viewer_state and pipeline.datamanager.train_dataset
+ viewer_state.init_scene(
+ dataset=pipeline.datamanager.train_dataset,
+ start_train=False,
+ )
+ while True:
+ viewer_state.vis["renderingState/isTraining"].write(False)
+ self._update_viewer_state(viewer_state, config, pipeline)
+
+ def _update_viewer_state(self, viewer_state: viewer_utils.ViewerState, config: TrainerConfig, pipeline: Pipeline):
+ """Updates the viewer state by rendering out scene with current pipeline
+ Returns the time taken to render scene.
+
+ """
+ # NOTE: step must be > 0 otherwise the rendering would not happen
+ step = 1
+ num_rays_per_batch = config.pipeline.datamanager.train_num_rays_per_batch
+ with TimeWriter(writer, EventName.ITER_VIS_TIME) as _:
+ try:
+ viewer_state.update_scene(self, step, pipeline.model, num_rays_per_batch)
+ except RuntimeError:
+ time.sleep(0.03) # sleep to allow buffer to reset
+ assert viewer_state.vis is not None
+ viewer_state.vis["renderingState/log_errors"].write(
+ "Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
+ )
+
+ def save_checkpoint(self, *args, **kwargs):
+ """
+ Mock method because we pass this instance to viewer_state.update_scene
+ """
+
+
+def entrypoint():
+ """Entrypoint for use with pyproject scripts."""
+ tyro.extras.set_accent_color("bright_yellow")
+ tyro.cli(RunViewer).main()
+
+
+if __name__ == "__main__":
+ entrypoint()
+
+# For sphinx docs
+get_parser_fn = lambda: tyro.extras.get_parser(RunViewer) # noqa
diff --git a/scripts/viewer/view_dataset.py b/scripts/viewer/view_dataset.py
index 1f5aa2201..4bd526552 100644
--- a/scripts/viewer/view_dataset.py
+++ b/scripts/viewer/view_dataset.py
@@ -3,22 +3,21 @@
view_dataset.py
"""
-from rich.console import Console
-
-CONSOLE = Console(width=120)
import time
from datetime import timedelta
from pathlib import Path
import torch
import tyro
+from rich.console import Console
from nerfstudio.configs.base_config import ViewerConfig
-from nerfstudio.data.datamanagers import AnnotatedDataParserUnion
+from nerfstudio.data.datamanagers.base_datamanager import AnnotatedDataParserUnion
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.viewer.server import viewer_utils
DEFAULT_TIMEOUT = timedelta(minutes=30)
+CONSOLE = Console(width=120)
# speedup for when input size to model doesn't change (much)
torch.backends.cudnn.benchmark = True # type: ignore
@@ -30,9 +29,10 @@ def main(
log_base_dir: Path = Path("/tmp/nerfstudio_viewer_logs"),
) -> None:
"""Main function."""
- viewer_state = viewer_utils.ViewerState(
+ viewer_state, _ = viewer_utils.setup_viewer(
viewer,
log_filename=log_base_dir / viewer.relative_log_filename,
+ datapath=dataparser.data,
)
dataset = InputDataset(dataparser.setup().get_dataparser_outputs(split="train"))
viewer_state.init_scene(dataset=dataset, start_train=False)