diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..84f99bf49 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,13 @@ +{ + "name": "NerfStudio Dev Container", + "dockerComposeFile": "docker-compose.yml", + "service": "nerfstudio", + "workspaceFolder": "/workspace", + "shutdownAction": "stopCompose", + "extensions": [ + "mutantdino.resourcemonitor", + "ms-azuretools.vscode-docker", + "nvidia.nsight-vscode-edition", + "ms-python.python" + ] +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..6376422cf --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.8" + +services: + nerfstudio: + shm_size: '12gb' + image: dromni/nerfstudio:0.1.17 + # build: + # context: .. + # dockerfile: ./Dockerfile + stdin_open: true + tty: true + environment: + NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics + DISPLAY: $DISPLAY + ports: + - "7007:7007" + volumes: + - ../:/workspace/ + - /tmp/.X11-unix:/tmp/.X11-unix + working_dir: /workspace/ + command: /bin/bash + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [ gpu ] \ No newline at end of file diff --git a/README.md b/README.md index ca2bd48b4..0b0d8592c 100644 --- a/README.md +++ b/README.md @@ -95,13 +95,25 @@ python -m pip install --upgrade pip ### Dependencies -Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) +Install pytorch with CUDA (this repo has been tested with CUDA 11.3 and CUDA 11.7) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) + +For CUDA 11.3: ```bash pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch ``` +For CUDA 11.7: + +```bash +pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117 +pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch +``` + +See [Dependencies](https://github.com/nerfstudio-project/nerfstudio/blob/main/docs/quickstart/installation.md#dependencies) +in the Installation documentation for more. + ### Installing nerfstudio Easy option: @@ -290,16 +302,18 @@ We provide the following support structures to make life easier for getting star # Citation +You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264). + If you use this library or find the documentation useful for your research, please consider citing: ``` -@misc{nerfstudio, - title={Nerfstudio: A Framework for Neural Radiance Field Development}, - author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi, - Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi, - Abhik Ahuja, David McAllister, Angjoo Kanazawa}, - year={2022}, - url={https://github.com/nerfstudio-project/nerfstudio}, +@article{nerfstudio, + author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, + Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, + Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo}, + title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, + journal = {arXiv preprint arXiv:2302.04264}, + year = {2023}, } ``` diff --git a/colab/demo.ipynb b/colab/demo.ipynb index e92f8b04d..d1c58ad6c 100644 --- a/colab/demo.ipynb +++ b/colab/demo.ipynb @@ -292,9 +292,9 @@ "if os.path.exists(f\"data/nerfstudio/{scene}/transforms.json\"):\n", " !ns-train nerfacto --viewer.websocket-port 7007 nerfstudio-data --data data/nerfstudio/$scene --downscale-factor 4\n", "else:\n", - " import IPython\n", - " display(IPython.display.HTML('

Error: Data processing did not complete

'))\n", - " display(IPython.display.HTML('

Please re-run `Downloading and Processing Data`, or view the FAQ for more info.

'))" + " from IPython.core.display import display, HTML\n", + " display(HTML('

Error: Data processing did not complete

'))\n", + " display(HTML('

Please re-run `Downloading and Processing Data`, or view the FAQ for more info.

'))" ] }, { @@ -383,4 +383,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/docs/developer_guides/pipelines/datamanagers.md b/docs/developer_guides/pipelines/datamanagers.md index bbe3413d9..32c55b63e 100644 --- a/docs/developer_guides/pipelines/datamanagers.md +++ b/docs/developer_guides/pipelines/datamanagers.md @@ -87,7 +87,7 @@ def next_train(self, step: int) -> Tuple[RayBundle, Dict]: You can see our code for more details. -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/datamanagers.py +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/data/datamanagers/base_datamanager.py :color: primary :outline: See the code! diff --git a/docs/index.md b/docs/index.md index 53d45784d..88e4c90ae 100644 --- a/docs/index.md +++ b/docs/index.md @@ -155,16 +155,18 @@ We'll be constantly growing this list! So make sure to check back in to see our ## Citation +You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264). + If you use this library or find the documentation useful for your research, please consider citing: ```none -@misc{nerfstudio, - title={Nerfstudio: A Framework for Neural Radiance Field Development}, - author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi, - Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi, - Abhik Ahuja, David McAllister, Angjoo Kanazawa}, - year={2022}, - url={https://github.com/nerfstudio-project/nerfstudio}, +@article{nerfstudio, + author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, + Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, + Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo}, + title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, + journal = {arXiv preprint arXiv:2302.04264}, + year = {2023}, } ``` diff --git a/docs/quickstart/first_nerf.md b/docs/quickstart/first_nerf.md index e7071c329..73e40a9d8 100644 --- a/docs/quickstart/first_nerf.md +++ b/docs/quickstart/first_nerf.md @@ -30,7 +30,7 @@ Navigating to the link at the end of the terminal will load the webviewer. If yo - All data configurations must go at the end. In this case, `nerfstudio-data` and all of its corresponding configurations come at the end after the model and viewer specification. ::: -## Resume from checkpoint / visualize existing run +## Resume from checkpoint It is possible to load a pretrained model by running @@ -40,6 +40,14 @@ ns-train nerfacto --data data/nerfstudio/poster --load-dir {outputs/.../nerfstud This will automatically start training. If you do not want it to train, add `--viewer.start-train False` to your training command. +## Visualize existing run + +Given a pretrained model checkpoint, you can start the viewer by running + +```bash +ns-viewer --load-config {outputs/.../config.yml} +``` + ## Exporting Results Once you have a NeRF model you can either render out a video or export a point cloud. diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index 97c63aac8..20ab8859c 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -17,12 +17,38 @@ python -m pip install --upgrade pip ## Dependencies -Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) +(pytorch)= +### pytorch + +Either pytorch 1.12.1 (with CUDA up to 11.3) or 1.13.1 (with CUDA 11.6/7/8) can be used. + +- To install 1.12.1 with CUDA 11.3: ```bash pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html -pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch +``` +- To install 1.13.1 with CUDA 11.7: + +Note that if a pytorch version prior to 1.13 is installed, +it should be uninstalled first to avoid upgrade issues (e.g. with functorch) + +```bash +pip uninstall torch torchvision functorch +``` + +Install pytorch 1.13.1 with CUDA (this repo has been tested with CUDA 11.7.1) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) + +```bash +pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117 +``` + +### tinycudann + +After pytorch, install the torch bindings for tinycudann: + +```bash +pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch ``` ## Installing nerfstudio @@ -134,9 +160,8 @@ While installing tiny-cuda, you run into: `The detected CUDA version mismatches **Solution**: -``` -pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html -``` +Reinstall pytorch with the correct CUDA version. +See [pytorch](pytorch) under Dependencies, above.
diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index 61c489edf..a43504694 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -100,12 +100,16 @@ def get_spiral_path( new_c2ws.append(c2wh[:3, :4]) new_c2ws = torch.stack(new_c2ws, dim=0) + times = None + if camera.times is not None: + times = torch.linspace(0, 1, steps)[:, None] return Cameras( fx=camera.fx[0], fy=camera.fy[0], cx=camera.cx[0], cy=camera.cy[0], camera_to_worlds=new_c2ws, + times=times, ) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index d7d228bae..80b7f6487 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -230,7 +230,7 @@ class Cameras(TensorDataclass): c_x_y: cx or cy for when h_w == None """ if isinstance(h_w, int): - h_w = torch.Tensor([h_w]).to(torch.int64).to(self.device) + h_w = torch.as_tensor([h_w]).to(torch.int64).to(self.device) elif isinstance(h_w, torch.Tensor): assert not torch.is_floating_point(h_w), f"height and width tensor must be of type int, not: {h_w.dtype}" h_w = h_w.to(torch.int64).to(self.device) @@ -238,7 +238,7 @@ class Cameras(TensorDataclass): h_w = h_w.unsqueeze(-1) # assert torch.all(h_w == h_w.view(-1)[0]), "Batched cameras of different h, w will be allowed in the future." elif h_w is None: - h_w = torch.Tensor((c_x_y * 2).to(torch.int64).to(self.device)) + h_w = torch.as_tensor((c_x_y * 2)).to(torch.int64).to(self.device) else: raise ValueError("Height must be an int, tensor, or None, received: " + str(type(h_w))) return h_w diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index d032c4d48..1c8c3e08d 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -49,6 +49,7 @@ from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserCo from nerfstudio.data.dataparsers.phototourism_dataparser import ( PhototourismDataParserConfig, ) +from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler from nerfstudio.data.utils.dataloaders import ( @@ -75,6 +76,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of "dnerf-data": DNeRFDataParserConfig(), "phototourism-data": PhototourismDataParserConfig(), "dycheck-data": DycheckDataParserConfig(), + "sdfstudio-data": SDFStudioDataParserConfig(), }, prefix_names=False, # Omit prefixes in subcommands themselves. ) diff --git a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py index d09aa86f9..2fdc2df85 100644 --- a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py +++ b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py @@ -123,6 +123,10 @@ class InstantNGP(DataParser): w, h = meta["w"], meta["h"] + camera_type = CameraType.PERSPECTIVE + if meta.get("is_fisheye", False): + camera_type = CameraType.FISHEYE + cameras = Cameras( fx=float(fl_x), fy=float(fl_y), @@ -132,7 +136,7 @@ class InstantNGP(DataParser): height=int(h), width=int(w), camera_to_worlds=camera_to_world, - camera_type=CameraType.PERSPECTIVE, + camera_type=camera_type, ) # TODO(ethan): add alpha background color diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py new file mode 100644 index 000000000..f25c8f3e7 --- /dev/null +++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py @@ -0,0 +1,158 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Datapaser for sdfstudio formatted data""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Type + +import torch +from rich.console import Console + +from nerfstudio.cameras import camera_utils +from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.data.dataparsers.base_dataparser import ( + DataParser, + DataParserConfig, + DataparserOutputs, +) +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.utils.io import load_from_json + +CONSOLE = Console() + + +@dataclass +class SDFStudioDataParserConfig(DataParserConfig): + """Scene dataset parser config""" + + _target: Type = field(default_factory=lambda: SDFStudio) + """target class to instantiate""" + data: Path = Path("data/DTU/scan65") + """Directory specifying location of data.""" + include_mono_prior: bool = False + """whether or not to load monocular depth and normal """ + include_foreground_mask: bool = False + """whether or not to load foreground mask""" + downscale_factor: int = 1 + scene_scale: float = 2.0 + """ + Sets the bounding cube to have edge length of this size. + The longest dimension of the Friends axis-aligned bbox will be scaled to this value. + """ + skip_every_for_val_split: int = 1 + """sub sampling validation images""" + auto_orient: bool = False + + +@dataclass +class SDFStudio(DataParser): + """SDFStudio Dataset""" + + config: SDFStudioDataParserConfig + + def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements + # load meta data + meta = load_from_json(self.config.data / "meta_data.json") + + indices = list(range(len(meta["frames"]))) + # subsample to avoid out-of-memory for validation set + if split != "train" and self.config.skip_every_for_val_split >= 1: + indices = indices[:: self.config.skip_every_for_val_split] + + image_filenames = [] + depth_filenames = [] + normal_filenames = [] + transform = None + fx = [] + fy = [] + cx = [] + cy = [] + camera_to_worlds = [] + for i, frame in enumerate(meta["frames"]): + if i not in indices: + continue + + image_filename = self.config.data / frame["rgb_path"] + depth_filename = self.config.data / frame["mono_depth_path"] + normal_filename = self.config.data / frame["mono_normal_path"] + + intrinsics = torch.tensor(frame["intrinsics"]) + camtoworld = torch.tensor(frame["camtoworld"]) + + # append data + image_filenames.append(image_filename) + depth_filenames.append(depth_filename) + normal_filenames.append(normal_filename) + fx.append(intrinsics[0, 0]) + fy.append(intrinsics[1, 1]) + cx.append(intrinsics[0, 2]) + cy.append(intrinsics[1, 2]) + camera_to_worlds.append(camtoworld) + + fx = torch.stack(fx) + fy = torch.stack(fy) + cx = torch.stack(cx) + cy = torch.stack(cy) + camera_to_worlds = torch.stack(camera_to_worlds) + + # Convert from COLMAP's/OPENCV's camera coordinate system to nerfstudio + camera_to_worlds[:, 0:3, 1:3] *= -1 + + if self.config.auto_orient: + camera_to_worlds, transform = camera_utils.auto_orient_and_center_poses( + camera_to_worlds, + method="up", + center_poses=False, + ) + + # scene box from meta data + meta_scene_box = meta["scene_box"] + aabb = torch.tensor(meta_scene_box["aabb"], dtype=torch.float32) + scene_box = SceneBox( + aabb=aabb, + ) + + height, width = meta["height"], meta["width"] + cameras = Cameras( + fx=fx, + fy=fy, + cx=cx, + cy=cy, + height=height, + width=width, + camera_to_worlds=camera_to_worlds[:, :3, :4], + camera_type=CameraType.PERSPECTIVE, + ) + + # TODO supports downsample + # cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor) + + assert meta["has_mono_prior"] == self.config.include_mono_prior, f"no mono prior in {self.config.data}" + + dataparser_outputs = DataparserOutputs( + image_filenames=image_filenames, + cameras=cameras, + scene_box=scene_box, + metadata={ + "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, + "normal_filenames": normal_filenames if len(normal_filenames) > 0 else None, + "transform": transform, + "camera_to_worlds": camera_to_worlds if len(camera_to_worlds) > 0 else None, + "include_mono_prior": self.config.include_mono_prior, + }, + ) + return dataparser_outputs diff --git a/nerfstudio/data/datasets/sdf_dataset.py b/nerfstudio/data/datasets/sdf_dataset.py new file mode 100644 index 000000000..48e0cff6d --- /dev/null +++ b/nerfstudio/data/datasets/sdf_dataset.py @@ -0,0 +1,97 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SDFStudio dataset. +""" + +from pathlib import Path +from typing import Dict + +import numpy as np +import torch + +from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.data.datasets.base_dataset import InputDataset + + +class SDFDataset(InputDataset): + """Dataset that returns images and depths. + + Args: + dataparser_outputs: description of where and how to read input images. + scale_factor: The scaling factor for the dataparser outputs. + """ + + def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): + super().__init__(dataparser_outputs, scale_factor) + + # can be none if monoprior not included + self.depth_filenames = self.metadata["depth_filenames"] + self.normal_filenames = self.metadata["normal_filenames"] + self.camera_to_worlds = self.metadata["camera_to_worlds"] + # can be none if auto orient not enabled in dataparser + self.transform = self.metadata["normal_filenames"] + self.include_mono_prior = self.metadata["include_mono_prior"] + + def get_metadata(self, data: Dict) -> Dict: + # TODO supports foreground_masks + metadata = {} + if self.include_mono_prior: + depth_filepath = self.depth_filenames[data["image_idx"]] + normal_filepath = self.normal_filenames[data["image_idx"]] + camtoworld = self.camera_to_worlds[data["image_idx"]] + + # Scale depth images to meter units and also by scaling applied to cameras + depth_image, normal_image = self.get_depths_and_normals( + depth_filepath=depth_filepath, normal_filename=normal_filepath, camtoworld=camtoworld + ) + metadata["depth_image"] = depth_image + metadata["normal_image"] = normal_image + + return metadata + + def get_depths_and_normals(self, depth_filepath: Path, normal_filename: Path, camtoworld: np.ndarray): + """function to process additional depths and normal information + Args: + depth_filepath: path to depth file + normal_filename: path to normal file + camtoworld: camera to world transformation matrix + """ + + # load mono depth + depth = np.load(depth_filepath) + depth = torch.from_numpy(depth).float() + + # load mono normal + normal = np.load(normal_filename) + + # transform normal to world coordinate system + normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here + normal = torch.from_numpy(normal).float() + + rot = camtoworld[:3, :3] + + normal_map = normal.reshape(3, -1) + normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0) + + normal_map = rot @ normal_map + normal = normal_map.permute(1, 0).reshape(*normal.shape[1:], 3) + + if self.transform is not None: + h, w, _ = normal.shape + normal = self.transform[:3, :3] @ normal.reshape(-1, 3).permute(1, 0) + normal = normal.permute(1, 0).reshape(h, w, 3) + + return depth, normal diff --git a/nerfstudio/field_components/temporal_grid.py b/nerfstudio/field_components/temporal_grid.py index 6bccc110a..3bbe274b7 100644 --- a/nerfstudio/field_components/temporal_grid.py +++ b/nerfstudio/field_components/temporal_grid.py @@ -116,7 +116,6 @@ class TemporalGridEncodeFunc(Function): @staticmethod @custom_bwd def backward(ctx, grad): - inputs, temporal_row_index, embeddings, offsets, dy_dx = ctx.saved_tensors B, D, grid_channel, C, L, S, H, gridtype = ctx.dims align_corners = ctx.align_corners @@ -336,7 +335,7 @@ class TemporalGridEncoder(nn.Module): """ outputs = TemporalGridEncodeFunc.apply( xyz, - self.get_temporal_index(time[:, 0]), + self.get_temporal_index(time[:, 0].float()), self.embeddings, self.offsets, self.per_level_scale, diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index a91c3b49f..fec3aa588 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -19,7 +19,7 @@ import sys from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Union import appdirs import mediapy @@ -29,6 +29,8 @@ import torch.nn.functional as F import tyro from rich.console import Console from torch import nn +from torch.cuda.amp import custom_bwd, custom_fwd +from torch.cuda.amp.grad_scaler import GradScaler from torchtyping import TensorType CONSOLE = Console(width=120) @@ -58,6 +60,26 @@ class UNet2DConditionOutput: sample: torch.FloatTensor +class _SDSGradient(torch.autograd.Function): # pylint: disable=abstract-method + """Custom gradient function for SDS loss. Since it is already computed, we can just return it.""" + + @staticmethod + @custom_fwd + def forward(ctx, input_tensor, gt_grad): # pylint: disable=arguments-differ + del input_tensor + ctx.save_for_backward(gt_grad) + # Return magniture of gradient, not the actual loss. + return torch.mean(gt_grad**2) ** 0.5 + + @staticmethod + @custom_bwd + def backward(ctx, grad): # pylint: disable=arguments-differ + del grad + (gt_grad,) = ctx.saved_tensors + batch_size = len(gt_grad) + return gt_grad / batch_size, None + + class StableDiffusion(nn.Module): """Stable Diffusion implementation Args: @@ -162,14 +184,16 @@ class StableDiffusion(nn.Module): text_embeddings: TensorType["N", "max_length", "embed_dim"], image: TensorType["BS", 3, "H", "W"], guidance_scale: float = 100.0, - ) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]: + grad_scaler: Optional[GradScaler] = None, + ) -> torch.Tensor: """Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/) Args: text_embeddings: Text embeddings image: Rendered image guidance_scale: How much to weigh the guidance + grad_scaler: Grad scaler Returns: - A tuple of Loss, latent, and gradient + The loss """ image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear") t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) @@ -186,7 +210,7 @@ class StableDiffusion(nn.Module): # perform guidance noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) - noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) # w(t), sigma_t^2 w = 1 - self.alphas[t] @@ -194,9 +218,11 @@ class StableDiffusion(nn.Module): grad = w * (noise_pred - noise) grad = torch.nan_to_num(grad) - sds_mse = torch.mean(torch.nan_to_num(torch.square(noise_pred - noise))) + if grad_scaler is not None: + latents = grad_scaler.scale(latents) + loss = _SDSGradient.apply(latents, grad) - return sds_mse, latents, grad + return loss def produce_latents( self, @@ -240,7 +266,7 @@ class StableDiffusion(nn.Module): # perform guidance noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) - noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) # compute the previous noisy sample x_t -> x_t-1 latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 353dd7db9..419a8a1f4 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -20,8 +20,10 @@ from enum import Enum import torch from torch import nn from torchtyping import TensorType +from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples +from nerfstudio.utils.math import masked_reduction L1Loss = nn.L1Loss MSELoss = nn.MSELoss @@ -308,3 +310,218 @@ def depth_loss( return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma) raise NotImplementedError("Provided depth loss type not implemented.") + + +def monosdf_normal_loss( + normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3] +) -> TensorType[0]: + """ + Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/ + Enforces consistency between the volume rendered normal and the predicted monocular normal. + With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf + Args: + normal_pred: volume rendered normal + normal_gt: monocular normal + """ + normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1) + normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1) + l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean() + cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean() + return l1 + cos + + +class MiDaSMSELoss(nn.Module): + """ + data term from MiDaS paper + """ + + def __init__(self, reduction_type: Literal["image", "batch"] = "batch"): + super().__init__() + + self.reduction_type = reduction_type + # reduction here is different from the image/batch-based reduction. This is either "mean" or "sum" + self.mse_loss = MSELoss(reduction="none") + + def forward( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + mse loss based on reduction function + """ + summed_mask = torch.sum(mask, (1, 2)) + image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2)) + image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type) + + return image_loss + + +# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py +class GradientLoss(nn.Module): + """ + multiscale, scale-invariant gradient matching term to the disparity space. + This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth + More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11 + """ + + def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"): + """ + Args: + scales: number of scales to use + reduction_type: either "batch" or "image" + """ + super().__init__() + self.reduction_type = reduction_type + self.__scales = scales + + def forward( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + gradient loss based on reduction function + """ + total = 0 + + for scale in range(self.__scales): + step = pow(2, scale) + + grad_loss = self.gradient_loss( + prediction[:, ::step, ::step], + target[:, ::step, ::step], + mask[:, ::step, ::step], + ) + total += grad_loss + + return total + + def gradient_loss( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + multiscale, scale-invariant gradient matching term to the disparity space. + This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth + More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11 + Args: + prediction: predicted depth map + target: ground truth depth map + reduction: reduction function, either reduction_batch_based or reduction_image_based + Returns: + gradient loss based on reduction function + """ + summed_mask = torch.sum(mask, (1, 2)) + diff = prediction - target + diff = torch.mul(mask, diff) + + grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1]) + mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1]) + grad_x = torch.mul(mask_x, grad_x) + + grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :]) + mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :]) + grad_y = torch.mul(mask_y, grad_y) + + image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2)) + image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type) + + return image_loss + + +class ScaleAndShiftInvariantLoss(nn.Module): + """ + Scale and shift invariant loss as described in + "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer" + https://arxiv.org/pdf/1907.01341.pdf + """ + + def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"): + """ + Args: + alpha: weight of the regularization term + scales: number of scales to use + reduction_type: either "batch" or "image" + """ + super().__init__() + self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type) + self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type) + self.__alpha = alpha + + self.__prediction_ssi = None + + @classmethod + def compute_scale_and_shift( + cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ): + """ + More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss + This function computes scale/shift required to normalizes predicted depth map, + to allow for using normalized depth maps as input from monocular depth estimation networks. + These networks are trained such that they predict normalized depth maps. + + Solves for scale/shift using a least squares approach with a closed form solution: + Based on: + https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7 + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + scale and shift for depth prediction + """ + # system matrix: A = [[a_00, a_01], [a_10, a_11]] + a_00 = torch.sum(mask * prediction * prediction, (1, 2)) + a_01 = torch.sum(mask * prediction, (1, 2)) + a_11 = torch.sum(mask, (1, 2)) + + # right hand side: b = [b_0, b_1] + b_0 = torch.sum(mask * prediction * target, (1, 2)) + b_1 = torch.sum(mask * target, (1, 2)) + + # solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b + scale = torch.zeros_like(b_0) + shift = torch.zeros_like(b_1) + + det = a_00 * a_11 - a_01 * a_01 + valid = det.nonzero() + + scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid] + shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid] + + return scale, shift + + def forward( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + Args: + prediction: predicted depth map (unnormalized) + target: ground truth depth map (normalized) + mask: mask of valid pixels + Returns: + scale and shift invariant loss + """ + scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask) + self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1) + + total = self.__data_loss(self.__prediction_ssi, target, mask) + if self.__alpha > 0: + total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask) + + return total + + def __get_prediction_ssi(self): + """ + scale and shift invariant prediction + from https://arxiv.org/pdf/1907.01341.pdf equation 1 + """ + return self.__prediction_ssi + + prediction_ssi = property(__get_prediction_ssi) diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index 5df52d2fa..ef0079ffd 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -509,7 +509,16 @@ class VolumetricSampler(Sampler): class ProposalNetworkSampler(Sampler): - """Sampler that uses a proposal network to generate samples.""" + """Sampler that uses a proposal network to generate samples. + + Args: + num_proposal_samples_per_ray: Number of samples to generate per ray for each proposal step. + num_nerf_samples_per_ray: Number of samples to generate per ray for the NERF model. + num_proposal_network_iterations: Number of proposal network iterations to run. + single_jitter: Use a same random jitter for all samples along a ray. + update_sched: A function that takes the iteration number of steps between updates. + initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set. + """ def __init__( self, @@ -518,6 +527,7 @@ class ProposalNetworkSampler(Sampler): num_proposal_network_iterations: int = 2, single_jitter: bool = False, update_sched: Callable = lambda x: 1, + initial_sampler: Optional[Sampler] = None, ) -> None: super().__init__() self.num_proposal_samples_per_ray = num_proposal_samples_per_ray @@ -528,7 +538,10 @@ class ProposalNetworkSampler(Sampler): raise ValueError("num_proposal_network_iterations must be >= 1") # samplers - self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter) + if initial_sampler is None: + self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter) + else: + self.initial_sampler = initial_sampler self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter) self._anneal = 1.0 diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index 736cd0870..9c0579031 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -137,7 +137,7 @@ class NGPModel(Model): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) def get_training_callbacks( self, training_callback_attributes: TrainingCallbackAttributes diff --git a/nerfstudio/models/mipnerf.py b/nerfstudio/models/mipnerf.py index b8c3bcc4f..31a337d02 100644 --- a/nerfstudio/models/mipnerf.py +++ b/nerfstudio/models/mipnerf.py @@ -86,7 +86,7 @@ class MipNerfModel(Model): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} @@ -173,8 +173,8 @@ class MipNerfModel(Model): image = torch.moveaxis(image, -1, 0)[None, ...] rgb_coarse = torch.moveaxis(rgb_coarse, -1, 0)[None, ...] rgb_fine = torch.moveaxis(rgb_fine, -1, 0)[None, ...] - rgb_coarse = torch.clip(rgb_coarse, min=-1, max=1) - rgb_fine = torch.clip(rgb_fine, min=-1, max=1) + rgb_coarse = torch.clip(rgb_coarse, min=0, max=1) + rgb_fine = torch.clip(rgb_fine, min=0, max=1) coarse_psnr = self.psnr(image, rgb_coarse) fine_psnr = self.psnr(image, rgb_fine) diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 7442bfe8b..4aa58d468 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -197,7 +197,7 @@ class NerfactoModel(Model): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} diff --git a/nerfstudio/models/nerfplayer_nerfacto.py b/nerfstudio/models/nerfplayer_nerfacto.py index f35a748b1..3811aba67 100644 --- a/nerfstudio/models/nerfplayer_nerfacto.py +++ b/nerfstudio/models/nerfplayer_nerfacto.py @@ -168,7 +168,7 @@ class NerfplayerNerfactoModel(NerfactoModel): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) self.temporal_distortion = True # for viewer def get_outputs(self, ray_bundle: RayBundle): diff --git a/nerfstudio/models/nerfplayer_ngp.py b/nerfstudio/models/nerfplayer_ngp.py index 39507a538..878694468 100644 --- a/nerfstudio/models/nerfplayer_ngp.py +++ b/nerfstudio/models/nerfplayer_ngp.py @@ -145,7 +145,7 @@ class NerfplayerNGPModel(NGPModel): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) self.temporal_distortion = True # for viewer def get_outputs(self, ray_bundle: RayBundle): diff --git a/nerfstudio/models/semantic_nerfw.py b/nerfstudio/models/semantic_nerfw.py index 941716dc8..22957d268 100644 --- a/nerfstudio/models/semantic_nerfw.py +++ b/nerfstudio/models/semantic_nerfw.py @@ -137,7 +137,7 @@ class SemanticNerfWModel(Model): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index 60fec9706..d5af1579d 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -201,7 +201,7 @@ class TensoRFModel(Model): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) # colliders if self.config.enable_collider: diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py index 3e731524f..53e838891 100644 --- a/nerfstudio/models/vanilla_nerf.py +++ b/nerfstudio/models/vanilla_nerf.py @@ -118,7 +118,7 @@ class NeRFModel(Model): # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) if getattr(self.config, "enable_temporal_distortion", False): params = self.config.temporal_distortion_params diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index cee6108a9..b0a0aaec4 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -19,6 +19,7 @@ from typing import Tuple import torch from torchtyping import TensorType +from typing_extensions import Literal _USE_NERFACC = True @@ -285,3 +286,28 @@ def safe_normalize( Normalized vectors. """ return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps) + + +def masked_reduction( + input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"] +): + """ + Whether to consolidate the input_tensor across the batch or across the image + Args: + input_tensor: input tensor + mask: mask tensor + reduction_type: either "batch" or "image" + Returns: + input_tensor: reduced input_tensor + """ + if reduction_type == "batch": + # avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0) + divisor = torch.sum(mask) + input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor + elif reduction_type == "image": + # avoid division by 0 (if M = sum(mask) = 0: image_loss = 0) + valid = mask.nonzero() + + input_tensor[valid] = input_tensor[valid] / mask[valid] + input_tensor = torch.mean(input_tensor) + return input_tensor diff --git a/pyproject.toml b/pyproject.toml index ba57c01d4..8901aa282 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nerfstudio" -version = "0.1.16" +version = "0.1.17" description = "All-in-one repository for state-of-the-art NeRFs" readme = "README.md" license = { text="Apache 2.0"} @@ -22,7 +22,7 @@ dependencies = [ "tyro>=0.3.31", "gdown>=4.6.0", "ninja>=1.10", - "functorch==0.2.1", + "functorch>=0.2.1", "h5py>=2.9.0", "imageio>=2.21.1", 'importlib-metadata>=6.0.0; python_version < "3.10"', @@ -32,7 +32,7 @@ dependencies = [ "mediapy>=1.1.0", "msgpack>=1.0.4", "msgpack_numpy>=0.4.8", - "nerfacc==0.3.4", + "nerfacc==0.3.3", "open3d>=0.16.0", "opencv-python==4.6.0.66", "Pillow>=9.3.0", @@ -45,7 +45,7 @@ dependencies = [ "rich>=12.5.1", "scikit-image>=0.19.3", "tensorboard==2.9.0", - "torch==1.12.1", + "torch>=1.12.1", "torchmetrics[image]>=0.9.3", "torchtyping>=0.1.4", "torchvision>=0.13.0", @@ -100,6 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint" ns-process-data = "scripts.process_data:entrypoint" ns-download-data = "scripts.downloads.download_data:entrypoint" ns-train = "scripts.train:entrypoint" +ns-viewer = "scripts.viewer.run_viewer:entrypoint" ns-eval = "scripts.eval:entrypoint" ns-render = "scripts.render:entrypoint" ns-export = "scripts.exporter:entrypoint" diff --git a/scripts/completions/install.py b/scripts/completions/install.py index 636ed1979..98e67fea2 100755 --- a/scripts/completions/install.py +++ b/scripts/completions/install.py @@ -10,27 +10,29 @@ import shutil import stat import subprocess import sys -from typing import List, Union +from typing import List, Optional, Union import tyro from rich.console import Console from rich.prompt import Confirm from typing_extensions import Literal, assert_never +from typing_extensions import get_args as typing_get_args + +if sys.version_info < (3, 10): + import importlib_metadata +else: + from importlib import metadata as importlib_metadata ConfigureMode = Literal["install", "uninstall"] ShellType = Literal["zsh", "bash"] CONSOLE = Console(width=120) +HEADER_LINE = "# Source nerfstudio autocompletions." -ENTRYPOINTS = [ - "ns-install-cli", - "ns-process-data", - "ns-download-data", - "ns-train", - "ns-eval", - "ns-render", - "ns-dev-test", -] + +def _get_all_entry_points() -> List[str]: + entry_points = importlib_metadata.distribution("nerfstudio").entry_points + return [x.name for x in entry_points] def _check_tyro_cli(script_path: pathlib.Path) -> bool: @@ -127,6 +129,62 @@ def _exclamation() -> str: return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!" +def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str: + if shell is None: + # Install the universal script + result_script = [] + for shell_type in typing_get_args(ShellType): + result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then') + result_script.append(_get_deactivate_script(commands, shell_type, add_header=False)) + result_script.append("fi") + source_lines = "\n".join(result_script) + + elif shell == "zsh": + source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands]) + elif shell == "bash": + source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands]) + else: + assert_never(shell) + + if add_header: + source_lines = f"\n{HEADER_LINE}\n{source_lines}" + return source_lines + + +def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str: + if shell is None: + # Install the universal script + result_script = [] + for shell_type in typing_get_args(ShellType): + result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then') + result_script.append(_get_source_script(completions_dir, shell_type, add_header=False)) + result_script.append("fi") + source_lines = "\n".join(result_script) + + elif shell == "zsh": + source_lines = "\n".join( + [ + "if ! command -v compdef &> /dev/null; then", + " autoload -Uz compinit", + " compinit", + "fi", + f"source {completions_dir / 'setup.zsh'}", + ] + ) + elif shell == "bash": + source_lines = "\n".join( + [ + f"source {completions_dir / 'setup.bash'}", + ] + ) + else: + assert_never(shell) + + if add_header: + source_lines = f"\n{HEADER_LINE}\n{source_lines}" + return source_lines + + def _update_rc( completions_dir: pathlib.Path, mode: ConfigureMode, @@ -141,36 +199,13 @@ def _update_rc( """ # Install or uninstall `source_line`. - header_line = "# Source nerfstudio autocompletions." - if shell == "zsh": - source_lines = "\n".join( - [ - "", - header_line, - "if ! command -v compdef &> /dev/null; then", - " autoload -Uz compinit", - " compinit", - "fi", - f"source {completions_dir / 'setup.zsh'}", - ] - ) - elif shell == "bash": - source_lines = "\n".join( - [ - "", - header_line, - f"source {completions_dir / 'setup.bash'}", - ] - ) - else: - assert_never(shell) - + source_lines = _get_source_script(completions_dir, shell) rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc" # Always try to uninstall previous completions. rc_source = rc_path.read_text() - while header_line in rc_source: - before_install, _, after_install = rc_source.partition(header_line) + while HEADER_LINE in rc_source: + before_install, _, after_install = rc_source.partition(HEADER_LINE) source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n") assert source_file.endswith(f"/completions/setup.{shell}") rc_source = before_install + after_install @@ -189,6 +224,108 @@ def _update_rc( assert mode == "uninstall" +def _update_conda_scripts( + commands: List[str], + completions_dir: pathlib.Path, + mode: ConfigureMode, +) -> None: + """Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts. + + Args: + completions_dir: Path to location of this script. + mode: Install or uninstall completions. + """ + + # Install or uninstall `source_line`. + activate_source_lines = _get_source_script(completions_dir, None) + deactivate_source_lines = _get_deactivate_script(commands, None) + + conda_path = pathlib.Path(os.environ["CONDA_PREFIX"]) + activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh" + deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh" + if mode == "uninstall": + if activate_path.exists(): + os.remove(activate_path) + if deactivate_path.exists(): + os.remove(deactivate_path) + CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.") + elif mode == "install": + # Install completions. + activate_path.parent.mkdir(exist_ok=True, parents=True) + deactivate_path.parent.mkdir(exist_ok=True, parents=True) + with activate_path.open("w+", encoding="utf8") as f: + f.write(activate_source_lines) + with deactivate_path.open("w+", encoding="utf8") as f: + f.write(deactivate_source_lines) + CONSOLE.log( + f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment" + " to try them out." + ) + else: + assert_never(mode) + + +def _get_conda_path() -> Optional[pathlib.Path]: + """ + Returns the path to the conda environment if + the nerfstudio package is installed in one. + """ + conda_path = None + if "CONDA_PREFIX" in os.environ: + # Conda is active, we will check if the Nerfstudio is installed in the conda env. + distribution = importlib_metadata.distribution("nerfstudio") + if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]): + conda_path = pathlib.Path(os.environ["CONDA_PREFIX"]) + return conda_path + + +def _generate_completions_files( + completions_dir: pathlib.Path, + scripts_dir: pathlib.Path, + shells_supported: List[ShellType], + shells_found: List[ShellType], +) -> None: + # Set to True to install completions for scripts as well. + include_scripts = False + + # Find tyro CLIs. + script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else [] + script_names = tuple(p.name for p in script_paths) + assert len(set(script_names)) == len(script_names) + + # Get existing completion files. + existing_completions = set() + for shell in shells_supported: + target_dir = completions_dir / shell + if target_dir.exists(): + existing_completions |= set(target_dir.glob("*")) + + # Get all entry_points. + entry_points = _get_all_entry_points() + + # Run generation jobs. + concurrent_executor = concurrent.futures.ThreadPoolExecutor() + with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"): + completion_paths = list( + concurrent_executor.map( + lambda path_or_entrypoint_and_shell: _generate_completion( + path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir + ), + itertools.product(script_paths + entry_points, shells_found), + ) + ) + + # Delete obsolete completion files. + for unexpected_path in set(p.absolute() for p in existing_completions) - set( + p.absolute() for p in completion_paths + ): + if unexpected_path.is_dir(): + shutil.rmtree(unexpected_path) + elif unexpected_path.exists(): + unexpected_path.unlink() + CONSOLE.log(f":broom: Deleted {unexpected_path}.") + + def main(mode: ConfigureMode = "install") -> None: """Main script. @@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None: CONSOLE.log("[bold red]$HOME is not set. Exiting.") return + # Get conda path if in conda environment. + conda_path = _get_conda_path() + # Try to locate the user's bashrc or zshrc. - shells_supported: List[ShellType] = ["zsh", "bash"] - shells_found: List[ShellType] = [] - for shell in shells_supported: - rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc" - if not rc_path.exists(): - CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.") - else: - CONSOLE.log(f":mag: Found {rc_path.name}!") - shells_found.append(shell) + shells_supported: List[ShellType] = list(typing_get_args(ShellType)) + if conda_path is not None: + # Running in conda; we have to support all shells. + shells_found = shells_supported + CONSOLE.log(f":mag: Detected conda environment {conda_path}!") + else: + shells_found: List[ShellType] = [] + for shell in shells_supported: + rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc" + if not rc_path.exists(): + CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.") + else: + CONSOLE.log(f":mag: Found {rc_path.name}!") + shells_found.append(shell) # Get scripts/ directory. completions_dir = pathlib.Path(__file__).absolute().parent @@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None: else: CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.") elif mode == "install": - # Set to True to install completions for scripts as well. - include_scripts = False - - # Find tyro CLIs. - script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else [] - script_names = tuple(p.name for p in script_paths) - assert len(set(script_names)) == len(script_names) - - # Get existing completion files. - existing_completions = set() - for shell in shells_supported: - target_dir = completions_dir / shell - if target_dir.exists(): - existing_completions |= set(target_dir.glob("*")) - - # Run generation jobs. - concurrent_executor = concurrent.futures.ThreadPoolExecutor() - with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"): - completion_paths = list( - concurrent_executor.map( - lambda path_or_entrypoint_and_shell: _generate_completion( - path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir - ), - itertools.product(script_paths + ENTRYPOINTS, shells_found), - ) - ) - - # Delete obsolete completion files. - for unexpected_path in set(p.absolute() for p in existing_completions) - set( - p.absolute() for p in completion_paths - ): - if unexpected_path.is_dir(): - shutil.rmtree(unexpected_path) - elif unexpected_path.exists(): - unexpected_path.unlink() - CONSOLE.log(f":broom: Deleted {unexpected_path}.") + _generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found) else: assert_never(mode) - # Install or uninstall from bashrc/zshrc. - for shell in shells_found: - _update_rc(completions_dir, mode, shell) + if conda_path is not None: + # In conda environment we add the completitions activation scripts. + commands = _get_all_entry_points() + _update_conda_scripts(commands, completions_dir, mode) + else: + # Install or uninstall from bashrc/zshrc. + for shell in shells_found: + _update_rc(completions_dir, mode, shell) CONSOLE.print("[bold]All done![/bold]") diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index 89c77391f..b6775a339 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -287,6 +287,75 @@ class PhototourismDownload(DatasetDownload): os.remove(download_path) +# credit to https://autonomousvision.github.io/sdfstudio/ +# pylint: disable=line-too-long +sdfstudio_downloads = { + "sdfstudio-demo-data": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/sdfstudio-demo-data.tar", + "dtu": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/DTU.tar", + "replica": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Replica.tar", + "scannet": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/scannet.tar", + "tanks-and-temple": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/tnt_advanced.tar", + "tanks-and-temple-highres": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/highresTNT.tar", + "heritage": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Heritage-Recon.tar", + "neural-rgbd-data": "http://kaldir.vc.in.tum.de/neural_rgbd/neural_rgbd_data.zip", + "all": None, +} + +SDFstudioCaptureName = tyro.extras.literal_type_from_choices(sdfstudio_downloads.keys()) + + +@dataclass +class SDFstudioDemoDownload(DatasetDownload): + """Download the sdfstudio dataset.""" + + dataset_name: SDFstudioCaptureName = "sdfstudio-demo-data" + + def download(self, save_dir: Path): + """Download the D-NeRF dataset (https://github.com/albertpumarola/D-NeRF).""" + # TODO: give this code the same structure as download_nerfstudio + + if self.dataset_name == "all": + for dataset_name in sdfstudio_downloads: + if dataset_name != "all": + SDFstudioDemoDownload(dataset_name=dataset_name).download(save_dir) + return + + assert ( + self.dataset_name in sdfstudio_downloads + ), f"Capture name {self.dataset_name} not found in {sdfstudio_downloads.keys()}" + + url = sdfstudio_downloads[self.dataset_name] + + target_path = str(save_dir / self.dataset_name) + os.makedirs(target_path, exist_ok=True) + + file_format = url[-4:] + + download_path = Path(f"{target_path}{file_format}") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + + os.system(f"curl -L {url} > {download_path}") + if file_format == ".tar": + with tarfile.open(download_path, "r") as tar_ref: + tar_ref.extractall(str(tmp_path)) + elif file_format == ".zip": + with zipfile.ZipFile(download_path, "r") as zip_ref: + zip_ref.extractall(str(target_path)) + return + else: + raise NotImplementedError + + inner_folders = os.listdir(tmp_path) + assert len(inner_folders) == 1, "There is more than one folder inside this zip file." + folder = os.path.join(tmp_path, inner_folders[0]) + shutil.rmtree(target_path) + shutil.move(folder, target_path) + shutil.rmtree(tmp_path) + os.remove(download_path) + + Commands = Union[ Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")], Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")], @@ -294,6 +363,7 @@ Commands = Union[ Annotated[Record3dDownload, tyro.conf.subcommand(name="record3d")], Annotated[DNerfDownload, tyro.conf.subcommand(name="dnerf")], Annotated[PhototourismDownload, tyro.conf.subcommand(name="phototourism")], + Annotated[SDFstudioDemoDownload, tyro.conf.subcommand(name="sdfstudio")], ] diff --git a/scripts/process_data.py b/scripts/process_data.py index f6f9a9a7f..752e1d15c 100755 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -737,7 +737,7 @@ class ProcessRealityCapture: """Process RealityCapture data into a nerfstudio dataset. This script assumes that cameras have been aligned using RealityCapture. After alignment, it is necessary to - export the camera poses as a `.csv` file. + export the camera poses as a `.csv` file using the `Internal/External camera parameters` option. This script does the following: diff --git a/scripts/viewer/__init__.py b/scripts/viewer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/viewer/run_viewer.py b/scripts/viewer/run_viewer.py new file mode 100644 index 000000000..9a96c6703 --- /dev/null +++ b/scripts/viewer/run_viewer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +""" +Starts viewer in eval mode. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass, field, fields +from pathlib import Path + +import tyro +from rich.console import Console + +from nerfstudio.configs.base_config import ViewerConfig +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.pipelines.base_pipeline import Pipeline +from nerfstudio.utils import writer +from nerfstudio.utils.eval_utils import eval_setup +from nerfstudio.utils.writer import EventName, TimeWriter +from nerfstudio.viewer.server import viewer_utils + +CONSOLE = Console(width=120) + + +@dataclass +class ViewerConfigWithoutNumRays(ViewerConfig): + """Configuration for viewer instantiation""" + + num_rays_per_chunk: tyro.conf.Suppress[int] = -1 + start_train: tyro.conf.Suppress[bool] = False + + def as_viewer_config(self): + """Converts the instance to ViewerConfig""" + return ViewerConfig(**{x.name: getattr(self, x.name) for x in fields(self)}) + + +@dataclass +class RunViewer: + """Load a checkpoint and start the viewer.""" + + load_config: Path + """Path to config YAML file.""" + viewer: ViewerConfigWithoutNumRays = field(default_factory=ViewerConfigWithoutNumRays) + """Viewer configuration""" + + def main(self) -> None: + """Main function.""" + config, pipeline, _ = eval_setup( + self.load_config, + eval_num_rays_per_chunk=None, + test_mode="test", + ) + num_rays_per_chunk = config.viewer.num_rays_per_chunk + assert self.viewer.num_rays_per_chunk == -1 + config.vis = "viewer" + config.viewer = self.viewer.as_viewer_config() + config.viewer.num_rays_per_chunk = num_rays_per_chunk + + self._start_viewer(config, pipeline) + + def _start_viewer(self, config: TrainerConfig, pipeline): + base_dir = config.get_base_dir() + viewer_log_path = base_dir / config.viewer.relative_log_filename + viewer_state, banner_messages = viewer_utils.setup_viewer( + config.viewer, log_filename=viewer_log_path, datapath=config.pipeline.datamanager.dataparser.data + ) + + # We don't need logging, but writer.GLOBAL_BUFFER needs to be populated + config.logging.local_writer.enable = False + writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages) + + assert viewer_state and pipeline.datamanager.train_dataset + viewer_state.init_scene( + dataset=pipeline.datamanager.train_dataset, + start_train=False, + ) + while True: + viewer_state.vis["renderingState/isTraining"].write(False) + self._update_viewer_state(viewer_state, config, pipeline) + + def _update_viewer_state(self, viewer_state: viewer_utils.ViewerState, config: TrainerConfig, pipeline: Pipeline): + """Updates the viewer state by rendering out scene with current pipeline + Returns the time taken to render scene. + + """ + # NOTE: step must be > 0 otherwise the rendering would not happen + step = 1 + num_rays_per_batch = config.pipeline.datamanager.train_num_rays_per_batch + with TimeWriter(writer, EventName.ITER_VIS_TIME) as _: + try: + viewer_state.update_scene(self, step, pipeline.model, num_rays_per_batch) + except RuntimeError: + time.sleep(0.03) # sleep to allow buffer to reset + assert viewer_state.vis is not None + viewer_state.vis["renderingState/log_errors"].write( + "Error: GPU out of memory. Reduce resolution to prevent viewer from crashing." + ) + + def save_checkpoint(self, *args, **kwargs): + """ + Mock method because we pass this instance to viewer_state.update_scene + """ + + +def entrypoint(): + """Entrypoint for use with pyproject scripts.""" + tyro.extras.set_accent_color("bright_yellow") + tyro.cli(RunViewer).main() + + +if __name__ == "__main__": + entrypoint() + +# For sphinx docs +get_parser_fn = lambda: tyro.extras.get_parser(RunViewer) # noqa diff --git a/scripts/viewer/view_dataset.py b/scripts/viewer/view_dataset.py index 1f5aa2201..4bd526552 100644 --- a/scripts/viewer/view_dataset.py +++ b/scripts/viewer/view_dataset.py @@ -3,22 +3,21 @@ view_dataset.py """ -from rich.console import Console - -CONSOLE = Console(width=120) import time from datetime import timedelta from pathlib import Path import torch import tyro +from rich.console import Console from nerfstudio.configs.base_config import ViewerConfig -from nerfstudio.data.datamanagers import AnnotatedDataParserUnion +from nerfstudio.data.datamanagers.base_datamanager import AnnotatedDataParserUnion from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.viewer.server import viewer_utils DEFAULT_TIMEOUT = timedelta(minutes=30) +CONSOLE = Console(width=120) # speedup for when input size to model doesn't change (much) torch.backends.cudnn.benchmark = True # type: ignore @@ -30,9 +29,10 @@ def main( log_base_dir: Path = Path("/tmp/nerfstudio_viewer_logs"), ) -> None: """Main function.""" - viewer_state = viewer_utils.ViewerState( + viewer_state, _ = viewer_utils.setup_viewer( viewer, log_filename=log_base_dir / viewer.relative_log_filename, + datapath=dataparser.data, ) dataset = InputDataset(dataparser.setup().get_dataparser_outputs(split="train")) viewer_state.init_scene(dataset=dataset, start_train=False)