This commit is contained in:
Justin committed 2023-02-16 06:49:44 +00:00
commit bae8e5cc30
34 files changed
+1081 -144

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
{
"name": "NerfStudio Dev Container",
"dockerComposeFile": "docker-compose.yml",
"service": "nerfstudio",
"workspaceFolder": "/workspace",
"shutdownAction": "stopCompose",
"extensions": [
"mutantdino.resourcemonitor",
"ms-azuretools.vscode-docker",
"nvidia.nsight-vscode-edition",
"ms-python.python"
]
}
+28
View File
@@ -0,0 +1,28 @@
version: "3.8"
services:
nerfstudio:
shm_size: '12gb'
image: dromni/nerfstudio:0.1.17
# build:
# context: ..
# dockerfile: ./Dockerfile
stdin_open: true
tty: true
environment:
NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
DISPLAY: $DISPLAY
ports:
- "7007:7007"
volumes:
- ../:/workspace/
- /tmp/.X11-unix:/tmp/.X11-unix
working_dir: /workspace/
command: /bin/bash
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
+22 -8
View File
@@ -95,13 +95,25 @@ python -m pip install --upgrade pip
### Dependencies
Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
Install pytorch with CUDA (this repo has been tested with CUDA 11.3 and CUDA 11.7) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
For CUDA 11.3:
```bash
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
For CUDA 11.7:
```bash
pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
See [Dependencies](https://github.com/nerfstudio-project/nerfstudio/blob/main/docs/quickstart/installation.md#dependencies)
in the Installation documentation for more.
### Installing nerfstudio
Easy option:
@@ -290,16 +302,18 @@ We provide the following support structures to make life easier for getting star
# Citation
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
If you use this library or find the documentation useful for your research, please consider citing:
```
@misc{nerfstudio,
title={Nerfstudio: A Framework for Neural Radiance Field Development},
author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi,
Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi,
Abhik Ahuja, David McAllister, Angjoo Kanazawa},
year={2022},
url={https://github.com/nerfstudio-project/nerfstudio},
@article{nerfstudio,
author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi,
Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin,
Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo},
title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development},
journal = {arXiv preprint arXiv:2302.04264},
year = {2023},
}
```
+4 -4
View File
@@ -292,9 +292,9 @@
"if os.path.exists(f\"data/nerfstudio/{scene}/transforms.json\"):\n",
" !ns-train nerfacto --viewer.websocket-port 7007 nerfstudio-data --data data/nerfstudio/$scene --downscale-factor 4\n",
"else:\n",
" import IPython\n",
" display(IPython.display.HTML('<h3 style=\"color:red\">Error: Data processing did not complete</h3>'))\n",
" display(IPython.display.HTML('<h3>Please re-run `Downloading and Processing Data`, or view the FAQ for more info.</h3>'))"
" from IPython.core.display import display, HTML\n",
" display(HTML('<h3 style=\"color:red\">Error: Data processing did not complete</h3>'))\n",
" display(HTML('<h3>Please re-run `Downloading and Processing Data`, or view the FAQ for more info.</h3>'))"
]
},
{
@@ -383,4 +383,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}
@@ -87,7 +87,7 @@ def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
You can see our code for more details.
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/datamanagers.py
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/data/datamanagers/base_datamanager.py
:color: primary
:outline:
See the code!
+9 -7
View File
@@ -155,16 +155,18 @@ We'll be constantly growing this list! So make sure to check back in to see our
## Citation
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
If you use this library or find the documentation useful for your research, please consider citing:
```none
@misc{nerfstudio,
title={Nerfstudio: A Framework for Neural Radiance Field Development},
author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi,
Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi,
Abhik Ahuja, David McAllister, Angjoo Kanazawa},
year={2022},
url={https://github.com/nerfstudio-project/nerfstudio},
@article{nerfstudio,
author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi,
Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin,
Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo},
title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development},
journal = {arXiv preprint arXiv:2302.04264},
year = {2023},
}
```
+9 -1
View File
@@ -30,7 +30,7 @@ Navigating to the link at the end of the terminal will load the webviewer. If yo
- All data configurations must go at the end. In this case, `nerfstudio-data` and all of its corresponding configurations come at the end after the model and viewer specification.
:::
## Resume from checkpoint / visualize existing run
## Resume from checkpoint
It is possible to load a pretrained model by running
@@ -40,6 +40,14 @@ ns-train nerfacto --data data/nerfstudio/poster --load-dir {outputs/.../nerfstud
This will automatically start training. If you do not want it to train, add `--viewer.start-train False` to your training command.
## Visualize existing run
Given a pretrained model checkpoint, you can start the viewer by running
```bash
ns-viewer --load-config {outputs/.../config.yml}
```
## Exporting Results
Once you have a NeRF model you can either render out a video or export a point cloud.
+30 -5
View File
@@ -17,12 +17,38 @@ python -m pip install --upgrade pip
## Dependencies
Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
(pytorch)=
### pytorch
Either pytorch 1.12.1 (with CUDA up to 11.3) or 1.13.1 (with CUDA 11.6/7/8) can be used.
- To install 1.12.1 with CUDA 11.3:
```bash
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
- To install 1.13.1 with CUDA 11.7:
Note that if a pytorch version prior to 1.13 is installed,
it should be uninstalled first to avoid upgrade issues (e.g. with functorch)
```bash
pip uninstall torch torchvision functorch
```
Install pytorch 1.13.1 with CUDA (this repo has been tested with CUDA 11.7.1) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
```bash
pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117
```
### tinycudann
After pytorch, install the torch bindings for tinycudann:
```bash
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
## Installing nerfstudio
@@ -134,9 +160,8 @@ While installing tiny-cuda, you run into: `The detected CUDA version mismatches
**Solution**:
```
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
```
Reinstall pytorch with the correct CUDA version.
See [pytorch](pytorch) under Dependencies, above.
<br />
+4
View File
@@ -100,12 +100,16 @@ def get_spiral_path(
new_c2ws.append(c2wh[:3, :4])
new_c2ws = torch.stack(new_c2ws, dim=0)
times = None
if camera.times is not None:
times = torch.linspace(0, 1, steps)[:, None]
return Cameras(
fx=camera.fx[0],
fy=camera.fy[0],
cx=camera.cx[0],
cy=camera.cy[0],
camera_to_worlds=new_c2ws,
times=times,
)
+2 -2
View File
@@ -230,7 +230,7 @@ class Cameras(TensorDataclass):
c_x_y: cx or cy for when h_w == None
"""
if isinstance(h_w, int):
h_w = torch.Tensor([h_w]).to(torch.int64).to(self.device)
h_w = torch.as_tensor([h_w]).to(torch.int64).to(self.device)
elif isinstance(h_w, torch.Tensor):
assert not torch.is_floating_point(h_w), f"height and width tensor must be of type int, not: {h_w.dtype}"
h_w = h_w.to(torch.int64).to(self.device)
@@ -238,7 +238,7 @@ class Cameras(TensorDataclass):
h_w = h_w.unsqueeze(-1)
# assert torch.all(h_w == h_w.view(-1)[0]), "Batched cameras of different h, w will be allowed in the future."
elif h_w is None:
h_w = torch.Tensor((c_x_y * 2).to(torch.int64).to(self.device))
h_w = torch.as_tensor((c_x_y * 2)).to(torch.int64).to(self.device)
else:
raise ValueError("Height must be an int, tensor, or None, received: " + str(type(h_w)))
return h_w
@@ -49,6 +49,7 @@ from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserCo
from nerfstudio.data.dataparsers.phototourism_dataparser import (
PhototourismDataParserConfig,
)
from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler
from nerfstudio.data.utils.dataloaders import (
@@ -75,6 +76,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of
"dnerf-data": DNeRFDataParserConfig(),
"phototourism-data": PhototourismDataParserConfig(),
"dycheck-data": DycheckDataParserConfig(),
"sdfstudio-data": SDFStudioDataParserConfig(),
},
prefix_names=False, # Omit prefixes in subcommands themselves.
)
@@ -123,6 +123,10 @@ class InstantNGP(DataParser):
w, h = meta["w"], meta["h"]
camera_type = CameraType.PERSPECTIVE
if meta.get("is_fisheye", False):
camera_type = CameraType.FISHEYE
cameras = Cameras(
fx=float(fl_x),
fy=float(fl_y),
@@ -132,7 +136,7 @@ class InstantNGP(DataParser):
height=int(h),
width=int(w),
camera_to_worlds=camera_to_world,
camera_type=CameraType.PERSPECTIVE,
camera_type=camera_type,
)
# TODO(ethan): add alpha background color
@@ -0,0 +1,158 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Datapaser for sdfstudio formatted data"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Type
import torch
from rich.console import Console
from nerfstudio.cameras import camera_utils
from nerfstudio.cameras.cameras import Cameras, CameraType
from nerfstudio.data.dataparsers.base_dataparser import (
DataParser,
DataParserConfig,
DataparserOutputs,
)
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.utils.io import load_from_json
CONSOLE = Console()
@dataclass
class SDFStudioDataParserConfig(DataParserConfig):
"""Scene dataset parser config"""
_target: Type = field(default_factory=lambda: SDFStudio)
"""target class to instantiate"""
data: Path = Path("data/DTU/scan65")
"""Directory specifying location of data."""
include_mono_prior: bool = False
"""whether or not to load monocular depth and normal """
include_foreground_mask: bool = False
"""whether or not to load foreground mask"""
downscale_factor: int = 1
scene_scale: float = 2.0
"""
Sets the bounding cube to have edge length of this size.
The longest dimension of the Friends axis-aligned bbox will be scaled to this value.
"""
skip_every_for_val_split: int = 1
"""sub sampling validation images"""
auto_orient: bool = False
@dataclass
class SDFStudio(DataParser):
"""SDFStudio Dataset"""
config: SDFStudioDataParserConfig
def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements
# load meta data
meta = load_from_json(self.config.data / "meta_data.json")
indices = list(range(len(meta["frames"])))
# subsample to avoid out-of-memory for validation set
if split != "train" and self.config.skip_every_for_val_split >= 1:
indices = indices[:: self.config.skip_every_for_val_split]
image_filenames = []
depth_filenames = []
normal_filenames = []
transform = None
fx = []
fy = []
cx = []
cy = []
camera_to_worlds = []
for i, frame in enumerate(meta["frames"]):
if i not in indices:
continue
image_filename = self.config.data / frame["rgb_path"]
depth_filename = self.config.data / frame["mono_depth_path"]
normal_filename = self.config.data / frame["mono_normal_path"]
intrinsics = torch.tensor(frame["intrinsics"])
camtoworld = torch.tensor(frame["camtoworld"])
# append data
image_filenames.append(image_filename)
depth_filenames.append(depth_filename)
normal_filenames.append(normal_filename)
fx.append(intrinsics[0, 0])
fy.append(intrinsics[1, 1])
cx.append(intrinsics[0, 2])
cy.append(intrinsics[1, 2])
camera_to_worlds.append(camtoworld)
fx = torch.stack(fx)
fy = torch.stack(fy)
cx = torch.stack(cx)
cy = torch.stack(cy)
camera_to_worlds = torch.stack(camera_to_worlds)
# Convert from COLMAP's/OPENCV's camera coordinate system to nerfstudio
camera_to_worlds[:, 0:3, 1:3] *= -1
if self.config.auto_orient:
camera_to_worlds, transform = camera_utils.auto_orient_and_center_poses(
camera_to_worlds,
method="up",
center_poses=False,
)
# scene box from meta data
meta_scene_box = meta["scene_box"]
aabb = torch.tensor(meta_scene_box["aabb"], dtype=torch.float32)
scene_box = SceneBox(
aabb=aabb,
)
height, width = meta["height"], meta["width"]
cameras = Cameras(
fx=fx,
fy=fy,
cx=cx,
cy=cy,
height=height,
width=width,
camera_to_worlds=camera_to_worlds[:, :3, :4],
camera_type=CameraType.PERSPECTIVE,
)
# TODO supports downsample
# cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor)
assert meta["has_mono_prior"] == self.config.include_mono_prior, f"no mono prior in {self.config.data}"
dataparser_outputs = DataparserOutputs(
image_filenames=image_filenames,
cameras=cameras,
scene_box=scene_box,
metadata={
"depth_filenames": depth_filenames if len(depth_filenames) > 0 else None,
"normal_filenames": normal_filenames if len(normal_filenames) > 0 else None,
"transform": transform,
"camera_to_worlds": camera_to_worlds if len(camera_to_worlds) > 0 else None,
"include_mono_prior": self.config.include_mono_prior,
},
)
return dataparser_outputs
+97
View File
@@ -0,0 +1,97 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
SDFStudio dataset.
"""
from pathlib import Path
from typing import Dict
import numpy as np
import torch
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
from nerfstudio.data.datasets.base_dataset import InputDataset
class SDFDataset(InputDataset):
"""Dataset that returns images and depths.
Args:
dataparser_outputs: description of where and how to read input images.
scale_factor: The scaling factor for the dataparser outputs.
"""
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
super().__init__(dataparser_outputs, scale_factor)
# can be none if monoprior not included
self.depth_filenames = self.metadata["depth_filenames"]
self.normal_filenames = self.metadata["normal_filenames"]
self.camera_to_worlds = self.metadata["camera_to_worlds"]
# can be none if auto orient not enabled in dataparser
self.transform = self.metadata["normal_filenames"]
self.include_mono_prior = self.metadata["include_mono_prior"]
def get_metadata(self, data: Dict) -> Dict:
# TODO supports foreground_masks
metadata = {}
if self.include_mono_prior:
depth_filepath = self.depth_filenames[data["image_idx"]]
normal_filepath = self.normal_filenames[data["image_idx"]]
camtoworld = self.camera_to_worlds[data["image_idx"]]
# Scale depth images to meter units and also by scaling applied to cameras
depth_image, normal_image = self.get_depths_and_normals(
depth_filepath=depth_filepath, normal_filename=normal_filepath, camtoworld=camtoworld
)
metadata["depth_image"] = depth_image
metadata["normal_image"] = normal_image
return metadata
def get_depths_and_normals(self, depth_filepath: Path, normal_filename: Path, camtoworld: np.ndarray):
"""function to process additional depths and normal information
Args:
depth_filepath: path to depth file
normal_filename: path to normal file
camtoworld: camera to world transformation matrix
"""
# load mono depth
depth = np.load(depth_filepath)
depth = torch.from_numpy(depth).float()
# load mono normal
normal = np.load(normal_filename)
# transform normal to world coordinate system
normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here
normal = torch.from_numpy(normal).float()
rot = camtoworld[:3, :3]
normal_map = normal.reshape(3, -1)
normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0)
normal_map = rot @ normal_map
normal = normal_map.permute(1, 0).reshape(*normal.shape[1:], 3)
if self.transform is not None:
h, w, _ = normal.shape
normal = self.transform[:3, :3] @ normal.reshape(-1, 3).permute(1, 0)
normal = normal.permute(1, 0).reshape(h, w, 3)
return depth, normal
+1 -2
View File
@@ -116,7 +116,6 @@ class TemporalGridEncodeFunc(Function):
@staticmethod
@custom_bwd
def backward(ctx, grad):
inputs, temporal_row_index, embeddings, offsets, dy_dx = ctx.saved_tensors
B, D, grid_channel, C, L, S, H, gridtype = ctx.dims
align_corners = ctx.align_corners
@@ -336,7 +335,7 @@ class TemporalGridEncoder(nn.Module):
"""
outputs = TemporalGridEncodeFunc.apply(
xyz,
self.get_temporal_index(time[:, 0]),
self.get_temporal_index(time[:, 0].float()),
self.embeddings,
self.offsets,
self.per_level_scale,
+33 -7
View File
@@ -19,7 +19,7 @@
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Tuple, Union
from typing import List, Optional, Union
import appdirs
import mediapy
@@ -29,6 +29,8 @@ import torch.nn.functional as F
import tyro
from rich.console import Console
from torch import nn
from torch.cuda.amp import custom_bwd, custom_fwd
from torch.cuda.amp.grad_scaler import GradScaler
from torchtyping import TensorType
CONSOLE = Console(width=120)
@@ -58,6 +60,26 @@ class UNet2DConditionOutput:
sample: torch.FloatTensor
class _SDSGradient(torch.autograd.Function): # pylint: disable=abstract-method
"""Custom gradient function for SDS loss. Since it is already computed, we can just return it."""
@staticmethod
@custom_fwd
def forward(ctx, input_tensor, gt_grad): # pylint: disable=arguments-differ
del input_tensor
ctx.save_for_backward(gt_grad)
# Return magniture of gradient, not the actual loss.
return torch.mean(gt_grad**2) ** 0.5
@staticmethod
@custom_bwd
def backward(ctx, grad): # pylint: disable=arguments-differ
del grad
(gt_grad,) = ctx.saved_tensors
batch_size = len(gt_grad)
return gt_grad / batch_size, None
class StableDiffusion(nn.Module):
"""Stable Diffusion implementation
Args:
@@ -162,14 +184,16 @@ class StableDiffusion(nn.Module):
text_embeddings: TensorType["N", "max_length", "embed_dim"],
image: TensorType["BS", 3, "H", "W"],
guidance_scale: float = 100.0,
) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]:
grad_scaler: Optional[GradScaler] = None,
) -> torch.Tensor:
"""Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/)
Args:
text_embeddings: Text embeddings
image: Rendered image
guidance_scale: How much to weigh the guidance
grad_scaler: Grad scaler
Returns:
A tuple of Loss, latent, and gradient
The loss
"""
image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear")
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
@@ -186,7 +210,7 @@ class StableDiffusion(nn.Module):
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# w(t), sigma_t^2
w = 1 - self.alphas[t]
@@ -194,9 +218,11 @@ class StableDiffusion(nn.Module):
grad = w * (noise_pred - noise)
grad = torch.nan_to_num(grad)
sds_mse = torch.mean(torch.nan_to_num(torch.square(noise_pred - noise)))
if grad_scaler is not None:
latents = grad_scaler.scale(latents)
loss = _SDSGradient.apply(latents, grad)
return sds_mse, latents, grad
return loss
def produce_latents(
self,
@@ -240,7 +266,7 @@ class StableDiffusion(nn.Module):
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore
+217
View File
@@ -20,8 +20,10 @@ from enum import Enum
import torch
from torch import nn
from torchtyping import TensorType
from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils.math import masked_reduction
L1Loss = nn.L1Loss
MSELoss = nn.MSELoss
@@ -308,3 +310,218 @@ def depth_loss(
return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma)
raise NotImplementedError("Provided depth loss type not implemented.")
def monosdf_normal_loss(
normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
) -> TensorType[0]:
"""
Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
Enforces consistency between the volume rendered normal and the predicted monocular normal.
With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf
Args:
normal_pred: volume rendered normal
normal_gt: monocular normal
"""
normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1)
normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1)
l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean()
cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean()
return l1 + cos
class MiDaSMSELoss(nn.Module):
"""
data term from MiDaS paper
"""
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
super().__init__()
self.reduction_type = reduction_type
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
self.mse_loss = MSELoss(reduction="none")
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
mse loss based on reduction function
"""
summed_mask = torch.sum(mask, (1, 2))
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
return image_loss
# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py
class GradientLoss(nn.Module):
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
"""
def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
"""
Args:
scales: number of scales to use
reduction_type: either "batch" or "image"
"""
super().__init__()
self.reduction_type = reduction_type
self.__scales = scales
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
gradient loss based on reduction function
"""
total = 0
for scale in range(self.__scales):
step = pow(2, scale)
grad_loss = self.gradient_loss(
prediction[:, ::step, ::step],
target[:, ::step, ::step],
mask[:, ::step, ::step],
)
total += grad_loss
return total
def gradient_loss(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
Args:
prediction: predicted depth map
target: ground truth depth map
reduction: reduction function, either reduction_batch_based or reduction_image_based
Returns:
gradient loss based on reduction function
"""
summed_mask = torch.sum(mask, (1, 2))
diff = prediction - target
diff = torch.mul(mask, diff)
grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1])
mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1])
grad_x = torch.mul(mask_x, grad_x)
grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :])
mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :])
grad_y = torch.mul(mask_y, grad_y)
image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
return image_loss
class ScaleAndShiftInvariantLoss(nn.Module):
"""
Scale and shift invariant loss as described in
"Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer"
https://arxiv.org/pdf/1907.01341.pdf
"""
def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
"""
Args:
alpha: weight of the regularization term
scales: number of scales to use
reduction_type: either "batch" or "image"
"""
super().__init__()
self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type)
self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type)
self.__alpha = alpha
self.__prediction_ssi = None
@classmethod
def compute_scale_and_shift(
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
):
"""
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
This function computes scale/shift required to normalizes predicted depth map,
to allow for using normalized depth maps as input from monocular depth estimation networks.
These networks are trained such that they predict normalized depth maps.
Solves for scale/shift using a least squares approach with a closed form solution:
Based on:
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
scale and shift for depth prediction
"""
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
a_01 = torch.sum(mask * prediction, (1, 2))
a_11 = torch.sum(mask, (1, 2))
# right hand side: b = [b_0, b_1]
b_0 = torch.sum(mask * prediction * target, (1, 2))
b_1 = torch.sum(mask * target, (1, 2))
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
scale = torch.zeros_like(b_0)
shift = torch.zeros_like(b_1)
det = a_00 * a_11 - a_01 * a_01
valid = det.nonzero()
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
return scale, shift
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map (unnormalized)
target: ground truth depth map (normalized)
mask: mask of valid pixels
Returns:
scale and shift invariant loss
"""
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
total = self.__data_loss(self.__prediction_ssi, target, mask)
if self.__alpha > 0:
total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask)
return total
def __get_prediction_ssi(self):
"""
scale and shift invariant prediction
from https://arxiv.org/pdf/1907.01341.pdf equation 1
"""
return self.__prediction_ssi
prediction_ssi = property(__get_prediction_ssi)
+15 -2
View File
@@ -509,7 +509,16 @@ class VolumetricSampler(Sampler):
class ProposalNetworkSampler(Sampler):
"""Sampler that uses a proposal network to generate samples."""
"""Sampler that uses a proposal network to generate samples.
Args:
num_proposal_samples_per_ray: Number of samples to generate per ray for each proposal step.
num_nerf_samples_per_ray: Number of samples to generate per ray for the NERF model.
num_proposal_network_iterations: Number of proposal network iterations to run.
single_jitter: Use a same random jitter for all samples along a ray.
update_sched: A function that takes the iteration number of steps between updates.
initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set.
"""
def __init__(
self,
@@ -518,6 +527,7 @@ class ProposalNetworkSampler(Sampler):
num_proposal_network_iterations: int = 2,
single_jitter: bool = False,
update_sched: Callable = lambda x: 1,
initial_sampler: Optional[Sampler] = None,
) -> None:
super().__init__()
self.num_proposal_samples_per_ray = num_proposal_samples_per_ray
@@ -528,7 +538,10 @@ class ProposalNetworkSampler(Sampler):
raise ValueError("num_proposal_network_iterations must be >= 1")
# samplers
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
if initial_sampler is None:
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
else:
self.initial_sampler = initial_sampler
self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter)
self._anneal = 1.0
+1 -1
View File
@@ -137,7 +137,7 @@ class NGPModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
+3 -3
View File
@@ -86,7 +86,7 @@ class MipNerfModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
@@ -173,8 +173,8 @@ class MipNerfModel(Model):
image = torch.moveaxis(image, -1, 0)[None, ...]
rgb_coarse = torch.moveaxis(rgb_coarse, -1, 0)[None, ...]
rgb_fine = torch.moveaxis(rgb_fine, -1, 0)[None, ...]
rgb_coarse = torch.clip(rgb_coarse, min=-1, max=1)
rgb_fine = torch.clip(rgb_fine, min=-1, max=1)
rgb_coarse = torch.clip(rgb_coarse, min=0, max=1)
rgb_fine = torch.clip(rgb_fine, min=0, max=1)
coarse_psnr = self.psnr(image, rgb_coarse)
fine_psnr = self.psnr(image, rgb_fine)
+1 -1
View File
@@ -197,7 +197,7 @@ class NerfactoModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
+1 -1
View File
@@ -168,7 +168,7 @@ class NerfplayerNerfactoModel(NerfactoModel):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
self.temporal_distortion = True # for viewer
def get_outputs(self, ray_bundle: RayBundle):
+1 -1
View File
@@ -145,7 +145,7 @@ class NerfplayerNGPModel(NGPModel):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
self.temporal_distortion = True # for viewer
def get_outputs(self, ray_bundle: RayBundle):
+1 -1
View File
@@ -137,7 +137,7 @@ class SemanticNerfWModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
+1 -1
View File
@@ -201,7 +201,7 @@ class TensoRFModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
# colliders
if self.config.enable_collider:
+1 -1
View File
@@ -118,7 +118,7 @@ class NeRFModel(Model):
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
if getattr(self.config, "enable_temporal_distortion", False):
params = self.config.temporal_distortion_params
+26
View File
@@ -19,6 +19,7 @@ from typing import Tuple
import torch
from torchtyping import TensorType
from typing_extensions import Literal
_USE_NERFACC = True
@@ -285,3 +286,28 @@ def safe_normalize(
Normalized vectors.
"""
return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps)
def masked_reduction(
input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
):
"""
Whether to consolidate the input_tensor across the batch or across the image
Args:
input_tensor: input tensor
mask: mask tensor
reduction_type: either "batch" or "image"
Returns:
input_tensor: reduced input_tensor
"""
if reduction_type == "batch":
# avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0)
divisor = torch.sum(mask)
input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor
elif reduction_type == "image":
# avoid division by 0 (if M = sum(mask) = 0: image_loss = 0)
valid = mask.nonzero()
input_tensor[valid] = input_tensor[valid] / mask[valid]
input_tensor = torch.mean(input_tensor)
return input_tensor
+5 -4
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "nerfstudio"
version = "0.1.16"
version = "0.1.17"
description = "All-in-one repository for state-of-the-art NeRFs"
readme = "README.md"
license = { text="Apache 2.0"}
@@ -22,7 +22,7 @@ dependencies = [
"tyro>=0.3.31",
"gdown>=4.6.0",
"ninja>=1.10",
"functorch==0.2.1",
"functorch>=0.2.1",
"h5py>=2.9.0",
"imageio>=2.21.1",
'importlib-metadata>=6.0.0; python_version < "3.10"',
@@ -32,7 +32,7 @@ dependencies = [
"mediapy>=1.1.0",
"msgpack>=1.0.4",
"msgpack_numpy>=0.4.8",
"nerfacc==0.3.4",
"nerfacc==0.3.3",
"open3d>=0.16.0",
"opencv-python==4.6.0.66",
"Pillow>=9.3.0",
@@ -45,7 +45,7 @@ dependencies = [
"rich>=12.5.1",
"scikit-image>=0.19.3",
"tensorboard==2.9.0",
"torch==1.12.1",
"torch>=1.12.1",
"torchmetrics[image]>=0.9.3",
"torchtyping>=0.1.4",
"torchvision>=0.13.0",
@@ -100,6 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint"
ns-process-data = "scripts.process_data:entrypoint"
ns-download-data = "scripts.downloads.download_data:entrypoint"
ns-train = "scripts.train:entrypoint"
ns-viewer = "scripts.viewer.run_viewer:entrypoint"
ns-eval = "scripts.eval:entrypoint"
ns-render = "scripts.render:entrypoint"
ns-export = "scripts.exporter:entrypoint"
+199 -84
View File
@@ -10,27 +10,29 @@ import shutil
import stat
import subprocess
import sys
from typing import List, Union
from typing import List, Optional, Union
import tyro
from rich.console import Console
from rich.prompt import Confirm
from typing_extensions import Literal, assert_never
from typing_extensions import get_args as typing_get_args
if sys.version_info < (3, 10):
import importlib_metadata
else:
from importlib import metadata as importlib_metadata
ConfigureMode = Literal["install", "uninstall"]
ShellType = Literal["zsh", "bash"]
CONSOLE = Console(width=120)
HEADER_LINE = "# Source nerfstudio autocompletions."
ENTRYPOINTS = [
"ns-install-cli",
"ns-process-data",
"ns-download-data",
"ns-train",
"ns-eval",
"ns-render",
"ns-dev-test",
]
def _get_all_entry_points() -> List[str]:
entry_points = importlib_metadata.distribution("nerfstudio").entry_points
return [x.name for x in entry_points]
def _check_tyro_cli(script_path: pathlib.Path) -> bool:
@@ -127,6 +129,62 @@ def _exclamation() -> str:
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str:
if shell is None:
# Install the universal script
result_script = []
for shell_type in typing_get_args(ShellType):
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
result_script.append(_get_deactivate_script(commands, shell_type, add_header=False))
result_script.append("fi")
source_lines = "\n".join(result_script)
elif shell == "zsh":
source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands])
elif shell == "bash":
source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands])
else:
assert_never(shell)
if add_header:
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
return source_lines
def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str:
if shell is None:
# Install the universal script
result_script = []
for shell_type in typing_get_args(ShellType):
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
result_script.append(_get_source_script(completions_dir, shell_type, add_header=False))
result_script.append("fi")
source_lines = "\n".join(result_script)
elif shell == "zsh":
source_lines = "\n".join(
[
"if ! command -v compdef &> /dev/null; then",
" autoload -Uz compinit",
" compinit",
"fi",
f"source {completions_dir / 'setup.zsh'}",
]
)
elif shell == "bash":
source_lines = "\n".join(
[
f"source {completions_dir / 'setup.bash'}",
]
)
else:
assert_never(shell)
if add_header:
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
return source_lines
def _update_rc(
completions_dir: pathlib.Path,
mode: ConfigureMode,
@@ -141,36 +199,13 @@ def _update_rc(
"""
# Install or uninstall `source_line`.
header_line = "# Source nerfstudio autocompletions."
if shell == "zsh":
source_lines = "\n".join(
[
"",
header_line,
"if ! command -v compdef &> /dev/null; then",
" autoload -Uz compinit",
" compinit",
"fi",
f"source {completions_dir / 'setup.zsh'}",
]
)
elif shell == "bash":
source_lines = "\n".join(
[
"",
header_line,
f"source {completions_dir / 'setup.bash'}",
]
)
else:
assert_never(shell)
source_lines = _get_source_script(completions_dir, shell)
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
# Always try to uninstall previous completions.
rc_source = rc_path.read_text()
while header_line in rc_source:
before_install, _, after_install = rc_source.partition(header_line)
while HEADER_LINE in rc_source:
before_install, _, after_install = rc_source.partition(HEADER_LINE)
source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n")
assert source_file.endswith(f"/completions/setup.{shell}")
rc_source = before_install + after_install
@@ -189,6 +224,108 @@ def _update_rc(
assert mode == "uninstall"
def _update_conda_scripts(
commands: List[str],
completions_dir: pathlib.Path,
mode: ConfigureMode,
) -> None:
"""Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts.
Args:
completions_dir: Path to location of this script.
mode: Install or uninstall completions.
"""
# Install or uninstall `source_line`.
activate_source_lines = _get_source_script(completions_dir, None)
deactivate_source_lines = _get_deactivate_script(commands, None)
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh"
deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh"
if mode == "uninstall":
if activate_path.exists():
os.remove(activate_path)
if deactivate_path.exists():
os.remove(deactivate_path)
CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.")
elif mode == "install":
# Install completions.
activate_path.parent.mkdir(exist_ok=True, parents=True)
deactivate_path.parent.mkdir(exist_ok=True, parents=True)
with activate_path.open("w+", encoding="utf8") as f:
f.write(activate_source_lines)
with deactivate_path.open("w+", encoding="utf8") as f:
f.write(deactivate_source_lines)
CONSOLE.log(
f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment"
" to try them out."
)
else:
assert_never(mode)
def _get_conda_path() -> Optional[pathlib.Path]:
"""
Returns the path to the conda environment if
the nerfstudio package is installed in one.
"""
conda_path = None
if "CONDA_PREFIX" in os.environ:
# Conda is active, we will check if the Nerfstudio is installed in the conda env.
distribution = importlib_metadata.distribution("nerfstudio")
if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]):
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
return conda_path
def _generate_completions_files(
completions_dir: pathlib.Path,
scripts_dir: pathlib.Path,
shells_supported: List[ShellType],
shells_found: List[ShellType],
) -> None:
# Set to True to install completions for scripts as well.
include_scripts = False
# Find tyro CLIs.
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Get all entry_points.
entry_points = _get_all_entry_points()
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_or_entrypoint_and_shell: _generate_completion(
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
),
itertools.product(script_paths + entry_points, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
def main(mode: ConfigureMode = "install") -> None:
"""Main script.
@@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None:
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
return
# Get conda path if in conda environment.
conda_path = _get_conda_path()
# Try to locate the user's bashrc or zshrc.
shells_supported: List[ShellType] = ["zsh", "bash"]
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
shells_supported: List[ShellType] = list(typing_get_args(ShellType))
if conda_path is not None:
# Running in conda; we have to support all shells.
shells_found = shells_supported
CONSOLE.log(f":mag: Detected conda environment {conda_path}!")
else:
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
# Get scripts/ directory.
completions_dir = pathlib.Path(__file__).absolute().parent
@@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None:
else:
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
elif mode == "install":
# Set to True to install completions for scripts as well.
include_scripts = False
# Find tyro CLIs.
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_or_entrypoint_and_shell: _generate_completion(
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
),
itertools.product(script_paths + ENTRYPOINTS, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
_generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found)
else:
assert_never(mode)
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
if conda_path is not None:
# In conda environment we add the completitions activation scripts.
commands = _get_all_entry_points()
_update_conda_scripts(commands, completions_dir, mode)
else:
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
CONSOLE.print("[bold]All done![/bold]")
+70
View File
@@ -287,6 +287,75 @@ class PhototourismDownload(DatasetDownload):
os.remove(download_path)
# credit to https://autonomousvision.github.io/sdfstudio/
# pylint: disable=line-too-long
sdfstudio_downloads = {
"sdfstudio-demo-data": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/sdfstudio-demo-data.tar",
"dtu": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/DTU.tar",
"replica": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Replica.tar",
"scannet": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/scannet.tar",
"tanks-and-temple": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/tnt_advanced.tar",
"tanks-and-temple-highres": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/highresTNT.tar",
"heritage": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Heritage-Recon.tar",
"neural-rgbd-data": "http://kaldir.vc.in.tum.de/neural_rgbd/neural_rgbd_data.zip",
"all": None,
}
SDFstudioCaptureName = tyro.extras.literal_type_from_choices(sdfstudio_downloads.keys())
@dataclass
class SDFstudioDemoDownload(DatasetDownload):
"""Download the sdfstudio dataset."""
dataset_name: SDFstudioCaptureName = "sdfstudio-demo-data"
def download(self, save_dir: Path):
"""Download the D-NeRF dataset (https://github.com/albertpumarola/D-NeRF)."""
# TODO: give this code the same structure as download_nerfstudio
if self.dataset_name == "all":
for dataset_name in sdfstudio_downloads:
if dataset_name != "all":
SDFstudioDemoDownload(dataset_name=dataset_name).download(save_dir)
return
assert (
self.dataset_name in sdfstudio_downloads
), f"Capture name {self.dataset_name} not found in {sdfstudio_downloads.keys()}"
url = sdfstudio_downloads[self.dataset_name]
target_path = str(save_dir / self.dataset_name)
os.makedirs(target_path, exist_ok=True)
file_format = url[-4:]
download_path = Path(f"{target_path}{file_format}")
tmp_path = str(save_dir / ".temp")
shutil.rmtree(tmp_path, ignore_errors=True)
os.makedirs(tmp_path, exist_ok=True)
os.system(f"curl -L {url} > {download_path}")
if file_format == ".tar":
with tarfile.open(download_path, "r") as tar_ref:
tar_ref.extractall(str(tmp_path))
elif file_format == ".zip":
with zipfile.ZipFile(download_path, "r") as zip_ref:
zip_ref.extractall(str(target_path))
return
else:
raise NotImplementedError
inner_folders = os.listdir(tmp_path)
assert len(inner_folders) == 1, "There is more than one folder inside this zip file."
folder = os.path.join(tmp_path, inner_folders[0])
shutil.rmtree(target_path)
shutil.move(folder, target_path)
shutil.rmtree(tmp_path)
os.remove(download_path)
Commands = Union[
Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")],
Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")],
@@ -294,6 +363,7 @@ Commands = Union[
Annotated[Record3dDownload, tyro.conf.subcommand(name="record3d")],
Annotated[DNerfDownload, tyro.conf.subcommand(name="dnerf")],
Annotated[PhototourismDownload, tyro.conf.subcommand(name="phototourism")],
Annotated[SDFstudioDemoDownload, tyro.conf.subcommand(name="sdfstudio")],
]
+1 -1
View File
@@ -737,7 +737,7 @@ class ProcessRealityCapture:
"""Process RealityCapture data into a nerfstudio dataset.
This script assumes that cameras have been aligned using RealityCapture. After alignment, it is necessary to
export the camera poses as a `.csv` file.
export the camera poses as a `.csv` file using the `Internal/External camera parameters` option.
This script does the following:
View File
Whitespace-only changes.
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python
"""
Starts viewer in eval mode.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field, fields
from pathlib import Path
import tyro
from rich.console import Console
from nerfstudio.configs.base_config import ViewerConfig
from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils import writer
from nerfstudio.utils.eval_utils import eval_setup
from nerfstudio.utils.writer import EventName, TimeWriter
from nerfstudio.viewer.server import viewer_utils
CONSOLE = Console(width=120)
@dataclass
class ViewerConfigWithoutNumRays(ViewerConfig):
"""Configuration for viewer instantiation"""
num_rays_per_chunk: tyro.conf.Suppress[int] = -1
start_train: tyro.conf.Suppress[bool] = False
def as_viewer_config(self):
"""Converts the instance to ViewerConfig"""
return ViewerConfig(**{x.name: getattr(self, x.name) for x in fields(self)})
@dataclass
class RunViewer:
"""Load a checkpoint and start the viewer."""
load_config: Path
"""Path to config YAML file."""
viewer: ViewerConfigWithoutNumRays = field(default_factory=ViewerConfigWithoutNumRays)
"""Viewer configuration"""
def main(self) -> None:
"""Main function."""
config, pipeline, _ = eval_setup(
self.load_config,
eval_num_rays_per_chunk=None,
test_mode="test",
)
num_rays_per_chunk = config.viewer.num_rays_per_chunk
assert self.viewer.num_rays_per_chunk == -1
config.vis = "viewer"
config.viewer = self.viewer.as_viewer_config()
config.viewer.num_rays_per_chunk = num_rays_per_chunk
self._start_viewer(config, pipeline)
def _start_viewer(self, config: TrainerConfig, pipeline):
base_dir = config.get_base_dir()
viewer_log_path = base_dir / config.viewer.relative_log_filename
viewer_state, banner_messages = viewer_utils.setup_viewer(
config.viewer, log_filename=viewer_log_path, datapath=config.pipeline.datamanager.dataparser.data
)
# We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
config.logging.local_writer.enable = False
writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages)
assert viewer_state and pipeline.datamanager.train_dataset
viewer_state.init_scene(
dataset=pipeline.datamanager.train_dataset,
start_train=False,
)
while True:
viewer_state.vis["renderingState/isTraining"].write(False)
self._update_viewer_state(viewer_state, config, pipeline)
def _update_viewer_state(self, viewer_state: viewer_utils.ViewerState, config: TrainerConfig, pipeline: Pipeline):
"""Updates the viewer state by rendering out scene with current pipeline
Returns the time taken to render scene.
"""
# NOTE: step must be > 0 otherwise the rendering would not happen
step = 1
num_rays_per_batch = config.pipeline.datamanager.train_num_rays_per_batch
with TimeWriter(writer, EventName.ITER_VIS_TIME) as _:
try:
viewer_state.update_scene(self, step, pipeline.model, num_rays_per_batch)
except RuntimeError:
time.sleep(0.03) # sleep to allow buffer to reset
assert viewer_state.vis is not None
viewer_state.vis["renderingState/log_errors"].write(
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
)
def save_checkpoint(self, *args, **kwargs):
"""
Mock method because we pass this instance to viewer_state.update_scene
"""
def entrypoint():
"""Entrypoint for use with pyproject scripts."""
tyro.extras.set_accent_color("bright_yellow")
tyro.cli(RunViewer).main()
if __name__ == "__main__":
entrypoint()
# For sphinx docs
get_parser_fn = lambda: tyro.extras.get_parser(RunViewer) # noqa
+5 -5
View File
@@ -3,22 +3,21 @@
view_dataset.py
"""
from rich.console import Console
CONSOLE = Console(width=120)
import time
from datetime import timedelta
from pathlib import Path
import torch
import tyro
from rich.console import Console
from nerfstudio.configs.base_config import ViewerConfig
from nerfstudio.data.datamanagers import AnnotatedDataParserUnion
from nerfstudio.data.datamanagers.base_datamanager import AnnotatedDataParserUnion
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.viewer.server import viewer_utils
DEFAULT_TIMEOUT = timedelta(minutes=30)
CONSOLE = Console(width=120)
# speedup for when input size to model doesn't change (much)
torch.backends.cudnn.benchmark = True # type: ignore
@@ -30,9 +29,10 @@ def main(
log_base_dir: Path = Path("/tmp/nerfstudio_viewer_logs"),
) -> None:
"""Main function."""
viewer_state = viewer_utils.ViewerState(
viewer_state, _ = viewer_utils.setup_viewer(
viewer,
log_filename=log_base_dir / viewer.relative_log_filename,
datapath=dataparser.data,
)
dataset = InputDataset(dataparser.setup().get_dataparser_outputs(split="train"))
viewer_state.init_scene(dataset=dataset, start_train=False)