mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-22 15:45:39 +08:00
Merge branch 'main' of https://github.com/nerfstudio-project/nerfstudio
This commit is contained in:
commit
bae8e5cc30
34 files changed
+1081
-144
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "NerfStudio Dev Container",
|
||||
"dockerComposeFile": "docker-compose.yml",
|
||||
"service": "nerfstudio",
|
||||
"workspaceFolder": "/workspace",
|
||||
"shutdownAction": "stopCompose",
|
||||
"extensions": [
|
||||
"mutantdino.resourcemonitor",
|
||||
"ms-azuretools.vscode-docker",
|
||||
"nvidia.nsight-vscode-edition",
|
||||
"ms-python.python"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
nerfstudio:
|
||||
shm_size: '12gb'
|
||||
image: dromni/nerfstudio:0.1.17
|
||||
# build:
|
||||
# context: ..
|
||||
# dockerfile: ./Dockerfile
|
||||
stdin_open: true
|
||||
tty: true
|
||||
environment:
|
||||
NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
|
||||
DISPLAY: $DISPLAY
|
||||
ports:
|
||||
- "7007:7007"
|
||||
volumes:
|
||||
- ../:/workspace/
|
||||
- /tmp/.X11-unix:/tmp/.X11-unix
|
||||
working_dir: /workspace/
|
||||
command: /bin/bash
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
@@ -95,13 +95,25 @@ python -m pip install --upgrade pip
|
||||
|
||||
### Dependencies
|
||||
|
||||
Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
|
||||
Install pytorch with CUDA (this repo has been tested with CUDA 11.3 and CUDA 11.7) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
|
||||
|
||||
For CUDA 11.3:
|
||||
|
||||
```bash
|
||||
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
|
||||
```
|
||||
|
||||
For CUDA 11.7:
|
||||
|
||||
```bash
|
||||
pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117
|
||||
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
|
||||
```
|
||||
|
||||
See [Dependencies](https://github.com/nerfstudio-project/nerfstudio/blob/main/docs/quickstart/installation.md#dependencies)
|
||||
in the Installation documentation for more.
|
||||
|
||||
### Installing nerfstudio
|
||||
|
||||
Easy option:
|
||||
@@ -290,16 +302,18 @@ We provide the following support structures to make life easier for getting star
|
||||
|
||||
# Citation
|
||||
|
||||
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
|
||||
|
||||
If you use this library or find the documentation useful for your research, please consider citing:
|
||||
|
||||
```
|
||||
@misc{nerfstudio,
|
||||
title={Nerfstudio: A Framework for Neural Radiance Field Development},
|
||||
author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi,
|
||||
Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi,
|
||||
Abhik Ahuja, David McAllister, Angjoo Kanazawa},
|
||||
year={2022},
|
||||
url={https://github.com/nerfstudio-project/nerfstudio},
|
||||
@article{nerfstudio,
|
||||
author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi,
|
||||
Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin,
|
||||
Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo},
|
||||
title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development},
|
||||
journal = {arXiv preprint arXiv:2302.04264},
|
||||
year = {2023},
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
+4
-4
@@ -292,9 +292,9 @@
|
||||
"if os.path.exists(f\"data/nerfstudio/{scene}/transforms.json\"):\n",
|
||||
" !ns-train nerfacto --viewer.websocket-port 7007 nerfstudio-data --data data/nerfstudio/$scene --downscale-factor 4\n",
|
||||
"else:\n",
|
||||
" import IPython\n",
|
||||
" display(IPython.display.HTML('<h3 style=\"color:red\">Error: Data processing did not complete</h3>'))\n",
|
||||
" display(IPython.display.HTML('<h3>Please re-run `Downloading and Processing Data`, or view the FAQ for more info.</h3>'))"
|
||||
" from IPython.core.display import display, HTML\n",
|
||||
" display(HTML('<h3 style=\"color:red\">Error: Data processing did not complete</h3>'))\n",
|
||||
" display(HTML('<h3>Please re-run `Downloading and Processing Data`, or view the FAQ for more info.</h3>'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -383,4 +383,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
}
|
||||
@@ -87,7 +87,7 @@ def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
|
||||
You can see our code for more details.
|
||||
|
||||
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/datamanagers.py
|
||||
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/data/datamanagers/base_datamanager.py
|
||||
:color: primary
|
||||
:outline:
|
||||
See the code!
|
||||
|
||||
+9
-7
@@ -155,16 +155,18 @@ We'll be constantly growing this list! So make sure to check back in to see our
|
||||
|
||||
## Citation
|
||||
|
||||
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
|
||||
|
||||
If you use this library or find the documentation useful for your research, please consider citing:
|
||||
|
||||
```none
|
||||
@misc{nerfstudio,
|
||||
title={Nerfstudio: A Framework for Neural Radiance Field Development},
|
||||
author={Matthew Tancik*, Ethan Weber*, Evonne Ng*, Ruilong Li, Brent Yi,
|
||||
Terrance Wang, Alexander Kristoffersen, Jake Austin, Kamyar Salahi,
|
||||
Abhik Ahuja, David McAllister, Angjoo Kanazawa},
|
||||
year={2022},
|
||||
url={https://github.com/nerfstudio-project/nerfstudio},
|
||||
@article{nerfstudio,
|
||||
author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi,
|
||||
Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin,
|
||||
Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo},
|
||||
title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development},
|
||||
journal = {arXiv preprint arXiv:2302.04264},
|
||||
year = {2023},
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ Navigating to the link at the end of the terminal will load the webviewer. If yo
|
||||
- All data configurations must go at the end. In this case, `nerfstudio-data` and all of its corresponding configurations come at the end after the model and viewer specification.
|
||||
:::
|
||||
|
||||
## Resume from checkpoint / visualize existing run
|
||||
## Resume from checkpoint
|
||||
|
||||
It is possible to load a pretrained model by running
|
||||
|
||||
@@ -40,6 +40,14 @@ ns-train nerfacto --data data/nerfstudio/poster --load-dir {outputs/.../nerfstud
|
||||
|
||||
This will automatically start training. If you do not want it to train, add `--viewer.start-train False` to your training command.
|
||||
|
||||
## Visualize existing run
|
||||
|
||||
Given a pretrained model checkpoint, you can start the viewer by running
|
||||
|
||||
```bash
|
||||
ns-viewer --load-config {outputs/.../config.yml}
|
||||
```
|
||||
|
||||
## Exporting Results
|
||||
|
||||
Once you have a NeRF model you can either render out a video or export a point cloud.
|
||||
|
||||
@@ -17,12 +17,38 @@ python -m pip install --upgrade pip
|
||||
|
||||
## Dependencies
|
||||
|
||||
Install pytorch with CUDA (this repo has been tested with CUDA 11.3) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
|
||||
(pytorch)=
|
||||
### pytorch
|
||||
|
||||
Either pytorch 1.12.1 (with CUDA up to 11.3) or 1.13.1 (with CUDA 11.6/7/8) can be used.
|
||||
|
||||
- To install 1.12.1 with CUDA 11.3:
|
||||
|
||||
```bash
|
||||
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
|
||||
```
|
||||
|
||||
- To install 1.13.1 with CUDA 11.7:
|
||||
|
||||
Note that if a pytorch version prior to 1.13 is installed,
|
||||
it should be uninstalled first to avoid upgrade issues (e.g. with functorch)
|
||||
|
||||
```bash
|
||||
pip uninstall torch torchvision functorch
|
||||
```
|
||||
|
||||
Install pytorch 1.13.1 with CUDA (this repo has been tested with CUDA 11.7.1) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)
|
||||
|
||||
```bash
|
||||
pip install torch torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117
|
||||
```
|
||||
|
||||
### tinycudann
|
||||
|
||||
After pytorch, install the torch bindings for tinycudann:
|
||||
|
||||
```bash
|
||||
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
|
||||
```
|
||||
|
||||
## Installing nerfstudio
|
||||
@@ -134,9 +160,8 @@ While installing tiny-cuda, you run into: `The detected CUDA version mismatches
|
||||
|
||||
**Solution**:
|
||||
|
||||
```
|
||||
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
```
|
||||
Reinstall pytorch with the correct CUDA version.
|
||||
See [pytorch](pytorch) under Dependencies, above.
|
||||
|
||||
<br />
|
||||
|
||||
|
||||
@@ -100,12 +100,16 @@ def get_spiral_path(
|
||||
new_c2ws.append(c2wh[:3, :4])
|
||||
new_c2ws = torch.stack(new_c2ws, dim=0)
|
||||
|
||||
times = None
|
||||
if camera.times is not None:
|
||||
times = torch.linspace(0, 1, steps)[:, None]
|
||||
return Cameras(
|
||||
fx=camera.fx[0],
|
||||
fy=camera.fy[0],
|
||||
cx=camera.cx[0],
|
||||
cy=camera.cy[0],
|
||||
camera_to_worlds=new_c2ws,
|
||||
times=times,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -230,7 +230,7 @@ class Cameras(TensorDataclass):
|
||||
c_x_y: cx or cy for when h_w == None
|
||||
"""
|
||||
if isinstance(h_w, int):
|
||||
h_w = torch.Tensor([h_w]).to(torch.int64).to(self.device)
|
||||
h_w = torch.as_tensor([h_w]).to(torch.int64).to(self.device)
|
||||
elif isinstance(h_w, torch.Tensor):
|
||||
assert not torch.is_floating_point(h_w), f"height and width tensor must be of type int, not: {h_w.dtype}"
|
||||
h_w = h_w.to(torch.int64).to(self.device)
|
||||
@@ -238,7 +238,7 @@ class Cameras(TensorDataclass):
|
||||
h_w = h_w.unsqueeze(-1)
|
||||
# assert torch.all(h_w == h_w.view(-1)[0]), "Batched cameras of different h, w will be allowed in the future."
|
||||
elif h_w is None:
|
||||
h_w = torch.Tensor((c_x_y * 2).to(torch.int64).to(self.device))
|
||||
h_w = torch.as_tensor((c_x_y * 2)).to(torch.int64).to(self.device)
|
||||
else:
|
||||
raise ValueError("Height must be an int, tensor, or None, received: " + str(type(h_w)))
|
||||
return h_w
|
||||
|
||||
@@ -49,6 +49,7 @@ from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserCo
|
||||
from nerfstudio.data.dataparsers.phototourism_dataparser import (
|
||||
PhototourismDataParserConfig,
|
||||
)
|
||||
from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler
|
||||
from nerfstudio.data.utils.dataloaders import (
|
||||
@@ -75,6 +76,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of
|
||||
"dnerf-data": DNeRFDataParserConfig(),
|
||||
"phototourism-data": PhototourismDataParserConfig(),
|
||||
"dycheck-data": DycheckDataParserConfig(),
|
||||
"sdfstudio-data": SDFStudioDataParserConfig(),
|
||||
},
|
||||
prefix_names=False, # Omit prefixes in subcommands themselves.
|
||||
)
|
||||
|
||||
@@ -123,6 +123,10 @@ class InstantNGP(DataParser):
|
||||
|
||||
w, h = meta["w"], meta["h"]
|
||||
|
||||
camera_type = CameraType.PERSPECTIVE
|
||||
if meta.get("is_fisheye", False):
|
||||
camera_type = CameraType.FISHEYE
|
||||
|
||||
cameras = Cameras(
|
||||
fx=float(fl_x),
|
||||
fy=float(fl_y),
|
||||
@@ -132,7 +136,7 @@ class InstantNGP(DataParser):
|
||||
height=int(h),
|
||||
width=int(w),
|
||||
camera_to_worlds=camera_to_world,
|
||||
camera_type=CameraType.PERSPECTIVE,
|
||||
camera_type=camera_type,
|
||||
)
|
||||
|
||||
# TODO(ethan): add alpha background color
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Datapaser for sdfstudio formatted data"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Type
|
||||
|
||||
import torch
|
||||
from rich.console import Console
|
||||
|
||||
from nerfstudio.cameras import camera_utils
|
||||
from nerfstudio.cameras.cameras import Cameras, CameraType
|
||||
from nerfstudio.data.dataparsers.base_dataparser import (
|
||||
DataParser,
|
||||
DataParserConfig,
|
||||
DataparserOutputs,
|
||||
)
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.utils.io import load_from_json
|
||||
|
||||
CONSOLE = Console()
|
||||
|
||||
|
||||
@dataclass
|
||||
class SDFStudioDataParserConfig(DataParserConfig):
|
||||
"""Scene dataset parser config"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: SDFStudio)
|
||||
"""target class to instantiate"""
|
||||
data: Path = Path("data/DTU/scan65")
|
||||
"""Directory specifying location of data."""
|
||||
include_mono_prior: bool = False
|
||||
"""whether or not to load monocular depth and normal """
|
||||
include_foreground_mask: bool = False
|
||||
"""whether or not to load foreground mask"""
|
||||
downscale_factor: int = 1
|
||||
scene_scale: float = 2.0
|
||||
"""
|
||||
Sets the bounding cube to have edge length of this size.
|
||||
The longest dimension of the Friends axis-aligned bbox will be scaled to this value.
|
||||
"""
|
||||
skip_every_for_val_split: int = 1
|
||||
"""sub sampling validation images"""
|
||||
auto_orient: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class SDFStudio(DataParser):
|
||||
"""SDFStudio Dataset"""
|
||||
|
||||
config: SDFStudioDataParserConfig
|
||||
|
||||
def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements
|
||||
# load meta data
|
||||
meta = load_from_json(self.config.data / "meta_data.json")
|
||||
|
||||
indices = list(range(len(meta["frames"])))
|
||||
# subsample to avoid out-of-memory for validation set
|
||||
if split != "train" and self.config.skip_every_for_val_split >= 1:
|
||||
indices = indices[:: self.config.skip_every_for_val_split]
|
||||
|
||||
image_filenames = []
|
||||
depth_filenames = []
|
||||
normal_filenames = []
|
||||
transform = None
|
||||
fx = []
|
||||
fy = []
|
||||
cx = []
|
||||
cy = []
|
||||
camera_to_worlds = []
|
||||
for i, frame in enumerate(meta["frames"]):
|
||||
if i not in indices:
|
||||
continue
|
||||
|
||||
image_filename = self.config.data / frame["rgb_path"]
|
||||
depth_filename = self.config.data / frame["mono_depth_path"]
|
||||
normal_filename = self.config.data / frame["mono_normal_path"]
|
||||
|
||||
intrinsics = torch.tensor(frame["intrinsics"])
|
||||
camtoworld = torch.tensor(frame["camtoworld"])
|
||||
|
||||
# append data
|
||||
image_filenames.append(image_filename)
|
||||
depth_filenames.append(depth_filename)
|
||||
normal_filenames.append(normal_filename)
|
||||
fx.append(intrinsics[0, 0])
|
||||
fy.append(intrinsics[1, 1])
|
||||
cx.append(intrinsics[0, 2])
|
||||
cy.append(intrinsics[1, 2])
|
||||
camera_to_worlds.append(camtoworld)
|
||||
|
||||
fx = torch.stack(fx)
|
||||
fy = torch.stack(fy)
|
||||
cx = torch.stack(cx)
|
||||
cy = torch.stack(cy)
|
||||
camera_to_worlds = torch.stack(camera_to_worlds)
|
||||
|
||||
# Convert from COLMAP's/OPENCV's camera coordinate system to nerfstudio
|
||||
camera_to_worlds[:, 0:3, 1:3] *= -1
|
||||
|
||||
if self.config.auto_orient:
|
||||
camera_to_worlds, transform = camera_utils.auto_orient_and_center_poses(
|
||||
camera_to_worlds,
|
||||
method="up",
|
||||
center_poses=False,
|
||||
)
|
||||
|
||||
# scene box from meta data
|
||||
meta_scene_box = meta["scene_box"]
|
||||
aabb = torch.tensor(meta_scene_box["aabb"], dtype=torch.float32)
|
||||
scene_box = SceneBox(
|
||||
aabb=aabb,
|
||||
)
|
||||
|
||||
height, width = meta["height"], meta["width"]
|
||||
cameras = Cameras(
|
||||
fx=fx,
|
||||
fy=fy,
|
||||
cx=cx,
|
||||
cy=cy,
|
||||
height=height,
|
||||
width=width,
|
||||
camera_to_worlds=camera_to_worlds[:, :3, :4],
|
||||
camera_type=CameraType.PERSPECTIVE,
|
||||
)
|
||||
|
||||
# TODO supports downsample
|
||||
# cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor)
|
||||
|
||||
assert meta["has_mono_prior"] == self.config.include_mono_prior, f"no mono prior in {self.config.data}"
|
||||
|
||||
dataparser_outputs = DataparserOutputs(
|
||||
image_filenames=image_filenames,
|
||||
cameras=cameras,
|
||||
scene_box=scene_box,
|
||||
metadata={
|
||||
"depth_filenames": depth_filenames if len(depth_filenames) > 0 else None,
|
||||
"normal_filenames": normal_filenames if len(normal_filenames) > 0 else None,
|
||||
"transform": transform,
|
||||
"camera_to_worlds": camera_to_worlds if len(camera_to_worlds) > 0 else None,
|
||||
"include_mono_prior": self.config.include_mono_prior,
|
||||
},
|
||||
)
|
||||
return dataparser_outputs
|
||||
@@ -0,0 +1,97 @@
|
||||
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
SDFStudio dataset.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
|
||||
|
||||
class SDFDataset(InputDataset):
|
||||
"""Dataset that returns images and depths.
|
||||
|
||||
Args:
|
||||
dataparser_outputs: description of where and how to read input images.
|
||||
scale_factor: The scaling factor for the dataparser outputs.
|
||||
"""
|
||||
|
||||
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
|
||||
super().__init__(dataparser_outputs, scale_factor)
|
||||
|
||||
# can be none if monoprior not included
|
||||
self.depth_filenames = self.metadata["depth_filenames"]
|
||||
self.normal_filenames = self.metadata["normal_filenames"]
|
||||
self.camera_to_worlds = self.metadata["camera_to_worlds"]
|
||||
# can be none if auto orient not enabled in dataparser
|
||||
self.transform = self.metadata["normal_filenames"]
|
||||
self.include_mono_prior = self.metadata["include_mono_prior"]
|
||||
|
||||
def get_metadata(self, data: Dict) -> Dict:
|
||||
# TODO supports foreground_masks
|
||||
metadata = {}
|
||||
if self.include_mono_prior:
|
||||
depth_filepath = self.depth_filenames[data["image_idx"]]
|
||||
normal_filepath = self.normal_filenames[data["image_idx"]]
|
||||
camtoworld = self.camera_to_worlds[data["image_idx"]]
|
||||
|
||||
# Scale depth images to meter units and also by scaling applied to cameras
|
||||
depth_image, normal_image = self.get_depths_and_normals(
|
||||
depth_filepath=depth_filepath, normal_filename=normal_filepath, camtoworld=camtoworld
|
||||
)
|
||||
metadata["depth_image"] = depth_image
|
||||
metadata["normal_image"] = normal_image
|
||||
|
||||
return metadata
|
||||
|
||||
def get_depths_and_normals(self, depth_filepath: Path, normal_filename: Path, camtoworld: np.ndarray):
|
||||
"""function to process additional depths and normal information
|
||||
Args:
|
||||
depth_filepath: path to depth file
|
||||
normal_filename: path to normal file
|
||||
camtoworld: camera to world transformation matrix
|
||||
"""
|
||||
|
||||
# load mono depth
|
||||
depth = np.load(depth_filepath)
|
||||
depth = torch.from_numpy(depth).float()
|
||||
|
||||
# load mono normal
|
||||
normal = np.load(normal_filename)
|
||||
|
||||
# transform normal to world coordinate system
|
||||
normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here
|
||||
normal = torch.from_numpy(normal).float()
|
||||
|
||||
rot = camtoworld[:3, :3]
|
||||
|
||||
normal_map = normal.reshape(3, -1)
|
||||
normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0)
|
||||
|
||||
normal_map = rot @ normal_map
|
||||
normal = normal_map.permute(1, 0).reshape(*normal.shape[1:], 3)
|
||||
|
||||
if self.transform is not None:
|
||||
h, w, _ = normal.shape
|
||||
normal = self.transform[:3, :3] @ normal.reshape(-1, 3).permute(1, 0)
|
||||
normal = normal.permute(1, 0).reshape(h, w, 3)
|
||||
|
||||
return depth, normal
|
||||
@@ -116,7 +116,6 @@ class TemporalGridEncodeFunc(Function):
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, grad):
|
||||
|
||||
inputs, temporal_row_index, embeddings, offsets, dy_dx = ctx.saved_tensors
|
||||
B, D, grid_channel, C, L, S, H, gridtype = ctx.dims
|
||||
align_corners = ctx.align_corners
|
||||
@@ -336,7 +335,7 @@ class TemporalGridEncoder(nn.Module):
|
||||
"""
|
||||
outputs = TemporalGridEncodeFunc.apply(
|
||||
xyz,
|
||||
self.get_temporal_index(time[:, 0]),
|
||||
self.get_temporal_index(time[:, 0].float()),
|
||||
self.embeddings,
|
||||
self.offsets,
|
||||
self.per_level_scale,
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import appdirs
|
||||
import mediapy
|
||||
@@ -29,6 +29,8 @@ import torch.nn.functional as F
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
from torch import nn
|
||||
from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
from torchtyping import TensorType
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
@@ -58,6 +60,26 @@ class UNet2DConditionOutput:
|
||||
sample: torch.FloatTensor
|
||||
|
||||
|
||||
class _SDSGradient(torch.autograd.Function): # pylint: disable=abstract-method
|
||||
"""Custom gradient function for SDS loss. Since it is already computed, we can just return it."""
|
||||
|
||||
@staticmethod
|
||||
@custom_fwd
|
||||
def forward(ctx, input_tensor, gt_grad): # pylint: disable=arguments-differ
|
||||
del input_tensor
|
||||
ctx.save_for_backward(gt_grad)
|
||||
# Return magniture of gradient, not the actual loss.
|
||||
return torch.mean(gt_grad**2) ** 0.5
|
||||
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, grad): # pylint: disable=arguments-differ
|
||||
del grad
|
||||
(gt_grad,) = ctx.saved_tensors
|
||||
batch_size = len(gt_grad)
|
||||
return gt_grad / batch_size, None
|
||||
|
||||
|
||||
class StableDiffusion(nn.Module):
|
||||
"""Stable Diffusion implementation
|
||||
Args:
|
||||
@@ -162,14 +184,16 @@ class StableDiffusion(nn.Module):
|
||||
text_embeddings: TensorType["N", "max_length", "embed_dim"],
|
||||
image: TensorType["BS", 3, "H", "W"],
|
||||
guidance_scale: float = 100.0,
|
||||
) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]:
|
||||
grad_scaler: Optional[GradScaler] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/)
|
||||
Args:
|
||||
text_embeddings: Text embeddings
|
||||
image: Rendered image
|
||||
guidance_scale: How much to weigh the guidance
|
||||
grad_scaler: Grad scaler
|
||||
Returns:
|
||||
A tuple of Loss, latent, and gradient
|
||||
The loss
|
||||
"""
|
||||
image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear")
|
||||
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
|
||||
@@ -186,7 +210,7 @@ class StableDiffusion(nn.Module):
|
||||
|
||||
# perform guidance
|
||||
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
||||
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
|
||||
# w(t), sigma_t^2
|
||||
w = 1 - self.alphas[t]
|
||||
@@ -194,9 +218,11 @@ class StableDiffusion(nn.Module):
|
||||
grad = w * (noise_pred - noise)
|
||||
grad = torch.nan_to_num(grad)
|
||||
|
||||
sds_mse = torch.mean(torch.nan_to_num(torch.square(noise_pred - noise)))
|
||||
if grad_scaler is not None:
|
||||
latents = grad_scaler.scale(latents)
|
||||
loss = _SDSGradient.apply(latents, grad)
|
||||
|
||||
return sds_mse, latents, grad
|
||||
return loss
|
||||
|
||||
def produce_latents(
|
||||
self,
|
||||
@@ -240,7 +266,7 @@ class StableDiffusion(nn.Module):
|
||||
|
||||
# perform guidance
|
||||
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
||||
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore
|
||||
|
||||
@@ -20,8 +20,10 @@ from enum import Enum
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.utils.math import masked_reduction
|
||||
|
||||
L1Loss = nn.L1Loss
|
||||
MSELoss = nn.MSELoss
|
||||
@@ -308,3 +310,218 @@ def depth_loss(
|
||||
return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma)
|
||||
|
||||
raise NotImplementedError("Provided depth loss type not implemented.")
|
||||
|
||||
|
||||
def monosdf_normal_loss(
|
||||
normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
|
||||
Enforces consistency between the volume rendered normal and the predicted monocular normal.
|
||||
With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf
|
||||
Args:
|
||||
normal_pred: volume rendered normal
|
||||
normal_gt: monocular normal
|
||||
"""
|
||||
normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1)
|
||||
normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1)
|
||||
l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean()
|
||||
cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean()
|
||||
return l1 + cos
|
||||
|
||||
|
||||
class MiDaSMSELoss(nn.Module):
|
||||
"""
|
||||
data term from MiDaS paper
|
||||
"""
|
||||
|
||||
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
super().__init__()
|
||||
|
||||
self.reduction_type = reduction_type
|
||||
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
|
||||
self.mse_loss = MSELoss(reduction="none")
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
mse loss based on reduction function
|
||||
"""
|
||||
summed_mask = torch.sum(mask, (1, 2))
|
||||
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
|
||||
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
|
||||
|
||||
return image_loss
|
||||
|
||||
|
||||
# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py
|
||||
class GradientLoss(nn.Module):
|
||||
"""
|
||||
multiscale, scale-invariant gradient matching term to the disparity space.
|
||||
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
|
||||
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
|
||||
"""
|
||||
|
||||
def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
"""
|
||||
Args:
|
||||
scales: number of scales to use
|
||||
reduction_type: either "batch" or "image"
|
||||
"""
|
||||
super().__init__()
|
||||
self.reduction_type = reduction_type
|
||||
self.__scales = scales
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
gradient loss based on reduction function
|
||||
"""
|
||||
total = 0
|
||||
|
||||
for scale in range(self.__scales):
|
||||
step = pow(2, scale)
|
||||
|
||||
grad_loss = self.gradient_loss(
|
||||
prediction[:, ::step, ::step],
|
||||
target[:, ::step, ::step],
|
||||
mask[:, ::step, ::step],
|
||||
)
|
||||
total += grad_loss
|
||||
|
||||
return total
|
||||
|
||||
def gradient_loss(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
multiscale, scale-invariant gradient matching term to the disparity space.
|
||||
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
|
||||
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
reduction: reduction function, either reduction_batch_based or reduction_image_based
|
||||
Returns:
|
||||
gradient loss based on reduction function
|
||||
"""
|
||||
summed_mask = torch.sum(mask, (1, 2))
|
||||
diff = prediction - target
|
||||
diff = torch.mul(mask, diff)
|
||||
|
||||
grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1])
|
||||
mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1])
|
||||
grad_x = torch.mul(mask_x, grad_x)
|
||||
|
||||
grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :])
|
||||
mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :])
|
||||
grad_y = torch.mul(mask_y, grad_y)
|
||||
|
||||
image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2))
|
||||
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
|
||||
|
||||
return image_loss
|
||||
|
||||
|
||||
class ScaleAndShiftInvariantLoss(nn.Module):
|
||||
"""
|
||||
Scale and shift invariant loss as described in
|
||||
"Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer"
|
||||
https://arxiv.org/pdf/1907.01341.pdf
|
||||
"""
|
||||
|
||||
def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
"""
|
||||
Args:
|
||||
alpha: weight of the regularization term
|
||||
scales: number of scales to use
|
||||
reduction_type: either "batch" or "image"
|
||||
"""
|
||||
super().__init__()
|
||||
self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type)
|
||||
self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type)
|
||||
self.__alpha = alpha
|
||||
|
||||
self.__prediction_ssi = None
|
||||
|
||||
@classmethod
|
||||
def compute_scale_and_shift(
|
||||
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
):
|
||||
"""
|
||||
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
|
||||
This function computes scale/shift required to normalizes predicted depth map,
|
||||
to allow for using normalized depth maps as input from monocular depth estimation networks.
|
||||
These networks are trained such that they predict normalized depth maps.
|
||||
|
||||
Solves for scale/shift using a least squares approach with a closed form solution:
|
||||
Based on:
|
||||
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
scale and shift for depth prediction
|
||||
"""
|
||||
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
|
||||
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
|
||||
a_01 = torch.sum(mask * prediction, (1, 2))
|
||||
a_11 = torch.sum(mask, (1, 2))
|
||||
|
||||
# right hand side: b = [b_0, b_1]
|
||||
b_0 = torch.sum(mask * prediction * target, (1, 2))
|
||||
b_1 = torch.sum(mask * target, (1, 2))
|
||||
|
||||
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
|
||||
scale = torch.zeros_like(b_0)
|
||||
shift = torch.zeros_like(b_1)
|
||||
|
||||
det = a_00 * a_11 - a_01 * a_01
|
||||
valid = det.nonzero()
|
||||
|
||||
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
|
||||
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
|
||||
|
||||
return scale, shift
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Args:
|
||||
prediction: predicted depth map (unnormalized)
|
||||
target: ground truth depth map (normalized)
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
scale and shift invariant loss
|
||||
"""
|
||||
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
|
||||
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
|
||||
|
||||
total = self.__data_loss(self.__prediction_ssi, target, mask)
|
||||
if self.__alpha > 0:
|
||||
total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask)
|
||||
|
||||
return total
|
||||
|
||||
def __get_prediction_ssi(self):
|
||||
"""
|
||||
scale and shift invariant prediction
|
||||
from https://arxiv.org/pdf/1907.01341.pdf equation 1
|
||||
"""
|
||||
return self.__prediction_ssi
|
||||
|
||||
prediction_ssi = property(__get_prediction_ssi)
|
||||
@@ -509,7 +509,16 @@ class VolumetricSampler(Sampler):
|
||||
|
||||
|
||||
class ProposalNetworkSampler(Sampler):
|
||||
"""Sampler that uses a proposal network to generate samples."""
|
||||
"""Sampler that uses a proposal network to generate samples.
|
||||
|
||||
Args:
|
||||
num_proposal_samples_per_ray: Number of samples to generate per ray for each proposal step.
|
||||
num_nerf_samples_per_ray: Number of samples to generate per ray for the NERF model.
|
||||
num_proposal_network_iterations: Number of proposal network iterations to run.
|
||||
single_jitter: Use a same random jitter for all samples along a ray.
|
||||
update_sched: A function that takes the iteration number of steps between updates.
|
||||
initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -518,6 +527,7 @@ class ProposalNetworkSampler(Sampler):
|
||||
num_proposal_network_iterations: int = 2,
|
||||
single_jitter: bool = False,
|
||||
update_sched: Callable = lambda x: 1,
|
||||
initial_sampler: Optional[Sampler] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_proposal_samples_per_ray = num_proposal_samples_per_ray
|
||||
@@ -528,7 +538,10 @@ class ProposalNetworkSampler(Sampler):
|
||||
raise ValueError("num_proposal_network_iterations must be >= 1")
|
||||
|
||||
# samplers
|
||||
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
|
||||
if initial_sampler is None:
|
||||
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
|
||||
else:
|
||||
self.initial_sampler = initial_sampler
|
||||
self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter)
|
||||
|
||||
self._anneal = 1.0
|
||||
|
||||
@@ -137,7 +137,7 @@ class NGPModel(Model):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
|
||||
@@ -86,7 +86,7 @@ class MipNerfModel(Model):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
@@ -173,8 +173,8 @@ class MipNerfModel(Model):
|
||||
image = torch.moveaxis(image, -1, 0)[None, ...]
|
||||
rgb_coarse = torch.moveaxis(rgb_coarse, -1, 0)[None, ...]
|
||||
rgb_fine = torch.moveaxis(rgb_fine, -1, 0)[None, ...]
|
||||
rgb_coarse = torch.clip(rgb_coarse, min=-1, max=1)
|
||||
rgb_fine = torch.clip(rgb_fine, min=-1, max=1)
|
||||
rgb_coarse = torch.clip(rgb_coarse, min=0, max=1)
|
||||
rgb_fine = torch.clip(rgb_fine, min=0, max=1)
|
||||
|
||||
coarse_psnr = self.psnr(image, rgb_coarse)
|
||||
fine_psnr = self.psnr(image, rgb_fine)
|
||||
|
||||
@@ -197,7 +197,7 @@ class NerfactoModel(Model):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
|
||||
@@ -168,7 +168,7 @@ class NerfplayerNerfactoModel(NerfactoModel):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
self.temporal_distortion = True # for viewer
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
|
||||
@@ -145,7 +145,7 @@ class NerfplayerNGPModel(NGPModel):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
self.temporal_distortion = True # for viewer
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
|
||||
@@ -137,7 +137,7 @@ class SemanticNerfWModel(Model):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
|
||||
@@ -201,7 +201,7 @@ class TensoRFModel(Model):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
# colliders
|
||||
if self.config.enable_collider:
|
||||
|
||||
@@ -118,7 +118,7 @@ class NeRFModel(Model):
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
if getattr(self.config, "enable_temporal_distortion", False):
|
||||
params = self.config.temporal_distortion_params
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import Tuple
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
_USE_NERFACC = True
|
||||
|
||||
@@ -285,3 +286,28 @@ def safe_normalize(
|
||||
Normalized vectors.
|
||||
"""
|
||||
return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps)
|
||||
|
||||
|
||||
def masked_reduction(
|
||||
input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
|
||||
):
|
||||
"""
|
||||
Whether to consolidate the input_tensor across the batch or across the image
|
||||
Args:
|
||||
input_tensor: input tensor
|
||||
mask: mask tensor
|
||||
reduction_type: either "batch" or "image"
|
||||
Returns:
|
||||
input_tensor: reduced input_tensor
|
||||
"""
|
||||
if reduction_type == "batch":
|
||||
# avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0)
|
||||
divisor = torch.sum(mask)
|
||||
input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor
|
||||
elif reduction_type == "image":
|
||||
# avoid division by 0 (if M = sum(mask) = 0: image_loss = 0)
|
||||
valid = mask.nonzero()
|
||||
|
||||
input_tensor[valid] = input_tensor[valid] / mask[valid]
|
||||
input_tensor = torch.mean(input_tensor)
|
||||
return input_tensor
|
||||
+5
-4
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "nerfstudio"
|
||||
version = "0.1.16"
|
||||
version = "0.1.17"
|
||||
description = "All-in-one repository for state-of-the-art NeRFs"
|
||||
readme = "README.md"
|
||||
license = { text="Apache 2.0"}
|
||||
@@ -22,7 +22,7 @@ dependencies = [
|
||||
"tyro>=0.3.31",
|
||||
"gdown>=4.6.0",
|
||||
"ninja>=1.10",
|
||||
"functorch==0.2.1",
|
||||
"functorch>=0.2.1",
|
||||
"h5py>=2.9.0",
|
||||
"imageio>=2.21.1",
|
||||
'importlib-metadata>=6.0.0; python_version < "3.10"',
|
||||
@@ -32,7 +32,7 @@ dependencies = [
|
||||
"mediapy>=1.1.0",
|
||||
"msgpack>=1.0.4",
|
||||
"msgpack_numpy>=0.4.8",
|
||||
"nerfacc==0.3.4",
|
||||
"nerfacc==0.3.3",
|
||||
"open3d>=0.16.0",
|
||||
"opencv-python==4.6.0.66",
|
||||
"Pillow>=9.3.0",
|
||||
@@ -45,7 +45,7 @@ dependencies = [
|
||||
"rich>=12.5.1",
|
||||
"scikit-image>=0.19.3",
|
||||
"tensorboard==2.9.0",
|
||||
"torch==1.12.1",
|
||||
"torch>=1.12.1",
|
||||
"torchmetrics[image]>=0.9.3",
|
||||
"torchtyping>=0.1.4",
|
||||
"torchvision>=0.13.0",
|
||||
@@ -100,6 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint"
|
||||
ns-process-data = "scripts.process_data:entrypoint"
|
||||
ns-download-data = "scripts.downloads.download_data:entrypoint"
|
||||
ns-train = "scripts.train:entrypoint"
|
||||
ns-viewer = "scripts.viewer.run_viewer:entrypoint"
|
||||
ns-eval = "scripts.eval:entrypoint"
|
||||
ns-render = "scripts.render:entrypoint"
|
||||
ns-export = "scripts.exporter:entrypoint"
|
||||
|
||||
+199
-84
@@ -10,27 +10,29 @@ import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import List, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
from rich.prompt import Confirm
|
||||
from typing_extensions import Literal, assert_never
|
||||
from typing_extensions import get_args as typing_get_args
|
||||
|
||||
if sys.version_info < (3, 10):
|
||||
import importlib_metadata
|
||||
else:
|
||||
from importlib import metadata as importlib_metadata
|
||||
|
||||
ConfigureMode = Literal["install", "uninstall"]
|
||||
ShellType = Literal["zsh", "bash"]
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
HEADER_LINE = "# Source nerfstudio autocompletions."
|
||||
|
||||
ENTRYPOINTS = [
|
||||
"ns-install-cli",
|
||||
"ns-process-data",
|
||||
"ns-download-data",
|
||||
"ns-train",
|
||||
"ns-eval",
|
||||
"ns-render",
|
||||
"ns-dev-test",
|
||||
]
|
||||
|
||||
def _get_all_entry_points() -> List[str]:
|
||||
entry_points = importlib_metadata.distribution("nerfstudio").entry_points
|
||||
return [x.name for x in entry_points]
|
||||
|
||||
|
||||
def _check_tyro_cli(script_path: pathlib.Path) -> bool:
|
||||
@@ -127,6 +129,62 @@ def _exclamation() -> str:
|
||||
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
|
||||
|
||||
|
||||
def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str:
|
||||
if shell is None:
|
||||
# Install the universal script
|
||||
result_script = []
|
||||
for shell_type in typing_get_args(ShellType):
|
||||
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
|
||||
result_script.append(_get_deactivate_script(commands, shell_type, add_header=False))
|
||||
result_script.append("fi")
|
||||
source_lines = "\n".join(result_script)
|
||||
|
||||
elif shell == "zsh":
|
||||
source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands])
|
||||
elif shell == "bash":
|
||||
source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands])
|
||||
else:
|
||||
assert_never(shell)
|
||||
|
||||
if add_header:
|
||||
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
|
||||
return source_lines
|
||||
|
||||
|
||||
def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str:
|
||||
if shell is None:
|
||||
# Install the universal script
|
||||
result_script = []
|
||||
for shell_type in typing_get_args(ShellType):
|
||||
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
|
||||
result_script.append(_get_source_script(completions_dir, shell_type, add_header=False))
|
||||
result_script.append("fi")
|
||||
source_lines = "\n".join(result_script)
|
||||
|
||||
elif shell == "zsh":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
"if ! command -v compdef &> /dev/null; then",
|
||||
" autoload -Uz compinit",
|
||||
" compinit",
|
||||
"fi",
|
||||
f"source {completions_dir / 'setup.zsh'}",
|
||||
]
|
||||
)
|
||||
elif shell == "bash":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
f"source {completions_dir / 'setup.bash'}",
|
||||
]
|
||||
)
|
||||
else:
|
||||
assert_never(shell)
|
||||
|
||||
if add_header:
|
||||
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
|
||||
return source_lines
|
||||
|
||||
|
||||
def _update_rc(
|
||||
completions_dir: pathlib.Path,
|
||||
mode: ConfigureMode,
|
||||
@@ -141,36 +199,13 @@ def _update_rc(
|
||||
"""
|
||||
|
||||
# Install or uninstall `source_line`.
|
||||
header_line = "# Source nerfstudio autocompletions."
|
||||
if shell == "zsh":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
"",
|
||||
header_line,
|
||||
"if ! command -v compdef &> /dev/null; then",
|
||||
" autoload -Uz compinit",
|
||||
" compinit",
|
||||
"fi",
|
||||
f"source {completions_dir / 'setup.zsh'}",
|
||||
]
|
||||
)
|
||||
elif shell == "bash":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
"",
|
||||
header_line,
|
||||
f"source {completions_dir / 'setup.bash'}",
|
||||
]
|
||||
)
|
||||
else:
|
||||
assert_never(shell)
|
||||
|
||||
source_lines = _get_source_script(completions_dir, shell)
|
||||
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
|
||||
|
||||
# Always try to uninstall previous completions.
|
||||
rc_source = rc_path.read_text()
|
||||
while header_line in rc_source:
|
||||
before_install, _, after_install = rc_source.partition(header_line)
|
||||
while HEADER_LINE in rc_source:
|
||||
before_install, _, after_install = rc_source.partition(HEADER_LINE)
|
||||
source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n")
|
||||
assert source_file.endswith(f"/completions/setup.{shell}")
|
||||
rc_source = before_install + after_install
|
||||
@@ -189,6 +224,108 @@ def _update_rc(
|
||||
assert mode == "uninstall"
|
||||
|
||||
|
||||
def _update_conda_scripts(
|
||||
commands: List[str],
|
||||
completions_dir: pathlib.Path,
|
||||
mode: ConfigureMode,
|
||||
) -> None:
|
||||
"""Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts.
|
||||
|
||||
Args:
|
||||
completions_dir: Path to location of this script.
|
||||
mode: Install or uninstall completions.
|
||||
"""
|
||||
|
||||
# Install or uninstall `source_line`.
|
||||
activate_source_lines = _get_source_script(completions_dir, None)
|
||||
deactivate_source_lines = _get_deactivate_script(commands, None)
|
||||
|
||||
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
|
||||
activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh"
|
||||
deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh"
|
||||
if mode == "uninstall":
|
||||
if activate_path.exists():
|
||||
os.remove(activate_path)
|
||||
if deactivate_path.exists():
|
||||
os.remove(deactivate_path)
|
||||
CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.")
|
||||
elif mode == "install":
|
||||
# Install completions.
|
||||
activate_path.parent.mkdir(exist_ok=True, parents=True)
|
||||
deactivate_path.parent.mkdir(exist_ok=True, parents=True)
|
||||
with activate_path.open("w+", encoding="utf8") as f:
|
||||
f.write(activate_source_lines)
|
||||
with deactivate_path.open("w+", encoding="utf8") as f:
|
||||
f.write(deactivate_source_lines)
|
||||
CONSOLE.log(
|
||||
f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment"
|
||||
" to try them out."
|
||||
)
|
||||
else:
|
||||
assert_never(mode)
|
||||
|
||||
|
||||
def _get_conda_path() -> Optional[pathlib.Path]:
|
||||
"""
|
||||
Returns the path to the conda environment if
|
||||
the nerfstudio package is installed in one.
|
||||
"""
|
||||
conda_path = None
|
||||
if "CONDA_PREFIX" in os.environ:
|
||||
# Conda is active, we will check if the Nerfstudio is installed in the conda env.
|
||||
distribution = importlib_metadata.distribution("nerfstudio")
|
||||
if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]):
|
||||
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
|
||||
return conda_path
|
||||
|
||||
|
||||
def _generate_completions_files(
|
||||
completions_dir: pathlib.Path,
|
||||
scripts_dir: pathlib.Path,
|
||||
shells_supported: List[ShellType],
|
||||
shells_found: List[ShellType],
|
||||
) -> None:
|
||||
# Set to True to install completions for scripts as well.
|
||||
include_scripts = False
|
||||
|
||||
# Find tyro CLIs.
|
||||
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
|
||||
script_names = tuple(p.name for p in script_paths)
|
||||
assert len(set(script_names)) == len(script_names)
|
||||
|
||||
# Get existing completion files.
|
||||
existing_completions = set()
|
||||
for shell in shells_supported:
|
||||
target_dir = completions_dir / shell
|
||||
if target_dir.exists():
|
||||
existing_completions |= set(target_dir.glob("*"))
|
||||
|
||||
# Get all entry_points.
|
||||
entry_points = _get_all_entry_points()
|
||||
|
||||
# Run generation jobs.
|
||||
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
|
||||
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
|
||||
completion_paths = list(
|
||||
concurrent_executor.map(
|
||||
lambda path_or_entrypoint_and_shell: _generate_completion(
|
||||
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
|
||||
),
|
||||
itertools.product(script_paths + entry_points, shells_found),
|
||||
)
|
||||
)
|
||||
|
||||
# Delete obsolete completion files.
|
||||
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
|
||||
p.absolute() for p in completion_paths
|
||||
):
|
||||
if unexpected_path.is_dir():
|
||||
shutil.rmtree(unexpected_path)
|
||||
elif unexpected_path.exists():
|
||||
unexpected_path.unlink()
|
||||
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
|
||||
|
||||
|
||||
def main(mode: ConfigureMode = "install") -> None:
|
||||
"""Main script.
|
||||
|
||||
@@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None:
|
||||
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
|
||||
return
|
||||
|
||||
# Get conda path if in conda environment.
|
||||
conda_path = _get_conda_path()
|
||||
|
||||
# Try to locate the user's bashrc or zshrc.
|
||||
shells_supported: List[ShellType] = ["zsh", "bash"]
|
||||
shells_found: List[ShellType] = []
|
||||
for shell in shells_supported:
|
||||
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
|
||||
if not rc_path.exists():
|
||||
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
|
||||
else:
|
||||
CONSOLE.log(f":mag: Found {rc_path.name}!")
|
||||
shells_found.append(shell)
|
||||
shells_supported: List[ShellType] = list(typing_get_args(ShellType))
|
||||
if conda_path is not None:
|
||||
# Running in conda; we have to support all shells.
|
||||
shells_found = shells_supported
|
||||
CONSOLE.log(f":mag: Detected conda environment {conda_path}!")
|
||||
else:
|
||||
shells_found: List[ShellType] = []
|
||||
for shell in shells_supported:
|
||||
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
|
||||
if not rc_path.exists():
|
||||
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
|
||||
else:
|
||||
CONSOLE.log(f":mag: Found {rc_path.name}!")
|
||||
shells_found.append(shell)
|
||||
|
||||
# Get scripts/ directory.
|
||||
completions_dir = pathlib.Path(__file__).absolute().parent
|
||||
@@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None:
|
||||
else:
|
||||
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
|
||||
elif mode == "install":
|
||||
# Set to True to install completions for scripts as well.
|
||||
include_scripts = False
|
||||
|
||||
# Find tyro CLIs.
|
||||
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
|
||||
script_names = tuple(p.name for p in script_paths)
|
||||
assert len(set(script_names)) == len(script_names)
|
||||
|
||||
# Get existing completion files.
|
||||
existing_completions = set()
|
||||
for shell in shells_supported:
|
||||
target_dir = completions_dir / shell
|
||||
if target_dir.exists():
|
||||
existing_completions |= set(target_dir.glob("*"))
|
||||
|
||||
# Run generation jobs.
|
||||
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
|
||||
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
|
||||
completion_paths = list(
|
||||
concurrent_executor.map(
|
||||
lambda path_or_entrypoint_and_shell: _generate_completion(
|
||||
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
|
||||
),
|
||||
itertools.product(script_paths + ENTRYPOINTS, shells_found),
|
||||
)
|
||||
)
|
||||
|
||||
# Delete obsolete completion files.
|
||||
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
|
||||
p.absolute() for p in completion_paths
|
||||
):
|
||||
if unexpected_path.is_dir():
|
||||
shutil.rmtree(unexpected_path)
|
||||
elif unexpected_path.exists():
|
||||
unexpected_path.unlink()
|
||||
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
|
||||
_generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found)
|
||||
else:
|
||||
assert_never(mode)
|
||||
|
||||
# Install or uninstall from bashrc/zshrc.
|
||||
for shell in shells_found:
|
||||
_update_rc(completions_dir, mode, shell)
|
||||
if conda_path is not None:
|
||||
# In conda environment we add the completitions activation scripts.
|
||||
commands = _get_all_entry_points()
|
||||
_update_conda_scripts(commands, completions_dir, mode)
|
||||
else:
|
||||
# Install or uninstall from bashrc/zshrc.
|
||||
for shell in shells_found:
|
||||
_update_rc(completions_dir, mode, shell)
|
||||
|
||||
CONSOLE.print("[bold]All done![/bold]")
|
||||
|
||||
|
||||
@@ -287,6 +287,75 @@ class PhototourismDownload(DatasetDownload):
|
||||
os.remove(download_path)
|
||||
|
||||
|
||||
# credit to https://autonomousvision.github.io/sdfstudio/
|
||||
# pylint: disable=line-too-long
|
||||
sdfstudio_downloads = {
|
||||
"sdfstudio-demo-data": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/sdfstudio-demo-data.tar",
|
||||
"dtu": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/DTU.tar",
|
||||
"replica": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Replica.tar",
|
||||
"scannet": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/scannet.tar",
|
||||
"tanks-and-temple": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/tnt_advanced.tar",
|
||||
"tanks-and-temple-highres": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/highresTNT.tar",
|
||||
"heritage": "https://s3.eu-central-1.amazonaws.com/avg-projects/monosdf/data/Heritage-Recon.tar",
|
||||
"neural-rgbd-data": "http://kaldir.vc.in.tum.de/neural_rgbd/neural_rgbd_data.zip",
|
||||
"all": None,
|
||||
}
|
||||
|
||||
SDFstudioCaptureName = tyro.extras.literal_type_from_choices(sdfstudio_downloads.keys())
|
||||
|
||||
|
||||
@dataclass
|
||||
class SDFstudioDemoDownload(DatasetDownload):
|
||||
"""Download the sdfstudio dataset."""
|
||||
|
||||
dataset_name: SDFstudioCaptureName = "sdfstudio-demo-data"
|
||||
|
||||
def download(self, save_dir: Path):
|
||||
"""Download the D-NeRF dataset (https://github.com/albertpumarola/D-NeRF)."""
|
||||
# TODO: give this code the same structure as download_nerfstudio
|
||||
|
||||
if self.dataset_name == "all":
|
||||
for dataset_name in sdfstudio_downloads:
|
||||
if dataset_name != "all":
|
||||
SDFstudioDemoDownload(dataset_name=dataset_name).download(save_dir)
|
||||
return
|
||||
|
||||
assert (
|
||||
self.dataset_name in sdfstudio_downloads
|
||||
), f"Capture name {self.dataset_name} not found in {sdfstudio_downloads.keys()}"
|
||||
|
||||
url = sdfstudio_downloads[self.dataset_name]
|
||||
|
||||
target_path = str(save_dir / self.dataset_name)
|
||||
os.makedirs(target_path, exist_ok=True)
|
||||
|
||||
file_format = url[-4:]
|
||||
|
||||
download_path = Path(f"{target_path}{file_format}")
|
||||
tmp_path = str(save_dir / ".temp")
|
||||
shutil.rmtree(tmp_path, ignore_errors=True)
|
||||
os.makedirs(tmp_path, exist_ok=True)
|
||||
|
||||
os.system(f"curl -L {url} > {download_path}")
|
||||
if file_format == ".tar":
|
||||
with tarfile.open(download_path, "r") as tar_ref:
|
||||
tar_ref.extractall(str(tmp_path))
|
||||
elif file_format == ".zip":
|
||||
with zipfile.ZipFile(download_path, "r") as zip_ref:
|
||||
zip_ref.extractall(str(target_path))
|
||||
return
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
inner_folders = os.listdir(tmp_path)
|
||||
assert len(inner_folders) == 1, "There is more than one folder inside this zip file."
|
||||
folder = os.path.join(tmp_path, inner_folders[0])
|
||||
shutil.rmtree(target_path)
|
||||
shutil.move(folder, target_path)
|
||||
shutil.rmtree(tmp_path)
|
||||
os.remove(download_path)
|
||||
|
||||
|
||||
Commands = Union[
|
||||
Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")],
|
||||
Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")],
|
||||
@@ -294,6 +363,7 @@ Commands = Union[
|
||||
Annotated[Record3dDownload, tyro.conf.subcommand(name="record3d")],
|
||||
Annotated[DNerfDownload, tyro.conf.subcommand(name="dnerf")],
|
||||
Annotated[PhototourismDownload, tyro.conf.subcommand(name="phototourism")],
|
||||
Annotated[SDFstudioDemoDownload, tyro.conf.subcommand(name="sdfstudio")],
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -737,7 +737,7 @@ class ProcessRealityCapture:
|
||||
"""Process RealityCapture data into a nerfstudio dataset.
|
||||
|
||||
This script assumes that cameras have been aligned using RealityCapture. After alignment, it is necessary to
|
||||
export the camera poses as a `.csv` file.
|
||||
export the camera poses as a `.csv` file using the `Internal/External camera parameters` option.
|
||||
|
||||
This script does the following:
|
||||
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Starts viewer in eval mode.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field, fields
|
||||
from pathlib import Path
|
||||
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
|
||||
from nerfstudio.configs.base_config import ViewerConfig
|
||||
from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.pipelines.base_pipeline import Pipeline
|
||||
from nerfstudio.utils import writer
|
||||
from nerfstudio.utils.eval_utils import eval_setup
|
||||
from nerfstudio.utils.writer import EventName, TimeWriter
|
||||
from nerfstudio.viewer.server import viewer_utils
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ViewerConfigWithoutNumRays(ViewerConfig):
|
||||
"""Configuration for viewer instantiation"""
|
||||
|
||||
num_rays_per_chunk: tyro.conf.Suppress[int] = -1
|
||||
start_train: tyro.conf.Suppress[bool] = False
|
||||
|
||||
def as_viewer_config(self):
|
||||
"""Converts the instance to ViewerConfig"""
|
||||
return ViewerConfig(**{x.name: getattr(self, x.name) for x in fields(self)})
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunViewer:
|
||||
"""Load a checkpoint and start the viewer."""
|
||||
|
||||
load_config: Path
|
||||
"""Path to config YAML file."""
|
||||
viewer: ViewerConfigWithoutNumRays = field(default_factory=ViewerConfigWithoutNumRays)
|
||||
"""Viewer configuration"""
|
||||
|
||||
def main(self) -> None:
|
||||
"""Main function."""
|
||||
config, pipeline, _ = eval_setup(
|
||||
self.load_config,
|
||||
eval_num_rays_per_chunk=None,
|
||||
test_mode="test",
|
||||
)
|
||||
num_rays_per_chunk = config.viewer.num_rays_per_chunk
|
||||
assert self.viewer.num_rays_per_chunk == -1
|
||||
config.vis = "viewer"
|
||||
config.viewer = self.viewer.as_viewer_config()
|
||||
config.viewer.num_rays_per_chunk = num_rays_per_chunk
|
||||
|
||||
self._start_viewer(config, pipeline)
|
||||
|
||||
def _start_viewer(self, config: TrainerConfig, pipeline):
|
||||
base_dir = config.get_base_dir()
|
||||
viewer_log_path = base_dir / config.viewer.relative_log_filename
|
||||
viewer_state, banner_messages = viewer_utils.setup_viewer(
|
||||
config.viewer, log_filename=viewer_log_path, datapath=config.pipeline.datamanager.dataparser.data
|
||||
)
|
||||
|
||||
# We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
|
||||
config.logging.local_writer.enable = False
|
||||
writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages)
|
||||
|
||||
assert viewer_state and pipeline.datamanager.train_dataset
|
||||
viewer_state.init_scene(
|
||||
dataset=pipeline.datamanager.train_dataset,
|
||||
start_train=False,
|
||||
)
|
||||
while True:
|
||||
viewer_state.vis["renderingState/isTraining"].write(False)
|
||||
self._update_viewer_state(viewer_state, config, pipeline)
|
||||
|
||||
def _update_viewer_state(self, viewer_state: viewer_utils.ViewerState, config: TrainerConfig, pipeline: Pipeline):
|
||||
"""Updates the viewer state by rendering out scene with current pipeline
|
||||
Returns the time taken to render scene.
|
||||
|
||||
"""
|
||||
# NOTE: step must be > 0 otherwise the rendering would not happen
|
||||
step = 1
|
||||
num_rays_per_batch = config.pipeline.datamanager.train_num_rays_per_batch
|
||||
with TimeWriter(writer, EventName.ITER_VIS_TIME) as _:
|
||||
try:
|
||||
viewer_state.update_scene(self, step, pipeline.model, num_rays_per_batch)
|
||||
except RuntimeError:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
assert viewer_state.vis is not None
|
||||
viewer_state.vis["renderingState/log_errors"].write(
|
||||
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
|
||||
)
|
||||
|
||||
def save_checkpoint(self, *args, **kwargs):
|
||||
"""
|
||||
Mock method because we pass this instance to viewer_state.update_scene
|
||||
"""
|
||||
|
||||
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
tyro.extras.set_accent_color("bright_yellow")
|
||||
tyro.cli(RunViewer).main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
|
||||
# For sphinx docs
|
||||
get_parser_fn = lambda: tyro.extras.get_parser(RunViewer) # noqa
|
||||
@@ -3,22 +3,21 @@
|
||||
view_dataset.py
|
||||
"""
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
|
||||
from nerfstudio.configs.base_config import ViewerConfig
|
||||
from nerfstudio.data.datamanagers import AnnotatedDataParserUnion
|
||||
from nerfstudio.data.datamanagers.base_datamanager import AnnotatedDataParserUnion
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.viewer.server import viewer_utils
|
||||
|
||||
DEFAULT_TIMEOUT = timedelta(minutes=30)
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
# speedup for when input size to model doesn't change (much)
|
||||
torch.backends.cudnn.benchmark = True # type: ignore
|
||||
@@ -30,9 +29,10 @@ def main(
|
||||
log_base_dir: Path = Path("/tmp/nerfstudio_viewer_logs"),
|
||||
) -> None:
|
||||
"""Main function."""
|
||||
viewer_state = viewer_utils.ViewerState(
|
||||
viewer_state, _ = viewer_utils.setup_viewer(
|
||||
viewer,
|
||||
log_filename=log_base_dir / viewer.relative_log_filename,
|
||||
datapath=dataparser.data,
|
||||
)
|
||||
dataset = InputDataset(dataparser.setup().get_dataparser_outputs(split="train"))
|
||||
viewer_state.init_scene(dataset=dataset, start_train=False)
|
||||
|
||||
Reference in new issue
Block a user