From 5630148c50c776fbf1fa16b69784408949697781 Mon Sep 17 00:00:00 2001 From: Alexander Kristoffersen Date: Tue, 13 Dec 2022 23:04:17 -0800 Subject: [PATCH 01/38] Changed tinycudann to wheel to work with py3.8 (#1126) Created using Colaboratory --- colab/demo.ipynb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/colab/demo.ipynb b/colab/demo.ipynb index 9609a6db4..148da3b88 100644 --- a/colab/demo.ipynb +++ b/colab/demo.ipynb @@ -7,7 +7,7 @@ "colab_type": "text" }, "source": [ - "\"Open" + "\"Open" ] }, { @@ -63,25 +63,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "RGr33zHaHak0", - "outputId": "2641dce6-cd57-4d31-d8b1-860bc1adf17f" + "outputId": "6ad38334-22d2-4675-bffb-0bae91b17577" }, "outputs": [ { - "name": "stdout", "output_type": "stream", + "name": "stdout", "text": [ "⏬ Downloading https://github.com/jaimergp/miniforge/releases/latest/download/Mambaforge-colab-Linux-x86_64.sh...\n", "📦 Installing...\n", "📌 Adjusting configuration...\n", "🩹 Patching environment...\n", - "⏲ Done in 0:00:27\n", + "⏲ Done in 0:00:18\n", "🔁 Restarting kernel...\n" ] } @@ -111,9 +111,8 @@ "\n", "# Installing TinyCuda\n", "%cd /content/\n", - "# !gdown \"https://drive.google.com/u/1/uc?id=1wglEFPbI4BjWMFYaKk7gHCNsWCZphKjs\" \n", - "# !pip install tinycudann-1.6-cp38-cp38-linux_x86_64.whl\n", - "!pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch\n", + "!gdown \"https://drive.google.com/u/0/uc?id=1mB3IxcpmTVmQN-So0t2lxjxbHvqWr2zf\" \n", + "!pip install tinycudann-1.5-cp38-cp38-linux_x86_64.whl\n", "\n", "# Installing COLMAP\n", "%cd /content/\n", From d51718ec2bc411b6a0ac16d017f80ae96d3aceb0 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 14 Dec 2022 16:26:05 -0800 Subject: [PATCH 02/38] Immediately update viewer when output or cmap changes (#1131) * Immediately update viewer when output or cmap changes * lint --- nerfstudio/viewer/server/viewer_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index ce70426f6..c3ebb950e 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -454,6 +454,19 @@ class ViewerState: else: self.prev_camera_matrix = camera_object["matrix"] self.camera_moving = True + + output_type = self.vis["renderingState/output_choice"].read() + if output_type is None: + output_type = OutputTypes.INIT + if self.prev_output_type != output_type: + self.camera_moving = True + + colormap_type = self.vis["renderingState/colormap_choice"].read() + if colormap_type is None: + colormap_type = ColormapTypes.INIT + if self.prev_colormap_type != colormap_type: + self.camera_moving = True + return camera_object def _apply_colormap(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6): From bc690632bbffee4d21dd504985e13bca2510bf13 Mon Sep 17 00:00:00 2001 From: Penn Date: Thu, 15 Dec 2022 13:24:31 -0800 Subject: [PATCH 03/38] Use `torch.where` for faster contract (#1132) use where for faster contract --- nerfstudio/field_components/spatial_distortions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nerfstudio/field_components/spatial_distortions.py b/nerfstudio/field_components/spatial_distortions.py index 449f4b5b9..02da548a3 100644 --- a/nerfstudio/field_components/spatial_distortions.py +++ b/nerfstudio/field_components/spatial_distortions.py @@ -65,12 +65,8 @@ class SceneContraction(SpatialDistortion): def forward(self, positions): def contract(x): - mag = torch.linalg.norm(x, ord=self.order, dim=-1) - mask = mag >= 1 - x_new = x.clone() - x_new[mask] = (2 - (1 / mag[mask][..., None])) * (x[mask] / mag[mask][..., None]) - - return x_new + mag = torch.linalg.norm(x, ord=self.order, dim=-1)[..., None] + return torch.where(mag < 1, x, (2 - (1 / mag)) * (x / mag)) if isinstance(positions, Gaussians): means = contract(positions.mean.clone()) From 3da0ca727d560e264cfaa4425c8818fab62601bf Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 15 Dec 2022 14:18:23 -0800 Subject: [PATCH 04/38] Update render ui to use leva (#1133) --- nerfstudio/viewer/app/src/index.scss | 29 +++- .../SidePanel/CameraPanel/CameraPanel.jsx | 144 ++++-------------- .../SidePanel/CameraPanel/CameraPropPanel.jsx | 73 +++++++++ 3 files changed, 124 insertions(+), 122 deletions(-) create mode 100644 nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx diff --git a/nerfstudio/viewer/app/src/index.scss b/nerfstudio/viewer/app/src/index.scss index 194c5ac2e..98e968795 100644 --- a/nerfstudio/viewer/app/src/index.scss +++ b/nerfstudio/viewer/app/src/index.scss @@ -242,18 +242,37 @@ $sidebar-width: 100% !default; // ------------------------------------ // Camera Pane -.CameraPanel { +.CameraPanel-props { + margin-top: 10px; + margin-bottom: 10px; + margin-left: 20px; + margin-right: 20px; +} + +.CameraPanel-path-row { + display: inline-block; + width: 45%; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 4px; + margin-right: 4px; +} + +.CameraPanel-row { + display: inline-block; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 4px; + margin-right: 4px; } .CameraPanel-top-button { - display: inline-block; - margin: 4px; - margin-top: 8px; + width: 100%; } .CameraPanel-render-button { display: inline-block; - width: 70%; + width: 92%; margin: 4px; margin-top: 8px; } diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 6a741d156..9958ea7b1 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -37,6 +37,7 @@ import { useDispatch, useSelector } from 'react-redux'; import { CSS2DObject } from 'three/examples/jsm/renderers/CSS2DRenderer'; import AddAPhotoIcon from '@mui/icons-material/AddAPhoto'; import FileDownloadOutlinedIcon from '@mui/icons-material/FileDownloadOutlined'; +import { LevaPanel, LevaStoreProvider, useCreateStore } from 'leva'; import FileUploadOutlinedIcon from '@mui/icons-material/FileUploadOutlined'; import { Stack } from '@mui/system'; import TextField from '@mui/material/TextField'; @@ -46,6 +47,8 @@ import { CameraHelper } from './CameraHelper'; import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; import { WebSocketContext } from '../../WebSocket/WebSocket'; import RenderModal from '../../RenderModal'; +import CameraPropPanel from './CameraPropPanel'; +import LevaTheme from '../../../themes/leva_theme.json'; const msgpack = require('msgpack-lite'); @@ -460,6 +463,9 @@ export default function CameraPanel(props) { const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); + // leva store + const cameraPropsStore = useCreateStore(); + const scene_state = sceneTree.get_scene_state(); // Template for sharing state between Vanilla JS Three.js and React components @@ -496,10 +502,6 @@ export default function CameraPanel(props) { }; // ui state - const [ui_render_height, setUIRenderHeight] = React.useState(render_height); - const [ui_render_width, setUIRenderWidth] = React.useState(render_width); - const [ui_seconds, setUISeconds] = React.useState(seconds); - const [ui_fps, setUIfps] = React.useState(fps); const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); // nonlinear render option @@ -868,15 +870,10 @@ export default function CameraPanel(props) { const new_properties = new Map(cameraProperties); setRenderHeight(camera_path_object.render_height); - setUIRenderHeight(camera_path_object.render_height); setRenderWidth(camera_path_object.render_width); - setUIRenderWidth(camera_path_object.render_width); setFps(camera_path_object.fps); - setUIfps(camera_path_object.fps); - setSeconds(camera_path_object.seconds); - setUISeconds(camera_path_object.seconds); set_smoothness_value(camera_path_object.smoothness_value); setIsCycle(camera_path_object.is_cycle); @@ -968,7 +965,7 @@ export default function CameraPanel(props) { return (
-
+
-
+
-
- { - if (e.target.validity.valid) { - setUIRenderHeight(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setRenderHeight(parseInt(e.target.value, 10)); - } else { - setUIRenderHeight(render_height); - } - } - }} - value={ui_render_height} - error={ui_render_height <= 0} - helperText={ui_render_height <= 0 ? 'Required' : ''} - variant="standard" - /> - { - if (e.target.validity.valid) { - setUIRenderWidth(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setRenderWidth(parseInt(e.target.value, 10)); - } else { - setUIRenderWidth(render_width); - } - } - }} - value={ui_render_width} - error={ui_render_width <= 0} - helperText={ui_render_width <= 0 ? 'Required' : ''} - variant="standard" - /> -
-
- { - if (e.target.validity.valid) { - setUISeconds(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setSeconds(parseInt(e.target.value, 10)); - } else { - setUISeconds(seconds); - } - } - }} - value={ui_seconds} - error={ui_seconds <= 0} - helperText={ui_seconds <= 0 ? 'Required' : ''} - variant="standard" - /> - { - if (e.target.validity.valid) { - setUIfps(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setFps(parseInt(e.target.value, 10)); - } else { - setUIfps(fps); - } - } - }} - value={ui_fps} - error={ui_fps <= 0} - helperText={ui_fps <= 0 ? 'Required' : ''} - variant="standard" +
+ + + +
@@ -1161,7 +1071,7 @@ export default function CameraPanel(props) { />
-
+
-
+
-
+
@@ -115,7 +119,8 @@ export default function StatusPanel(props: StatusPanelProps) { set_is_images_visible(!is_images_visible); }} style={{ textTransform: 'none' }} - startIcon={} + startIcon={} + disabled={camera_choice === 'Render Camera'} > {cameras_button} diff --git a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index 90fe975e0..94990f2d9 100644 --- a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -9,6 +9,8 @@ import OpenWithIcon from '@mui/icons-material/OpenWith'; import PublicOffSharpIcon from '@mui/icons-material/PublicOffSharp'; import PublicSharpIcon from '@mui/icons-material/PublicSharp'; import SyncOutlinedIcon from '@mui/icons-material/SyncOutlined'; +import ThreeDRotationIcon from '@mui/icons-material/ThreeDRotation'; +import VideoCameraBackIcon from '@mui/icons-material/VideoCameraBackOutlined'; import WebRtcWindow from '../WebRtcWindow/WebRtcWindow'; import { WebSocketContext } from '../WebSocket/WebSocket'; @@ -19,27 +21,36 @@ function CameraToggle() { const camera_choice = useSelector( (state) => state.renderingState.camera_choice, ); - const set_camera_choice = (event: SelectChangeEvent) => { - const value = event.target.value; - dispatch({ - type: 'write', - path: 'renderingState/camera_choice', - data: value, - }); + const set_camera_choice = (event: SelectChangeEvent, value: string[]) => { + if (value != null) { + dispatch({ + type: 'write', + path: 'renderingState/camera_choice', + data: value, + }); + } }; return ( - Viewport - Render View + + + Viewport + + + + Render View + ); } diff --git a/nerfstudio/viewer/app/src/themes/theme.ts b/nerfstudio/viewer/app/src/themes/theme.ts index 554445561..4a973209e 100644 --- a/nerfstudio/viewer/app/src/themes/theme.ts +++ b/nerfstudio/viewer/app/src/themes/theme.ts @@ -126,10 +126,13 @@ export const appTheme = createTheme({ styleOverrides: { root: { color: '#eeeeee', - backgroundColor: '#393e46', + backgroundColor: 'FFD369', '&.Mui-selected': { - color: '#FFD369', - backgroundColor: '#555555', + color: '#222831', + backgroundColor: '#FFD369', + ':hover': { + backgroundColor: '#FFD369', + }, }, '&.Mui-disabled': { color: '#999999', From 3bf490e845897a74db278e027f085967758082a2 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Thu, 15 Dec 2022 17:29:41 -0800 Subject: [PATCH 06/38] adding data subset (#1135) * adding nerfstudio-dataset * removed file --- scripts/downloads/download_data.py | 68 +++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index c444991a9..12a0a0227 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -15,6 +15,7 @@ from rich.console import Console from typing_extensions import Annotated from nerfstudio.configs.base_config import PrintableConfig +from nerfstudio.utils.scripts import run_command CONSOLE = Console(width=120) @@ -84,17 +85,39 @@ def grab_file_id(zip_url: str) -> str: return s.split("/")[0] +nerfstudio_dataset = [ + "Egypt", + "person", + "kitchen", + "plane", + "dozer", + "floating-tree", + "aspen", + "stump", + "sculpture", + "Giannini-Hall", +] nerfstudio_file_ids = { "bww_entrance": grab_file_id("https://drive.google.com/file/d/1ylkRHtfB3n3IRLf2wplpfxzPTq7nES9I/view?usp=sharing"), "campanile": grab_file_id("https://drive.google.com/file/d/13aOfGJRRH05pOOk9ikYGTwqFc2L1xskU/view?usp=sharing"), "desolation": grab_file_id("https://drive.google.com/file/d/14IzOOQm9KBJ3kPbunQbUTHPnXnmZus-f/view?usp=sharing"), - "dozer": grab_file_id("https://drive.google.com/file/d/1-OR5F_V5S4s-yzxohbwTylaXjzYLu8ZR/view?usp=sharing"), "library": grab_file_id("https://drive.google.com/file/d/1Hjbh_-BuaWETQExn2x2qGD74UwrFugHx/view?usp=sharing"), "poster": grab_file_id("https://drive.google.com/file/d/1dmjWGXlJnUxwosN6MVooCDQe970PkD-1/view?usp=sharing"), "redwoods2": grab_file_id("https://drive.google.com/file/d/1rg-4NoXT8p6vkmbWxMOY6PSG4j3rfcJ8/view?usp=sharing"), "storefront": grab_file_id("https://drive.google.com/file/d/16b792AguPZWDA_YC4igKCwXJqW0Tb21o/view?usp=sharing"), "vegetation": grab_file_id("https://drive.google.com/file/d/1wBhLQ2odycrtU39y2akVurXEAt9SsVI3/view?usp=sharing"), + "Egypt": "https://data.nerf.studio/nerfstudio/Egypt.zip", + "person": "https://data.nerf.studio/nerfstudio/person.zip", + "kitchen": "https://data.nerf.studio/nerfstudio/kitchen.zip", + "plane": "https://data.nerf.studio/nerfstudio/plane.zip", + "dozer": "https://data.nerf.studio/nerfstudio/dozer.zip", + "floating-tree": "https://data.nerf.studio/nerfstudio/floating-tree.zip", + "aspen": "https://data.nerf.studio/nerfstudio/aspen.zip", + "stump": "https://data.nerf.studio/nerfstudio/stump.zip", + "sculpture": "https://data.nerf.studio/nerfstudio/sculpture.zip", + "Giannini-Hall": "https://data.nerf.studio/nerfstudio/Giannini-Hall.zip", "all": None, + "nerfstudio-dataset": nerfstudio_dataset, } NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_ids.keys()) @@ -103,14 +126,33 @@ NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_id def download_capture_name(save_dir: Path, dataset_name: str, capture_name: str, capture_name_to_file_id: dict): """Download specific captures a given dataset and capture name.""" - url = f"https://drive.google.com/uc?id={capture_name_to_file_id[capture_name]}" - target_path = str(save_dir / f"{dataset_name}/{capture_name}") - os.makedirs(target_path, exist_ok=True) - download_path = Path(f"{target_path}.zip") - tmp_path = str(save_dir / ".temp") - shutil.rmtree(tmp_path, ignore_errors=True) - os.makedirs(tmp_path, exist_ok=True) - gdown.download(url, output=str(download_path)) + file_id_or_zip_url = capture_name_to_file_id[capture_name] + if file_id_or_zip_url.endswith(".zip"): + url = file_id_or_zip_url # zip url + target_path = str(save_dir / f"{dataset_name}/{capture_name}") + os.makedirs(target_path, exist_ok=True) + download_path = Path(f"{target_path}.zip") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + try: + os.remove(download_path) + except OSError: + pass + run_command(f"wget {url} -O {download_path}", verbose=True) + else: + url = f"https://drive.google.com/uc?id={file_id_or_zip_url}" # file id + target_path = str(save_dir / f"{dataset_name}/{capture_name}") + os.makedirs(target_path, exist_ok=True) + download_path = Path(f"{target_path}.zip") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + try: + os.remove(download_path) + except OSError: + pass + gdown.download(url, output=str(download_path)) with zipfile.ZipFile(download_path, "r") as zip_ref: zip_ref.extractall(tmp_path) inner_folders = os.listdir(tmp_path) @@ -132,7 +174,13 @@ class NerfstudioDownload(DatasetDownload): """Download the nerfstudio dataset.""" if self.capture_name == "all": for capture_name in nerfstudio_file_ids: - if capture_name != "all": + if capture_name not in ("all", "nerfstudio-dataset"): + download_capture_name(save_dir, "nerfstudio", capture_name, nerfstudio_file_ids) + return + + if self.capture_name == "nerfstudio-dataset": + for capture_name in nerfstudio_dataset: + if capture_name not in ("all", "nerfstudio-dataset"): download_capture_name(save_dir, "nerfstudio", capture_name, nerfstudio_file_ids) return From 29df5943343a0f7d6fa4bff2918880501c97d07b Mon Sep 17 00:00:00 2001 From: Nikhil Mohan <47728081+nikmo33@users.noreply.github.com> Date: Fri, 16 Dec 2022 20:16:20 +0000 Subject: [PATCH 07/38] Minimal disk dataparser (#1128) * wip * merge upstream * clkeanup * cleanup * cleanup * cleanup * more cleanup * cleanup * cleanup docs * lint * style fix --- .../data/datamanagers/base_datamanager.py | 2 + .../data/datamanagers/semantic_datamanager.py | 1 + .../data/dataparsers/minimal_dataparser.py | 97 ++++++++++++++++++ tests/data/minimal_parser/train.npz | Bin 0 -> 8422 bytes tests/data/minimal_parser/val.npz | Bin 0 -> 8422 bytes tests/test_train.py | 10 ++ 6 files changed, 110 insertions(+) create mode 100644 nerfstudio/data/dataparsers/minimal_dataparser.py create mode 100644 tests/data/minimal_parser/train.npz create mode 100644 tests/data/minimal_parser/val.npz diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 140d018ba..49b759a9d 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -42,6 +42,7 @@ from nerfstudio.data.dataparsers.friends_dataparser import FriendsDataParserConf from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( InstantNGPDataParserConfig, ) +from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConfig from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserConfig from nerfstudio.data.dataparsers.phototourism_dataparser import ( @@ -65,6 +66,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of tyro.extras.subcommand_type_from_defaults( { "nerfstudio-data": NerfstudioDataParserConfig(), + "minimal-parser": MinimalDataParserConfig(), "blender-data": BlenderDataParserConfig(), "friends-data": FriendsDataParserConfig(), "instant-ngp-data": InstantNGPDataParserConfig(), diff --git a/nerfstudio/data/datamanagers/semantic_datamanager.py b/nerfstudio/data/datamanagers/semantic_datamanager.py index 188c7f6c3..322b1d3bd 100644 --- a/nerfstudio/data/datamanagers/semantic_datamanager.py +++ b/nerfstudio/data/datamanagers/semantic_datamanager.py @@ -41,6 +41,7 @@ class SemanticDataManager(VanillaDataManager): # pylint: disable=abstract-metho """ def create_train_dataset(self) -> SemanticDataset: + self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") return SemanticDataset( dataparser_outputs=self.train_dataparser_outputs, scale_factor=self.config.camera_res_scale_factor, diff --git a/nerfstudio/data/dataparsers/minimal_dataparser.py b/nerfstudio/data/dataparsers/minimal_dataparser.py new file mode 100644 index 000000000..b6ffd8518 --- /dev/null +++ b/nerfstudio/data/dataparsers/minimal_dataparser.py @@ -0,0 +1,97 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Data parser for pre-prepared datasets for all cameras, with no additional processing needed +Optional fields - semantics, mask_filenames, cameras.distortion_params, cameras.times +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Type + +import numpy as np +import torch + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.data.dataparsers.base_dataparser import ( + DataParser, + DataParserConfig, + DataparserOutputs, + Semantics, +) +from nerfstudio.data.scene_box import SceneBox + + +@dataclass +class MinimalDataParserConfig(DataParserConfig): + """Minimal dataset config""" + + _target: Type = field(default_factory=lambda: MinimalDataParser) + """target class to instantiate""" + data: Path = Path("/home/nikhil/nerfstudio-main/tests/data/lego_test/minimal_parser") + + +@dataclass +class MinimalDataParser(DataParser): + """Minimal DatasetParser""" + + config: MinimalDataParserConfig + + def _generate_dataparser_outputs(self, split="train"): + filepath = self.config.data / f"{split}.npz" + data = np.load(filepath, allow_pickle=True) + + image_filenames = data["image_filenames"].tolist() + mask_filenames = data["mask_filenames"].tolist() if "mask_filenames" in data.keys() else None + + metadata = None + if "semantics" in data.keys(): + semantics = data["semantics"].item() + metadata = { + "semantics": Semantics( + filenames=semantics["filenames"].tolist(), + classes=semantics["classes"].tolist(), + colors=semantics["colors"].tolist(), + mask_classes=semantics["mask_classes"].tolist(), + ) + } + + scene_box_aabb = torch.from_numpy(data["scene_box"]) + scene_box = SceneBox(aabb=scene_box_aabb) + + camera_np = data["cameras"].item() + distortion_params = None + if "distortion_params" in camera_np.keys(): + distortion_params = torch.from_numpy(camera_np["distortion_params"]) + cameras = Cameras( + fx=torch.from_numpy(camera_np["fx"]), + fy=torch.from_numpy(camera_np["fy"]), + cx=torch.from_numpy(camera_np["cx"]), + cy=torch.from_numpy(camera_np["cy"]), + distortion_params=distortion_params, + height=torch.from_numpy(camera_np["height"]), + width=torch.from_numpy(camera_np["width"]), + camera_to_worlds=torch.from_numpy(camera_np["camera_to_worlds"])[:, :3, :4], + camera_type=torch.from_numpy(camera_np["camera_type"]), + times=torch.from_numpy(camera_np["times"]) if "times" in camera_np.keys() else None, + ) + + dataparser_outputs = DataparserOutputs( + image_filenames=image_filenames, + cameras=cameras, + scene_box=scene_box, + mask_filenames=mask_filenames, + metadata=metadata, + ) + return dataparser_outputs diff --git a/tests/data/minimal_parser/train.npz b/tests/data/minimal_parser/train.npz new file mode 100644 index 0000000000000000000000000000000000000000..9d8acbb15f7b0060ffaa046218d6aff7b79c0aff GIT binary patch literal 8422 zcmeHMdvH|M8Q(leAV3HZAPC5!LIQ;FNC*LvdoTg5ERVQ_&URWR>~1!>Ta#?g?hO#c zG2j5AqC8umh=72!+KPZRq9j@Yu>%OGQ&c)-%0oM&6cLr8)qcN|{npKrO>Tz%+0Xer z_nz~;zu!3fnm(~@fORyxZ)C1u_UPnEOM6UeHnu8pyz zsAZ@?477H}?CHfnkFm_NJeXeU3Ap{~6RhcrW~OIa)60B*!|$vr@%c+Vn7+qZ5%AD_ zKzCMq=sjah=Fm**L)KfNq`EkFRZV4eZI;{T_hePpR2W{T-|wtVD)D>VzN&!XuW=hm z($XZAR8`uvDr0Pxr82fC#wFvH%Xqu&kYS6l#V(ZzLBUd^w%U^<6CJUX62H$^DZ4v5(vs>#pd?53 zaHIt#OMW_f%#=~1OGYWm8<+K~>T=oBkg0arOHaM`Fd?8NABUi+TIpo+{tjG_wL`~X5 zh!4KlubEoHKI_3J@_GQ1CJ1q&9`FYnW{LelkMY9&w9*{vg!gh{otSMymg|Z7S&6X@ z43rXc?#P>U=rM#(#EL#xgLCX7TAd(^7-1JRfjndpmz5Z}KpgOe5pp2Z1@DkWJrScB zVqC)x@-a({>+qb-=)wnTf_&OT$cOgO)Gu}(c5pz~zt}S{34kwNg@@?f5uE#dv1A6?yIrxGu>oazv>0ll?z&>QH#Abi*QAfms-!m|Ln<2K3{sG@+VuW4J4>&bb zN9cf{$G=I47qu}%TzACCHq7YpUO--8#EP7)#Ed=E3i*fUj=r-jJ(?!i=JSO|V9iPl z`|t(Zk(kFC`VJjl3y+7UTO`Qf>~1f?2Vy}D*f*ZrBK~(3`hc^>Z12%$$g)0sn$cx^ z;ol3W8)}5OVGp&AM!*Q{BTulE82R!3Lcd$`kND67=<~kC83UjE8%d8h5#r)?*fvXC zd)P-kxh`$(3&=9Q+Vi`@eF*=1i_b!=Ar6iyJO-{k@@$Vk)DiWE9-htWqLy5fmik1? zEfVSiKd23!!SIb-@Enb}?!Xinc(W4oIe=Kg>p`!IBF{1_%|aithyk&;q{lTzY;&jpM`XM5DNR)YBA(k`Yd!TuDP?no&BZQwsmX47gwsae_g(;y|iak!QwiL#!^o` znJoDElS^gdgZJN8YZot+s(I+WbM&a1BRipI=&XF=iIAN)-S1X&}-ngN?T3@f7K6Sr(-|(A0;^6D~;?&9Fpl^!>xlkQxy-vJ+3wk&6~BC_vff*taDuv8}zSz84$Ft?c2Lo#j<=^f&jmJ;)J$l zb$79TU8-PN#PHgI3F64%DFQWt{tM3!SMrPN>Lz~t(MKB9N&E1OLp%BIO!4?*NlFGe zz;|OqgSL8AN&))#)ir}^y<2I@alN+%$6-w1H01Mw|5NnX216D>kCxQ zz&qij;D_(O*LFXbE5E56MuQ< zK7~)@{>|6(l{GAyZf-kER#KcJnws=fw;l? z@fx7_O{abzYG%i}%-G4-yw{6l8t$%|elKd}Z48or07-wl9H4Gk;N^C`SGd|hbx(od z+zX;yQN%)Ruu`MH!ivChh#}LPyt!*-274O{Z^P_zc*t93P%V+v5{u*rLuR=go#9O( zs03GwawO}Ig8pc`923&d4%Wn*Cv#Afv38jYP~TjQaa4_z7@&}dN112H@h(RSO|e5? z$$X!`f-V~;u-A$3ns1i{OBJHATp`x&jH=zE!PcsHwRUSpo;8ra;Owj^jyj? zIhY!c@*YFp>rxk5^Sz~pE`P!n?t_J?c4-S)n5Hb83tHe&PB-KXwe4p{ZhL)f-P{dS zJFvR@bbG2iB`)6rz5>Srh{qN;etua_OLyS2{526*;MVM0y#Gw1AI8QQ`f+SLsJQWG zQ_nXzrk>Y|Z85Q_aj8wgS|V@wgVQwBsoJz~-ydn9`K@rcfv&Wdn%xmcsy07M!&Q6q mW03=Ojps~1!>Ta#?g?hO#c zG2j5AqC8umh=72!+KPZRq9j@Yu>%OGQ&c)-%0oM&6cLr8)qcN|{npKrO>Tz%+0Xer z_nz~;zu!3fnm(~@fORyxZ)C1u_UPnEOM6UeHnu8pyz zsAZ@?477H}?CHfnkFm_NJeXeU3Ap{~6RhcrW~OIa)60B*!|$vr@%c+Vn7+qZ5%AD_ zKzCMq=sjah=Fm**L)KfNq`EkFRZV4eZI;{T_hePpR2W{T-|wtVD)D>VzN&!XuW=hm z($XZAR8`uvDr0Pxr82fC#wFvH%Xqu&kYS6l#V(ZzLBUd^w%U^<6CJUX62H$^DZ4v5(vs>#pd?53 zaHIt#OMW_f%#=~1OGYWm8<+K~>T=oBkg0arOHaM`Fd?8NABUi+TIpo+{tjG_wL`~X5 zh!4KlubEoHKI_3J@_GQ1CJ1q&9`FYnW{LelkMY9&w9*{vg!gh{otSMymg|Z7S&6X@ z43rXc?#P>U=rM#(#EL#xgLCX7TAd(^7-1JRfjndpmz5Z}KpgOe5pp2Z1@DkWJrScB zVqC)x@-a({>+qb-=)wnTf_&OT$cOgO)Gu}(c5pz~zt}S{34kwNg@@?f5uE#dv1A6?yIrxGu>oazv>0ll?z&>QH#Abi*QAfms-!m|Ln<2K3{sG@+VuW4J4>&bb zN9cf{$G=I47qu}%TzACCHq7YpUO--8#EP7)#Ed=E3i*fUj=r-jJ(?!i=JSO|V9iPl z`|t(Zk(kFC`VJjl3y+7UTO`Qf>~1f?2Vy}D*f*ZrBK~(3`hc^>Z12%$$g)0sn$cx^ z;ol3W8)}5OVGp&AM!*Q{BTulE82R!3Lcd$`kND67=<~kC83UjE8%d8h5#r)?*fvXC zd)P-kxh`$(3&=9Q+Vi`@eF*=1i_b!=Ar6iyJO-{k@@$Vk)DiWE9-htWqLy5fmik1? zEfVSiKd23!!SIb-@Enb}?!Xinc(W4oIe=Kg>p`!IBF{1_%|aithyk&;q{lTzY;&jpM`XM5DNR)YBA(k`Yd!TuDP?no&BZQwsmX47gwsae_g(;y|iak!QwiL#!^o` znJoDElS^gdgZJN8YZot+s(I+WbM&a1BRipI=&XF=iIAN)-S1X&}-ngN?T3@f7K6Sr(-|(A0;^6D~;?&9Fpl^!>xlkQxy-vJ+3wk&6~BC_vff*taDuv8}zSz84$Ft?c2Lo#j<=^f&jmJ;)J$l zb$79TU8-PN#PHgI3F64%DFQWt{tM3!SMrPN>Lz~t(MKB9N&E1OLp%BIO!4?*NlFGe zz;|OqgSL8AN&))#)ir}^y<2I@alN+%$6-w1H01Mw|5NnX216D>kCxQ zz&qij;D_(O*LFXbE5E56MuQ< zK7~)@{>|6(l{GAyZf-kER#KcJnws=fw;l? z@fx7_O{abzYG%i}%-G4-yw{6l8t$%|elKd}Z48or07-wl9H4Gk;N^C`SGd|hbx(od z+zX;yQN%)Ruu`MH!ivChh#}LPyt!*-274O{Z^P_zc*t93P%V+v5{u*rLuR=go#9O( zs03GwawO}Ig8pc`923&d4%Wn*Cv#Afv38jYP~TjQaa4_z7@&}dN112H@h(RSO|e5? z$$X!`f-V~;u-A$3ns1i{OBJHATp`x&jH=zE!PcsHwRUSpo;8ra;Owj^jyj? zIhY!c@*YFp>rxk5^Sz~pE`P!n?t_J?c4-S)n5Hb83tHe&PB-KXwe4p{ZhL)f-P{dS zJFvR@bbG2iB`)6rz5>Srh{qN;etua_OLyS2{526*;MVM0y#Gw1AI8QQ`f+SLsJQWG zQ_nXzrk>Y|Z85Q_aj8wgS|V@wgVQwBsoJz~-ydn9`K@rcfv&Wdn%xmcsy07M!&Q6q mW03=Ojps Date: Fri, 16 Dec 2022 12:42:45 -0800 Subject: [PATCH 08/38] Upgraded Gdown to fix data downloading. (#1140) upgraded gdown to fix some data downloads. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fe908f8c6..1127b0f7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "appdirs>=1.4", "av==9.2.0", "tyro>=0.3.31", - "gdown==4.5.1", + "gdown==4.6.0", "ninja==1.10.2.3", "functorch==0.2.1", "h5py>=2.9.0", From c338de4aa4b0509e3468508c88d7e5981990edc1 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 18 Dec 2022 10:46:39 -0800 Subject: [PATCH 09/38] Support non-pinhole rendering (#1141) --- nerfstudio/cameras/camera_paths.py | 30 ++++- nerfstudio/cameras/cameras.py | 2 +- nerfstudio/viewer/app/package.json | 2 +- .../SidePanel/CameraPanel/CameraPanel.jsx | 115 +++++++++++------- .../SidePanel/CameraPanel/CameraPropPanel.jsx | 22 ++++ .../src/modules/ViewerWindow/ViewerWindow.jsx | 91 +++++++------- nerfstudio/viewer/app/src/reducer.js | 1 + nerfstudio/viewer/server/utils.py | 15 ++- nerfstudio/viewer/server/viewer_utils.py | 13 +- 9 files changed, 191 insertions(+), 100 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index fb243917d..f5eb67501 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -23,7 +23,7 @@ import torch import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.camera_utils import get_interpolated_poses_many -from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length @@ -121,6 +121,19 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: image_height = camera_path["render_height"] image_width = camera_path["render_width"] + if "camera_type" in camera_path: + camera_type = camera_path["camera_type"] + else: + camera_type = "perspective" + + if "camera_type" not in camera_path: + camera_type = CameraType.PERSPECTIVE + elif camera_path["camera_type"] == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_path["camera_type"] == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE c2ws = [] fxs = [] @@ -129,11 +142,15 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: # pose c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3] c2ws.append(c2w) - # field of view - fov = camera["fov"] - focal_length = three_js_perspective_camera_focal_length(fov, image_height) - fxs.append(focal_length) - fys.append(focal_length) + if camera_type == CameraType.EQUIRECTANGULAR: + fxs.append(image_width / 2) + fys.append(image_height) + else: + # field of view + fov = camera["fov"] + focal_length = three_js_perspective_camera_focal_length(fov, image_height) + fxs.append(focal_length) + fys.append(focal_length) camera_to_worlds = torch.stack(c2ws, dim=0) fx = torch.tensor(fxs) @@ -144,4 +161,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: cx=image_width / 2, cy=image_height / 2, camera_to_worlds=camera_to_worlds, + camera_type=camera_type, ) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 0299fa4bb..eb10f414b 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -634,7 +634,7 @@ class Cameras(TensorDataclass): directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0] * sin_theta / theta, mask).float() directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1] * sin_theta / theta, mask).float() - directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask) + directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float() if CameraType.EQUIRECTANGULAR.value in cam_types: mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index b6183729e..78d4997f3 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "22-12-02-0", + "version": "22-12-16-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 9958ea7b1..4efab1b6f 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -112,12 +112,12 @@ function FovSelector(props) { camera.setFocalLength(val / camera.aspect); } } else if (fovLabel === FOV_LABELS.FOV) { + camera.fov = val; setGlobalFov(val); } else { - const old_fov = camera.fov; camera.setFocalLength(val / camera.aspect); const new_fov = camera.getEffectiveFOV(); - camera.fov = old_fov; + camera.fov = new_fov; setGlobalFov(new_fov); } @@ -216,6 +216,8 @@ function CameraList(props) { const [slider_value, set_slider_value] = React.useState(0); const [expanded, setExpanded] = React.useState(null); + const camera_type = useSelector((state) => state.renderingState.camera_type); + const handleChange = (cameraUUID: string) => (event: React.SyntheticEvent, isExpanded: boolean) => { @@ -399,7 +401,7 @@ function CameraList(props) { - {isAnimated('FOV') && ( + {isAnimated('FOV') && camera_type !== 'equirectangular' && ( state.renderingState.render_width, ); + const camera_type = useSelector((state) => state.renderingState.camera_type); const setRenderHeight = (value) => { dispatch({ @@ -501,6 +504,22 @@ export default function CameraPanel(props) { }); }; + const setCameraType = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/camera_type', + data: value, + }); + }; + + const setFieldOfView = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/field_of_view', + data: value, + }); + }; + // ui state const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); @@ -516,7 +535,7 @@ export default function CameraPanel(props) { // set slider and render camera back to 0 if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); - camera_render.fov = new_camera_list[0].fov; + setFieldOfView(new_camera_list[0].fov); set_slider_value(slider_min); } }; @@ -718,7 +737,7 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); } else { sceneTree.delete(['Camera Path', 'Curve']); } @@ -768,7 +787,7 @@ export default function CameraPanel(props) { fov = curve_object.curve_fovs.getPoint(point).z; const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); setGlobalFov(fov); } }, [slider_value, render_height, render_width]); @@ -828,6 +847,7 @@ export default function CameraPanel(props) { // const myData const camera_path_object = { keyframes, + camera_type, render_height, render_width, camera_path, @@ -871,6 +891,7 @@ export default function CameraPanel(props) { setRenderHeight(camera_path_object.render_height); setRenderWidth(camera_path_object.render_width); + setCameraType(camera_path_object.camera_type); setFps(camera_path_object.fps); setSeconds(camera_path_object.seconds); @@ -1024,52 +1045,54 @@ export default function CameraPanel(props) { />
-
- - - - -
+ sx={{ + mt: 1, + }} + > + + + + +
+ )}
From 5710b09b6d9d8ded9f49fba0f8548ada811ddb0a Mon Sep 17 00:00:00 2001 From: Dreek Fire Date: Sat, 31 Dec 2022 23:30:27 -0800 Subject: [PATCH 31/38] Add hashmap size and num_levels to instant_ngp_field, issue 1158 (#1196) add hashmap size and num_levels to instant_ngp_field, issue 1158 --- nerfstudio/fields/instant_ngp_field.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index f1042072b..18a806b68 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -61,6 +61,8 @@ class TCNNInstantNGPField(Field): num_images: number of images, requried if use_appearance_embedding is True appearance_embedding_dim: dimension of appearance embedding contraction_type: type of contraction + num_levels: number of levels of the hashmap for the base mlp + log2_hashmap_size: size of the hashmap for the base mlp """ def __init__( @@ -75,6 +77,8 @@ class TCNNInstantNGPField(Field): num_images: Optional[int] = None, appearance_embedding_dim: int = 32, contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE, + num_levels: int = 16, + log2_hashmap_size: int = 19, ) -> None: super().__init__() @@ -104,9 +108,9 @@ class TCNNInstantNGPField(Field): n_output_dims=1 + self.geo_feat_dim, encoding_config={ "otype": "HashGrid", - "n_levels": 16, + "n_levels": num_levels, "n_features_per_level": 2, - "log2_hashmap_size": 19, + "log2_hashmap_size": log2_hashmap_size, "base_resolution": 16, "per_level_scale": per_level_scale, }, From fa6f27975146045d066a7123144e5a15098b7da7 Mon Sep 17 00:00:00 2001 From: dozeri83 <109915435+dozeri83@users.noreply.github.com> Date: Mon, 2 Jan 2023 02:14:55 +0200 Subject: [PATCH 32/38] fixed an exception bug in case of num_samples is 0 (#1198) fixed an exception bug when num samples is 0 , "IndexError tensors used as indices must be long, byte or bool tensors" change type to long ( was float32) --- nerfstudio/model_components/ray_samplers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index 40a848ed3..89a238c72 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -472,7 +472,7 @@ class VolumetricSampler(Sampler): if num_samples == 0: # create a single fake sample and update packed_info accordingly # this says the last ray in packed_info has 1 sample, which starts and ends at 1 - ray_indices = torch.zeros((1,), dtype=starts.dtype, device=rays_o.device) + ray_indices = torch.zeros((1,), dtype=torch.long, device=rays_o.device) starts = torch.ones((1, 1), dtype=starts.dtype, device=rays_o.device) ends = torch.ones((1, 1), dtype=ends.dtype, device=rays_o.device) From 3da877050df2c9b347652a5a84ad6d5d8c768650 Mon Sep 17 00:00:00 2001 From: Michal Pandy Date: Mon, 2 Jan 2023 09:13:38 +0000 Subject: [PATCH 33/38] Depth supervision for Nerfacto (#1173) * Implement depth supervision * Fixes * Change normalize dtype * refactor * Change depth weight * Small refactor * Change docstring * Removu unused import * Add assumption to depth comment * Implement URF loss * Remove default depth loss type param * URF fixes * Fix depth loss scale * Visualize gt depths * Fixes & documentation * Documentation and formatting * Fix formatting problems * Fix linter problems * Fix tests * Fix tests --- docs/quickstart/data_conventions.md | 4 + nerfstudio/cameras/camera_utils.py | 15 ++ nerfstudio/cameras/cameras.py | 4 +- nerfstudio/configs/method_configs.py | 34 +++++ .../data/datamanagers/depth_datamanager.py | 48 +++++++ .../data/dataparsers/nerfstudio_dataparser.py | 20 +++ nerfstudio/data/datasets/depth_dataset.py | 54 ++++++++ nerfstudio/data/utils/data_utils.py | 27 ++++ nerfstudio/model_components/losses.py | 108 +++++++++++++++ nerfstudio/models/depth_nerfacto.py | 131 ++++++++++++++++++ tests/cameras/test_cameras.py | 9 +- tests/test_train.py | 10 +- 12 files changed, 460 insertions(+), 4 deletions(-) create mode 100644 nerfstudio/data/datamanagers/depth_datamanager.py create mode 100644 nerfstudio/data/datasets/depth_dataset.py create mode 100644 nerfstudio/models/depth_nerfacto.py diff --git a/docs/quickstart/data_conventions.md b/docs/quickstart/data_conventions.md index 98095af95..6cd959274 100644 --- a/docs/quickstart/data_conventions.md +++ b/docs/quickstart/data_conventions.md @@ -67,3 +67,7 @@ For a transform matrix, the first 3 columns are the +X, +Y, and +Z defining the ] } ``` + +**Depth images** + +To train with depth supervision, you can also provide a `depth_file_path` for each frame in your `transforms.json` and use one of the methods that support additional depth losses (e.g., depth-nerfacto). The depths are assumed to be 16-bit or 32-bit and to be in millimeters to remain consistent with [Polyform](https://github.com/PolyCam/polyform). You can adjust this scaling factor using the `depth_unit_scale_factor` parameter in `NerfstudioDataParserConfig`. Note that by default, we resize the depth images to match the shape of the RGB images. diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index d3b0ca619..ef7152dfb 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -230,6 +230,21 @@ def normalize(x) -> TensorType[...]: return x / torch.linalg.norm(x) +def normalize_with_norm(x: torch.Tensor, dim: int) -> Tuple[torch.Tensor, torch.Tensor]: + """Normalize tensor along axis and return normalized value with norms. + + Args: + x: tensor to normalize. + dim: axis along which to normalize. + + Returns: + Tuple of normalized tensor and corresponding norm. + """ + + norm = torch.maximum(torch.linalg.vector_norm(x, dim=dim, keepdims=True), torch.tensor([_EPS]).to(x)) + return x / norm, norm + + def viewmatrix(lookat, up, pos) -> TensorType[...]: """Returns a camera transformation matrix. diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index eb10f414b..d56ac9a6f 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -24,7 +24,6 @@ from typing import Dict, List, Optional, Tuple, Union import cv2 import torch import torchvision -from torch.nn.functional import normalize from torchtyping import TensorType import nerfstudio.utils.poses as pose_utils @@ -666,7 +665,7 @@ class Cameras(TensorDataclass): directions_stack = torch.sum( directions_stack[..., None, :] * rotation, dim=-1 ) # (..., 1, 3) * (..., 3, 3) -> (..., 3) - directions_stack = normalize(directions_stack, dim=-1) + directions_stack, directions_norm = camera_utils.normalize_with_norm(directions_stack, -1) assert directions_stack.shape == (3,) + num_rays_shape + (3,) origins = c2w[..., :3, 3] # (..., 3) @@ -691,6 +690,7 @@ class Cameras(TensorDataclass): pixel_area=pixel_area, camera_indices=camera_indices, times=times, + metadata={"directions_norm": directions_norm[0].detach()}, ) def to_json( diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index e7995cd71..d11c377b5 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -26,6 +26,7 @@ from nerfacc import ContractionType from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig +from nerfstudio.data.datamanagers.depth_datamanager import DepthDataManagerConfig from nerfstudio.data.datamanagers.semantic_datamanager import SemanticDataManagerConfig from nerfstudio.data.datamanagers.variable_res_datamanager import ( VariableResDataManagerConfig, @@ -44,6 +45,7 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConf from nerfstudio.engine.schedulers import SchedulerConfig from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind +from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.nerfacto import NerfactoModelConfig @@ -56,6 +58,7 @@ from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig method_configs: Dict[str, TrainerConfig] = {} descriptions = { "nerfacto": "Recommended real-time model tuned for real captures. This model will be continually updated.", + "depth-nerfacto": "Nerfacto with depth supervision.", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", "instant-ngp-bounded": "Implementation of Instant-NGP. Recommended for bounded real and synthetic scenes", "mipnerf": "High quality model for bounded scenes. (slow)", @@ -97,6 +100,37 @@ method_configs["nerfacto"] = TrainerConfig( vis="viewer", ) +method_configs["depth-nerfacto"] = TrainerConfig( + method_name="depth-nerfacto", + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + datamanager=DepthDataManagerConfig( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=4096, + eval_num_rays_per_batch=4096, + camera_optimizer=CameraOptimizerConfig( + mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) + ), + ), + model=DepthNerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": None, + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": None, + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", +) + method_configs["instant-ngp"] = TrainerConfig( method_name="instant-ngp", steps_per_eval_batch=500, diff --git a/nerfstudio/data/datamanagers/depth_datamanager.py b/nerfstudio/data/datamanagers/depth_datamanager.py new file mode 100644 index 000000000..465e79e2d --- /dev/null +++ b/nerfstudio/data/datamanagers/depth_datamanager.py @@ -0,0 +1,48 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Depth datamanager. +""" + +from dataclasses import dataclass, field +from typing import Type + +from nerfstudio.data.datamanagers import base_datamanager +from nerfstudio.data.datasets.depth_dataset import DepthDataset + + +@dataclass +class DepthDataManagerConfig(base_datamanager.VanillaDataManagerConfig): + """A depth datamanager - required to use with .setup()""" + + _target: Type = field(default_factory=lambda: DepthDataManager) + + +class DepthDataManager(base_datamanager.VanillaDataManager): # pylint: disable=abstract-method + """Data manager implementation for data that also requires processing depth data. + Args: + config: the DataManagerConfig used to instantiate class + """ + + def create_train_dataset(self) -> DepthDataset: + self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") + return DepthDataset( + dataparser_outputs=self.train_dataparser_outputs, + ) + + def create_eval_dataset(self) -> DepthDataset: + return DepthDataset( + dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split), + ) diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index e311a08cc..07adab431 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -62,6 +62,8 @@ class NerfstudioDataParserConfig(DataParserConfig): """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" train_split_percentage: float = 0.9 """The percent of images to use for training. The remaining images are for eval.""" + depth_unit_scale_factor: float = 1e-3 + """Scales the depth values to meters. Default value is 0.001 for a millimiter to meter conversion.""" @dataclass @@ -83,6 +85,7 @@ class Nerfstudio(DataParser): image_filenames = [] mask_filenames = [] + depth_filenames = [] poses = [] num_skipped_image_filenames = 0 @@ -152,6 +155,12 @@ class Nerfstudio(DataParser): downsample_folder_prefix="masks_", ) mask_filenames.append(mask_fname) + + if "depth_file_path" in frame: + depth_filepath = PurePath(frame["depth_file_path"]) + depth_fname = self._get_fname(depth_filepath, data_dir, downsample_folder_prefix="depths_") + depth_filenames.append(depth_fname) + if num_skipped_image_filenames >= 0: CONSOLE.log(f"Skipping {num_skipped_image_filenames} files in dataset split {split}.") assert ( @@ -166,6 +175,12 @@ class Nerfstudio(DataParser): Different number of image and mask filenames. You should check that mask_path is specified for every frame (or zero frames) in transforms.json. """ + assert len(depth_filenames) == 0 or ( + len(depth_filenames) == len(image_filenames) + ), """ + Different number of image and depth filenames. + You should check that depth_file_path is specified for every frame (or zero frames) in transforms.json. + """ # filter image_filenames and poses based on train/eval split percentage num_images = len(image_filenames) @@ -208,6 +223,7 @@ class Nerfstudio(DataParser): # Choose image_filenames and poses based on split, but after auto orient and scaling the poses. image_filenames = [image_filenames[i] for i in indices] mask_filenames = [mask_filenames[i] for i in indices] if len(mask_filenames) > 0 else [] + depth_filenames = [depth_filenames[i] for i in indices] if len(depth_filenames) > 0 else [] poses = poses[indices] # in x,y,z order @@ -265,6 +281,10 @@ class Nerfstudio(DataParser): mask_filenames=mask_filenames if len(mask_filenames) > 0 else None, dataparser_scale=scale_factor, dataparser_transform=transform_matrix, + metadata={ + "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, + "depth_unit_scale_factor": self.config.depth_unit_scale_factor, + }, ) return dataparser_outputs diff --git a/nerfstudio/data/datasets/depth_dataset.py b/nerfstudio/data/datasets/depth_dataset.py new file mode 100644 index 000000000..4d0400afb --- /dev/null +++ b/nerfstudio/data/datasets/depth_dataset.py @@ -0,0 +1,54 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Depth dataset. +""" + +from typing import Dict + +from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.data.utils.data_utils import get_depth_image_from_path + + +class DepthDataset(InputDataset): + """Dataset that returns images and depths. + + Args: + dataparser_outputs: description of where and how to read input images. + scale_factor: The scaling factor for the dataparser outputs. + """ + + def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): + super().__init__(dataparser_outputs, scale_factor) + assert ( + "depth_filenames" in dataparser_outputs.metadata.keys() + and dataparser_outputs.metadata["depth_filenames"] is not None + ) + self.depth_filenames = self.metadata["depth_filenames"] + self.depth_unit_scale_factor = self.metadata["depth_unit_scale_factor"] + + def get_metadata(self, data: Dict) -> Dict: + filepath = self.depth_filenames[data["image_idx"]] + height = int(self._dataparser_outputs.cameras.height[data["image_idx"]]) + width = int(self._dataparser_outputs.cameras.width[data["image_idx"]]) + + # Scale depth images to meter units and also by scaling applied to cameras + scale_factor = self.depth_unit_scale_factor * self._dataparser_outputs.dataparser_scale + depth_image = get_depth_image_from_path( + filepath=filepath, height=height, width=width, scale_factor=scale_factor + ) + + return {"depth_image": depth_image} diff --git a/nerfstudio/data/utils/data_utils.py b/nerfstudio/data/utils/data_utils.py index cba914532..22a3438dd 100644 --- a/nerfstudio/data/utils/data_utils.py +++ b/nerfstudio/data/utils/data_utils.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import List, Tuple, Union +import cv2 import numpy as np import torch from PIL import Image @@ -51,3 +52,29 @@ def get_semantics_and_mask_tensors_from_path( semantics = torch.from_numpy(np.array(pil_image, dtype="int64"))[..., None] mask = torch.sum(semantics == mask_indices, dim=-1, keepdim=True) == 0 return semantics, mask + + +def get_depth_image_from_path( + filepath: Path, + height: int, + width: int, + scale_factor: float, + interpolation: int = cv2.INTER_NEAREST, +) -> torch.Tensor: + """Loads, rescales and resizes depth images. + + Assumes filepath points to a 16-bit or 32-bit depth image. + Args: + filepath: Path to depth image. + height: Target depth image height. + width: Target depth image width. + scale_factor: Factor by which to scale depth image. + interpolation: Depth value interpolation for resizing. + + Returns: + Depth image torch tensor with shape [width, height, 1]. + """ + image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH) + image = image.astype(np.float64) * scale_factor + image = cv2.resize(image, (width, height), interpolation=interpolation) + return torch.from_numpy(image[:, :, np.newaxis]) diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 1946e4abc..983bee555 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -15,6 +15,7 @@ """ Collection of Losses. """ +from enum import Enum import torch from torch import nn @@ -29,6 +30,16 @@ LOSSES = {"L1": L1Loss, "MSE": MSELoss} EPS = 1.0e-7 +# Sigma scale factor from Urban Radiance Fields (Rematas et al., 2022) +URF_SIGMA_SCALE_FACTOR = 3.0 + + +class DephtLossType(Enum): + """Types of depth losses for depth supervision.""" + + DS_NERF = 1 + URF = 2 + def outer( t0_starts: TensorType[..., "num_samples_0"], @@ -194,3 +205,100 @@ def pred_normal_loss( ): """Loss between normals calculated from density and normals from prediction network.""" return (weights[..., 0] * (1.0 - torch.sum(normals * pred_normals, dim=-1))).sum(dim=-1) + + +def ds_nerf_depth_loss( + weights: TensorType[..., "num_samples", 1], + termination_depth: TensorType[..., 1], + steps: TensorType[..., "num_samples", 1], + lengths: TensorType[..., "num_samples", 1], + sigma: TensorType[0], +) -> TensorType[..., 1]: + """Depth loss from Depth-supervised NeRF (Deng et al., 2022). + + Args: + weights: Weights predicted for each sample. + termination_depth: Ground truth depth of rays. + steps: Sampling distances along rays. + lengths: Distances between steps. + sigma: Uncertainty around depth values. + Returns: + Depth loss scalar. + """ + loss = -torch.log(weights + EPS) * torch.exp(-((steps - termination_depth[:, None]) ** 2) / (2 * sigma)) * lengths + return torch.mean(loss.sum(-2)) + + +def urban_radiance_field_depth_loss( + weights: TensorType[..., "num_samples", 1], + termination_depth: TensorType[..., 1], + predicted_depth: TensorType[..., 1], + steps: TensorType[..., "num_samples", 1], + sigma: TensorType[0], +) -> TensorType[..., 1]: + """Lidar losses from Urban Radiance Fields (Rematas et al., 2022). + + Args: + weights: Weights predicted for each sample. + termination_depth: Ground truth depth of rays. + predicted_depth: Depth prediction from the network. + steps: Sampling distances along rays. + sigma: Uncertainty around depth values. + Returns: + Depth loss scalar. + """ + # Expected depth loss + expected_depth_loss = (termination_depth - predicted_depth) ** 2 + + # Line of sight losses + target_distribution = torch.distributions.normal.Normal(0.0, sigma / URF_SIGMA_SCALE_FACTOR) + termination_depth = termination_depth[:, None] + line_of_sight_loss_near_mask = torch.logical_and( + steps <= termination_depth + sigma, steps >= termination_depth - sigma + ) + line_of_sight_loss_near = (weights - torch.exp(target_distribution.log_prob(steps - termination_depth))) ** 2 + line_of_sight_loss_near = (line_of_sight_loss_near_mask * line_of_sight_loss_near).sum(-2) + line_of_sight_loss_empty_mask = steps < termination_depth - sigma + line_of_sight_loss_empty = (line_of_sight_loss_empty_mask * weights**2).sum(-2) + line_of_sight_loss = line_of_sight_loss_near + line_of_sight_loss_empty + + return torch.mean(expected_depth_loss + line_of_sight_loss) + + +def depth_loss( + weights: TensorType[..., "num_samples", 1], + ray_samples: RaySamples, + termination_depth: TensorType[..., 1], + predicted_depth: TensorType[..., 1], + sigma: TensorType[0], + directions_norm: TensorType[..., 1], + is_euclidean: bool, + depth_loss_type: DephtLossType, +) -> TensorType[0]: + """Implementation of depth losses. + + Args: + weights: Weights predicted for each sample. + ray_samples: Samples along rays corresponding to weights. + termination_depth: Ground truth depth of rays. + predicted_depth: Depth prediction from the network. + sigma: Uncertainty around depth value. + directions_norm: Norms of ray direction vectors in the camera frame. + is_euclidean: Whether ground truth depths correponds to normalized direction vectors. + depth_loss_type: Type of depth loss to apply. + + Returns: + Depth loss scalar. + """ + if not is_euclidean: + termination_depth = termination_depth / directions_norm + steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2 + + if depth_loss_type == DephtLossType.DS_NERF: + lengths = ray_samples.frustums.ends - ray_samples.frustums.starts + return ds_nerf_depth_loss(weights, termination_depth, steps, lengths, sigma) + + if depth_loss_type == DephtLossType.URF: + return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma) + + raise NotImplementedError("Provided depth loss type not implemented.") diff --git a/nerfstudio/models/depth_nerfacto.py b/nerfstudio/models/depth_nerfacto.py new file mode 100644 index 000000000..727173b7d --- /dev/null +++ b/nerfstudio/models/depth_nerfacto.py @@ -0,0 +1,131 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Nerfacto augumented with depth supervision. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, Tuple, Type + +import torch + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.model_components.losses import DephtLossType, depth_loss +from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig +from nerfstudio.utils import colormaps + + +@dataclass +class DepthNerfactoModelConfig(NerfactoModelConfig): + """Additional parameters for depth supervision.""" + + _target: Type = field(default_factory=lambda: DepthNerfactoModel) + depth_loss_mult: float = 1e-3 + """Lambda of the depth loss.""" + is_euclidean_depth: bool = False + """Whether input depth maps are Euclidean distances (or z-distances).""" + depth_sigma: float = 0.01 + """Uncertainty around depth values in meters (defaults to 1cm).""" + should_decay_sigma: bool = False + """Whether to exponentially decay sigma.""" + starting_depth_sigma: float = 0.2 + """Starting uncertainty around depth values in meters (defaults to 0.2m).""" + sigma_decay_rate: float = 0.99985 + """Rate of exponetial decay.""" + depth_loss_type: DephtLossType = DephtLossType.DS_NERF + """Depth loss type.""" + + +class DepthNerfactoModel(NerfactoModel): + """Depth loss augumented nerfacto model. + + Args: + config: Nerfacto configuration to instantiate model + """ + + config: DepthNerfactoModelConfig + + def populate_modules(self): + """Set the fields and modules.""" + super().populate_modules() + + if self.config.should_decay_sigma: + self.depth_sigma = torch.tensor([self.config.starting_depth_sigma]) + else: + self.depth_sigma = torch.tensor([self.config.depth_sigma]) + + def get_outputs(self, ray_bundle: RayBundle): + outputs = super().get_outputs(ray_bundle) + outputs["directions_norm"] = ray_bundle.metadata["directions_norm"] + return outputs + + def get_metrics_dict(self, outputs, batch): + metrics_dict = super().get_metrics_dict(outputs, batch) + if self.training: + metrics_dict["depth_loss"] = 0.0 + sigma = self._get_sigma().to(self.device) + termination_depth = batch["depth_image"].to(self.device) + for i in range(len(outputs["weights_list"])): + metrics_dict["depth_loss"] += depth_loss( + weights=outputs["weights_list"][i], + ray_samples=outputs["ray_samples_list"][i], + termination_depth=termination_depth, + predicted_depth=outputs["depth"], + sigma=sigma, + directions_norm=outputs["directions_norm"], + is_euclidean=self.config.is_euclidean_depth, + depth_loss_type=self.config.depth_loss_type, + ) / len(outputs["weights_list"]) + + return metrics_dict + + def get_loss_dict(self, outputs, batch, metrics_dict=None): + loss_dict = super().get_loss_dict(outputs, batch, metrics_dict) + if self.training: + assert metrics_dict is not None and "depth_loss" in metrics_dict + loss_dict["depth_loss"] = self.config.depth_loss_mult * metrics_dict["depth_loss"] + + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Appends ground truth depth to the depth image.""" + metrics, images = super().get_image_metrics_and_images(outputs, batch) + ground_truth_depth = batch["depth_image"] + if not self.config.is_euclidean_depth: + ground_truth_depth = ground_truth_depth / outputs["directions_norm"] + + ground_truth_depth_colormap = colormaps.apply_depth_colormap(ground_truth_depth) + predicted_depth_colormap = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + near_plane=torch.min(ground_truth_depth), + far_plane=torch.max(ground_truth_depth), + ) + images["depth"] = torch.cat([ground_truth_depth_colormap, predicted_depth_colormap], dim=1) + metrics["depth_mse"] = torch.nn.functional.mse_loss(outputs["depth"], ground_truth_depth) + return metrics, images + + def _get_sigma(self): + if not self.config.should_decay_sigma: + return self.depth_sigma + + self.depth_sigma = torch.maximum( # pylint: disable=attribute-defined-outside-init + self.config.sigma_decay_rate * self.depth_sigma, torch.tensor([self.config.depth_sigma]) + ) + return self.depth_sigma diff --git a/tests/cameras/test_cameras.py b/tests/cameras/test_cameras.py index b33f43860..4f1b469cc 100644 --- a/tests/cameras/test_cameras.py +++ b/tests/cameras/test_cameras.py @@ -281,7 +281,14 @@ def check_generate_rays_shape(): def _check_dataclass_allclose(ipt, other): for field in dataclasses.fields(ipt): if getattr(ipt, field.name) is not None: - assert torch.allclose(getattr(ipt, field.name), getattr(other, field.name)) + if isinstance(getattr(ipt, field.name), dict): + ipt_dict = getattr(ipt, field.name) + other_dict = getattr(other, field.name) + for k, v in ipt_dict.items(): + assert k in other_dict + assert torch.allclose(v, other_dict[k]) + else: + assert torch.allclose(getattr(ipt, field.name), getattr(other, field.name)) return True diff --git a/tests/test_train.py b/tests/test_train.py index 63a8099e9..a3f1b6c60 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -15,7 +15,15 @@ from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConf from nerfstudio.engine.trainer import TrainerConfig from scripts.train import train_loop -BLACKLIST = ["base", "semantic-nerfw", "instant-ngp", "instant-ngp-bounded", "nerfacto", "phototourism"] +BLACKLIST = [ + "base", + "semantic-nerfw", + "instant-ngp", + "instant-ngp-bounded", + "nerfacto", + "phototourism", + "depth-nerfacto", +] def set_reduced_config(config: TrainerConfig): From 102c00a9be1aba1fad654a0f19d85ab2a542ca63 Mon Sep 17 00:00:00 2001 From: Rajendra Singh Date: Tue, 3 Jan 2023 22:31:46 +0530 Subject: [PATCH 34/38] Added missing arg to ns-download-data command (#1201) Correct ns-download-data command `ns-download-data nerfstudio --capture-name=vegetation` will give error: ``` usage: ns-download-data [-h] --dataset {blender,friends,nerfstudio,record3d} [--capture-name {None,bww_entrance,campanile,desolation,dozer,library,poster,redwoods2,sf_street,storefront,vegetation}] [--save-dir PATH] ns-download-data: error: the following arguments are required: --dataset ``` Thus adding `--dataset` to the command. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 923753098..46cf7fff6 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ The following will train a _nerfacto_ model, our recommended model for real worl ```bash # Download some test data: -ns-download-data nerfstudio --capture-name=poster +ns-download-data --dataset nerfstudio --capture-name=poster # Train model ns-train nerfacto --data data/nerfstudio/poster ``` From 31be97d59c828f4aa036085fa81d5a3bc8026576 Mon Sep 17 00:00:00 2001 From: Liam Schoneveld Date: Wed, 4 Jan 2023 14:23:20 +0900 Subject: [PATCH 35/38] Render cameras with render times for dynamic nerfs (#1199) * Render cameras with render times for dynamic nerfs * Add --dynamic flag to cmd. Clamp render times. * Fix for issues with clamping renderTime * Only add render_time to camera path if dynamic nerf * change DEFAULT_TIME to DEFAULT_RENDER_TIME Co-authored-by: Liam Schoneveld --- nerfstudio/cameras/camera_paths.py | 7 + .../SidePanel/CameraPanel/CameraPanel.jsx | 205 +++++++++++++++++- .../modules/SidePanel/CameraPanel/curve.js | 5 + 3 files changed, 211 insertions(+), 6 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index f5eb67501..475814f5d 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -152,6 +152,12 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: fxs.append(focal_length) fys.append(focal_length) + # Iff ALL cameras in the path have a "time" value, construct Cameras with times + if all("render_time" in camera for camera in camera_path["camera_path"]): + times = torch.tensor([camera["render_time"] for camera in camera_path["camera_path"]]) + else: + times = None + camera_to_worlds = torch.stack(c2ws, dim=0) fx = torch.tensor(fxs) fy = torch.tensor(fys) @@ -162,4 +168,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: cy=image_height / 2, camera_to_worlds=camera_to_worlds, camera_type=camera_type, + times=times, ) diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index e74eaf90e..6dc11d30a 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -63,6 +63,104 @@ function set_camera_position(camera, matrix) { mat.decompose(camera.position, camera.quaternion, camera.scale); } + +function RenderTimeSelector(props) { + const disabled = props.disabled; + const isGlobal = props.isGlobal; + const camera = props.camera; + const dispatch = props.dispatch; + const globalRenderTime = props.globalRenderTime; + const setGlobalRenderTime = props.setGlobalRenderTime; + const applyAll = props.applyAll; + const changeMain = props.changeMain; + const setAllCameraRenderTime = props.setAllCameraRenderTime; + + const getRenderTimeLabel = () => { + if (!isGlobal) { + return camera.renderTime; + } + camera.renderTime = globalRenderTime + return globalRenderTime; + }; + + const [UIRenderTime, setUIRenderTime] = React.useState( + isGlobal ? globalRenderTime : getRenderTimeLabel(), + ); + + const [valid, setValid] = React.useState(true); + + useEffect( + () => setUIRenderTime(getRenderTimeLabel()), + [camera, globalRenderTime], + ); + + const setRndrTime = (val) => { + if (!isGlobal) { + camera.renderTime = val; + } else { + camera.renderTime = val; + setGlobalRenderTime(val); + } + + if (applyAll) { + setAllCameraRenderTime(val); + } + + if (changeMain) { + dispatch({ + type: 'write', + path: 'renderingState/render_time', + data: camera.renderTime, + }); + } + }; + + const handleValidation = (e) => { + const valueFloat = parseFloat(e.target.value); + let valueStr = String(valueFloat); + if (e.target.value >= 0 && e.target.value <= 1){ + setValid(true); + if (valueFloat === 1.0) { + valueStr = '1.0'; + } + if (valueFloat === 0.0) { + valueStr = '0.0'; + } + setUIRenderTime(parseFloat(valueStr)); + setRndrTime(parseFloat(valueStr)); + } else { + setValid(false); + } + }; + + return ( + setUIRenderTime(e.target.value)} + onBlur={(e) => handleValidation(e)} + disabled={disabled} + sx={{ + input: { + '-webkit-text-fill-color': `${ + disabled ? '#24B6FF' : '#EBEBEB' + } !important`, + color: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, + }, + }} + value={UIRenderTime} + error={!valid} + helperText={!valid ? 'RenderTime should be between 0.0 and 1.0' : ''} + variant="standard" + /> + ); +} + function FovSelector(props) { const fovLabel = props.fovLabel; const setFovLabel = props.setFovLabel; @@ -279,6 +377,7 @@ function CameraList(props) { set_camera_position(camera_render, first_camera.matrix); camera_render_helper.set_visibility(true); camera_render.fov = first_camera.fov; + camera_render.renderTime = first_camera.renderTime; } set_slider_value(slider_min); }; @@ -392,6 +491,7 @@ function CameraList(props) { e.stopPropagation(); set_camera_position(camera_main, camera.matrix); camera_main.fov = camera.fov; + camera_main.renderTime = camera.renderTime; set_slider_value(camera.properties.get('TIME')); }} > @@ -414,7 +514,16 @@ function CameraList(props) { changeMain={false} /> )} - {!isAnimated('FOV') && ( + {isAnimated('RenderTime') && ( + + )} + {!isAnimated('FOV') && !isAnimated('RenderTime') && (

Animated camera properties will show up here!

@@ -452,6 +561,7 @@ export default function CameraPanel(props) { ); const websocket = useContext(WebSocketContext).socket; const DEFAULT_FOV = 50; + const DEFAULT_RENDER_TIME = 0.0; // react state const [cameras, setCameras] = React.useState([]); @@ -466,6 +576,7 @@ export default function CameraPanel(props) { const [render_modal_open, setRenderModalOpen] = React.useState(false); const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); + const [globalRenderTime, setGlobalRenderTime] = React.useState(DEFAULT_RENDER_TIME); // leva store const cameraPropsStore = useCreateStore(); @@ -491,6 +602,17 @@ export default function CameraPanel(props) { ); const camera_type = useSelector((state) => state.renderingState.camera_type); + const [display_render_time, set_display_render_time] = React.useState(false); + + const receive_temporal_dist = e => { + const msg = msgpack.decode(new Uint8Array(e.data)); + if (msg.path === "/model/has_temporal_distortion") { + set_display_render_time(msg.data === "true"); + websocket.removeEventListener("message", receive_temporal_dist); + } + } + websocket.addEventListener('message', receive_temporal_dist); + const setRenderHeight = (value) => { dispatch({ type: 'write', @@ -522,6 +644,14 @@ export default function CameraPanel(props) { }); }; + const setRenderTime = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/render_time', + data: parseFloat(value), + }); + }; + // ui state const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); @@ -538,6 +668,7 @@ export default function CameraPanel(props) { if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); setFieldOfView(new_camera_list[0].fov); + setRenderTime(new_camera_list[0].renderTime) set_slider_value(slider_min); } }; @@ -546,6 +677,7 @@ export default function CameraPanel(props) { const camera_main_copy = camera_main.clone(); camera_main_copy.aspect = 1.0; camera_main_copy.fov = globalFov; + camera_main_copy.renderTime = globalRenderTime; const new_camera_properties = new Map(); camera_main_copy.properties = new_camera_properties; new_camera_properties.set('FOV', globalFov); @@ -828,11 +960,21 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); - camera_path.push({ - camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix - fov, - aspect: camera_render.aspect, - }); + if (display_render_time) { + const renderTime = curve_object.curve_render_times.getPoint(pt).z; + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] + }); + } else { + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + }); + } } const keyframes = []; @@ -985,6 +1127,12 @@ export default function CameraPanel(props) { } }; + const setAllCameraRenderTime = (val) => { + for (let i = 0; i < cameras.length; i += 1) { + cameras[i].renderTime = val; + } + }; + return (
@@ -1047,6 +1195,51 @@ export default function CameraPanel(props) { />
+ {display_render_time && ( +
+ + + + +
+ )} {camera_type !== 'equirectangular' && (
diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js index 7715a0df9..5cf956f2d 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js @@ -30,6 +30,7 @@ export function get_curve_object_from_cameras( const lookats = []; const ups = []; const fovs = []; + const render_times = []; for (let i = 0; i < cameras.length; i += 1) { const camera = cameras[i]; @@ -45,23 +46,27 @@ export function get_curve_object_from_cameras( lookats.push(lookat); // Reuse catmullromcurve3 for 1d values. TODO fix this fovs.push(new THREE.Vector3(0, 0, camera.fov)); + render_times.push(new THREE.Vector3(0, 0, camera.renderTime)); } let curve_positions = null; let curve_lookats = null; let curve_ups = null; let curve_fovs = null; + let curve_render_times = null; curve_positions = get_catmull_rom_curve(positions, is_cycle, smoothness_value); curve_lookats = get_catmull_rom_curve(lookats, is_cycle, smoothness_value); curve_ups = get_catmull_rom_curve(ups, is_cycle, smoothness_value); curve_fovs = get_catmull_rom_curve(fovs, is_cycle, smoothness_value / 10); + curve_render_times = get_catmull_rom_curve(render_times, is_cycle, smoothness_value); const curve_object = { curve_positions, curve_lookats, curve_ups, curve_fovs, + curve_render_times, }; return curve_object; } From 99ab8bd669a590efcbce0ff27727b16e0403d71d Mon Sep 17 00:00:00 2001 From: Rohan Mathur Date: Wed, 4 Jan 2023 00:26:09 -0500 Subject: [PATCH 36/38] Added ETA to base config for the event writer (#1203) added eta to base config --- nerfstudio/configs/base_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index aa0177ce3..9131a50d5 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -88,6 +88,7 @@ class LocalWriterConfig(InstantiateConfig): writer.EventName.CURR_TEST_PSNR, writer.EventName.VIS_RAYS_PER_SEC, writer.EventName.TEST_RAYS_PER_SEC, + writer.EventName.ETA, ) """specifies which stats will be logged/printed to terminal""" max_log_size: int = 10 From 86d8118b6d02da538373e661efa0ddc5645b1f59 Mon Sep 17 00:00:00 2001 From: lightfield botanist <74843139+3a1b2c3@users.noreply.github.com> Date: Sat, 7 Jan 2023 14:22:55 +1000 Subject: [PATCH 37/38] fix notebook, box drawing (#1202) * fix notebook * samples * run all cells * box center * try saving again * import order * cube with different sides * black * black * update metadata * fix test Co-authored-by: Katrin Schmid Co-authored-by: Matt Tancik --- .../model_components/visualize_cameras.ipynb | 7521 +++++++++-------- .../model_components/visualize_samples.ipynb | 116 +- nerfstudio/utils/plotly_utils.py | 14 +- tests/utils/test_visualization.py | 18 +- 4 files changed, 3885 insertions(+), 3784 deletions(-) diff --git a/docs/nerfology/model_components/visualize_cameras.ipynb b/docs/nerfology/model_components/visualize_cameras.ipynb index 0ae46b222..99bb095a0 100644 --- a/docs/nerfology/model_components/visualize_cameras.ipynb +++ b/docs/nerfology/model_components/visualize_cameras.ipynb @@ -56,13 +56,99 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 2, "metadata": { "tags": [ "hide-input" ] }, "outputs": [ + { + "data": { + "text/html": [ + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "application/vnd.plotly.v1+json": { @@ -13736,9 +13822,9 @@ } }, "text/html": [ - "