From 5630148c50c776fbf1fa16b69784408949697781 Mon Sep 17 00:00:00 2001 From: Alexander Kristoffersen Date: Tue, 13 Dec 2022 23:04:17 -0800 Subject: [PATCH 01/24] Changed tinycudann to wheel to work with py3.8 (#1126) Created using Colaboratory --- colab/demo.ipynb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/colab/demo.ipynb b/colab/demo.ipynb index 9609a6db4..148da3b88 100644 --- a/colab/demo.ipynb +++ b/colab/demo.ipynb @@ -7,7 +7,7 @@ "colab_type": "text" }, "source": [ - "\"Open" + "\"Open" ] }, { @@ -63,25 +63,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "RGr33zHaHak0", - "outputId": "2641dce6-cd57-4d31-d8b1-860bc1adf17f" + "outputId": "6ad38334-22d2-4675-bffb-0bae91b17577" }, "outputs": [ { - "name": "stdout", "output_type": "stream", + "name": "stdout", "text": [ "⏬ Downloading https://github.com/jaimergp/miniforge/releases/latest/download/Mambaforge-colab-Linux-x86_64.sh...\n", "📦 Installing...\n", "📌 Adjusting configuration...\n", "🩹 Patching environment...\n", - "⏲ Done in 0:00:27\n", + "⏲ Done in 0:00:18\n", "🔁 Restarting kernel...\n" ] } @@ -111,9 +111,8 @@ "\n", "# Installing TinyCuda\n", "%cd /content/\n", - "# !gdown \"https://drive.google.com/u/1/uc?id=1wglEFPbI4BjWMFYaKk7gHCNsWCZphKjs\" \n", - "# !pip install tinycudann-1.6-cp38-cp38-linux_x86_64.whl\n", - "!pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch\n", + "!gdown \"https://drive.google.com/u/0/uc?id=1mB3IxcpmTVmQN-So0t2lxjxbHvqWr2zf\" \n", + "!pip install tinycudann-1.5-cp38-cp38-linux_x86_64.whl\n", "\n", "# Installing COLMAP\n", "%cd /content/\n", From d51718ec2bc411b6a0ac16d017f80ae96d3aceb0 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 14 Dec 2022 16:26:05 -0800 Subject: [PATCH 02/24] Immediately update viewer when output or cmap changes (#1131) * Immediately update viewer when output or cmap changes * lint --- nerfstudio/viewer/server/viewer_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index ce70426f6..c3ebb950e 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -454,6 +454,19 @@ class ViewerState: else: self.prev_camera_matrix = camera_object["matrix"] self.camera_moving = True + + output_type = self.vis["renderingState/output_choice"].read() + if output_type is None: + output_type = OutputTypes.INIT + if self.prev_output_type != output_type: + self.camera_moving = True + + colormap_type = self.vis["renderingState/colormap_choice"].read() + if colormap_type is None: + colormap_type = ColormapTypes.INIT + if self.prev_colormap_type != colormap_type: + self.camera_moving = True + return camera_object def _apply_colormap(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6): From bc690632bbffee4d21dd504985e13bca2510bf13 Mon Sep 17 00:00:00 2001 From: Penn Date: Thu, 15 Dec 2022 13:24:31 -0800 Subject: [PATCH 03/24] Use `torch.where` for faster contract (#1132) use where for faster contract --- nerfstudio/field_components/spatial_distortions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nerfstudio/field_components/spatial_distortions.py b/nerfstudio/field_components/spatial_distortions.py index 449f4b5b9..02da548a3 100644 --- a/nerfstudio/field_components/spatial_distortions.py +++ b/nerfstudio/field_components/spatial_distortions.py @@ -65,12 +65,8 @@ class SceneContraction(SpatialDistortion): def forward(self, positions): def contract(x): - mag = torch.linalg.norm(x, ord=self.order, dim=-1) - mask = mag >= 1 - x_new = x.clone() - x_new[mask] = (2 - (1 / mag[mask][..., None])) * (x[mask] / mag[mask][..., None]) - - return x_new + mag = torch.linalg.norm(x, ord=self.order, dim=-1)[..., None] + return torch.where(mag < 1, x, (2 - (1 / mag)) * (x / mag)) if isinstance(positions, Gaussians): means = contract(positions.mean.clone()) From 3da0ca727d560e264cfaa4425c8818fab62601bf Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 15 Dec 2022 14:18:23 -0800 Subject: [PATCH 04/24] Update render ui to use leva (#1133) --- nerfstudio/viewer/app/src/index.scss | 29 +++- .../SidePanel/CameraPanel/CameraPanel.jsx | 144 ++++-------------- .../SidePanel/CameraPanel/CameraPropPanel.jsx | 73 +++++++++ 3 files changed, 124 insertions(+), 122 deletions(-) create mode 100644 nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx diff --git a/nerfstudio/viewer/app/src/index.scss b/nerfstudio/viewer/app/src/index.scss index 194c5ac2e..98e968795 100644 --- a/nerfstudio/viewer/app/src/index.scss +++ b/nerfstudio/viewer/app/src/index.scss @@ -242,18 +242,37 @@ $sidebar-width: 100% !default; // ------------------------------------ // Camera Pane -.CameraPanel { +.CameraPanel-props { + margin-top: 10px; + margin-bottom: 10px; + margin-left: 20px; + margin-right: 20px; +} + +.CameraPanel-path-row { + display: inline-block; + width: 45%; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 4px; + margin-right: 4px; +} + +.CameraPanel-row { + display: inline-block; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 4px; + margin-right: 4px; } .CameraPanel-top-button { - display: inline-block; - margin: 4px; - margin-top: 8px; + width: 100%; } .CameraPanel-render-button { display: inline-block; - width: 70%; + width: 92%; margin: 4px; margin-top: 8px; } diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 6a741d156..9958ea7b1 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -37,6 +37,7 @@ import { useDispatch, useSelector } from 'react-redux'; import { CSS2DObject } from 'three/examples/jsm/renderers/CSS2DRenderer'; import AddAPhotoIcon from '@mui/icons-material/AddAPhoto'; import FileDownloadOutlinedIcon from '@mui/icons-material/FileDownloadOutlined'; +import { LevaPanel, LevaStoreProvider, useCreateStore } from 'leva'; import FileUploadOutlinedIcon from '@mui/icons-material/FileUploadOutlined'; import { Stack } from '@mui/system'; import TextField from '@mui/material/TextField'; @@ -46,6 +47,8 @@ import { CameraHelper } from './CameraHelper'; import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; import { WebSocketContext } from '../../WebSocket/WebSocket'; import RenderModal from '../../RenderModal'; +import CameraPropPanel from './CameraPropPanel'; +import LevaTheme from '../../../themes/leva_theme.json'; const msgpack = require('msgpack-lite'); @@ -460,6 +463,9 @@ export default function CameraPanel(props) { const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); + // leva store + const cameraPropsStore = useCreateStore(); + const scene_state = sceneTree.get_scene_state(); // Template for sharing state between Vanilla JS Three.js and React components @@ -496,10 +502,6 @@ export default function CameraPanel(props) { }; // ui state - const [ui_render_height, setUIRenderHeight] = React.useState(render_height); - const [ui_render_width, setUIRenderWidth] = React.useState(render_width); - const [ui_seconds, setUISeconds] = React.useState(seconds); - const [ui_fps, setUIfps] = React.useState(fps); const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); // nonlinear render option @@ -868,15 +870,10 @@ export default function CameraPanel(props) { const new_properties = new Map(cameraProperties); setRenderHeight(camera_path_object.render_height); - setUIRenderHeight(camera_path_object.render_height); setRenderWidth(camera_path_object.render_width); - setUIRenderWidth(camera_path_object.render_width); setFps(camera_path_object.fps); - setUIfps(camera_path_object.fps); - setSeconds(camera_path_object.seconds); - setUISeconds(camera_path_object.seconds); set_smoothness_value(camera_path_object.smoothness_value); setIsCycle(camera_path_object.is_cycle); @@ -968,7 +965,7 @@ export default function CameraPanel(props) { return (
-
+
-
+
-
- { - if (e.target.validity.valid) { - setUIRenderHeight(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setRenderHeight(parseInt(e.target.value, 10)); - } else { - setUIRenderHeight(render_height); - } - } - }} - value={ui_render_height} - error={ui_render_height <= 0} - helperText={ui_render_height <= 0 ? 'Required' : ''} - variant="standard" - /> - { - if (e.target.validity.valid) { - setUIRenderWidth(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setRenderWidth(parseInt(e.target.value, 10)); - } else { - setUIRenderWidth(render_width); - } - } - }} - value={ui_render_width} - error={ui_render_width <= 0} - helperText={ui_render_width <= 0 ? 'Required' : ''} - variant="standard" - /> -
-
- { - if (e.target.validity.valid) { - setUISeconds(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setSeconds(parseInt(e.target.value, 10)); - } else { - setUISeconds(seconds); - } - } - }} - value={ui_seconds} - error={ui_seconds <= 0} - helperText={ui_seconds <= 0 ? 'Required' : ''} - variant="standard" - /> - { - if (e.target.validity.valid) { - setUIfps(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setFps(parseInt(e.target.value, 10)); - } else { - setUIfps(fps); - } - } - }} - value={ui_fps} - error={ui_fps <= 0} - helperText={ui_fps <= 0 ? 'Required' : ''} - variant="standard" +
+ + + +
@@ -1161,7 +1071,7 @@ export default function CameraPanel(props) { />
-
+
-
+
-
+
@@ -115,7 +119,8 @@ export default function StatusPanel(props: StatusPanelProps) { set_is_images_visible(!is_images_visible); }} style={{ textTransform: 'none' }} - startIcon={} + startIcon={} + disabled={camera_choice === 'Render Camera'} > {cameras_button} diff --git a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index 90fe975e0..94990f2d9 100644 --- a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -9,6 +9,8 @@ import OpenWithIcon from '@mui/icons-material/OpenWith'; import PublicOffSharpIcon from '@mui/icons-material/PublicOffSharp'; import PublicSharpIcon from '@mui/icons-material/PublicSharp'; import SyncOutlinedIcon from '@mui/icons-material/SyncOutlined'; +import ThreeDRotationIcon from '@mui/icons-material/ThreeDRotation'; +import VideoCameraBackIcon from '@mui/icons-material/VideoCameraBackOutlined'; import WebRtcWindow from '../WebRtcWindow/WebRtcWindow'; import { WebSocketContext } from '../WebSocket/WebSocket'; @@ -19,27 +21,36 @@ function CameraToggle() { const camera_choice = useSelector( (state) => state.renderingState.camera_choice, ); - const set_camera_choice = (event: SelectChangeEvent) => { - const value = event.target.value; - dispatch({ - type: 'write', - path: 'renderingState/camera_choice', - data: value, - }); + const set_camera_choice = (event: SelectChangeEvent, value: string[]) => { + if (value != null) { + dispatch({ + type: 'write', + path: 'renderingState/camera_choice', + data: value, + }); + } }; return ( - Viewport - Render View + + + Viewport + + + + Render View + ); } diff --git a/nerfstudio/viewer/app/src/themes/theme.ts b/nerfstudio/viewer/app/src/themes/theme.ts index 554445561..4a973209e 100644 --- a/nerfstudio/viewer/app/src/themes/theme.ts +++ b/nerfstudio/viewer/app/src/themes/theme.ts @@ -126,10 +126,13 @@ export const appTheme = createTheme({ styleOverrides: { root: { color: '#eeeeee', - backgroundColor: '#393e46', + backgroundColor: 'FFD369', '&.Mui-selected': { - color: '#FFD369', - backgroundColor: '#555555', + color: '#222831', + backgroundColor: '#FFD369', + ':hover': { + backgroundColor: '#FFD369', + }, }, '&.Mui-disabled': { color: '#999999', From 3bf490e845897a74db278e027f085967758082a2 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Thu, 15 Dec 2022 17:29:41 -0800 Subject: [PATCH 06/24] adding data subset (#1135) * adding nerfstudio-dataset * removed file --- scripts/downloads/download_data.py | 68 +++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index c444991a9..12a0a0227 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -15,6 +15,7 @@ from rich.console import Console from typing_extensions import Annotated from nerfstudio.configs.base_config import PrintableConfig +from nerfstudio.utils.scripts import run_command CONSOLE = Console(width=120) @@ -84,17 +85,39 @@ def grab_file_id(zip_url: str) -> str: return s.split("/")[0] +nerfstudio_dataset = [ + "Egypt", + "person", + "kitchen", + "plane", + "dozer", + "floating-tree", + "aspen", + "stump", + "sculpture", + "Giannini-Hall", +] nerfstudio_file_ids = { "bww_entrance": grab_file_id("https://drive.google.com/file/d/1ylkRHtfB3n3IRLf2wplpfxzPTq7nES9I/view?usp=sharing"), "campanile": grab_file_id("https://drive.google.com/file/d/13aOfGJRRH05pOOk9ikYGTwqFc2L1xskU/view?usp=sharing"), "desolation": grab_file_id("https://drive.google.com/file/d/14IzOOQm9KBJ3kPbunQbUTHPnXnmZus-f/view?usp=sharing"), - "dozer": grab_file_id("https://drive.google.com/file/d/1-OR5F_V5S4s-yzxohbwTylaXjzYLu8ZR/view?usp=sharing"), "library": grab_file_id("https://drive.google.com/file/d/1Hjbh_-BuaWETQExn2x2qGD74UwrFugHx/view?usp=sharing"), "poster": grab_file_id("https://drive.google.com/file/d/1dmjWGXlJnUxwosN6MVooCDQe970PkD-1/view?usp=sharing"), "redwoods2": grab_file_id("https://drive.google.com/file/d/1rg-4NoXT8p6vkmbWxMOY6PSG4j3rfcJ8/view?usp=sharing"), "storefront": grab_file_id("https://drive.google.com/file/d/16b792AguPZWDA_YC4igKCwXJqW0Tb21o/view?usp=sharing"), "vegetation": grab_file_id("https://drive.google.com/file/d/1wBhLQ2odycrtU39y2akVurXEAt9SsVI3/view?usp=sharing"), + "Egypt": "https://data.nerf.studio/nerfstudio/Egypt.zip", + "person": "https://data.nerf.studio/nerfstudio/person.zip", + "kitchen": "https://data.nerf.studio/nerfstudio/kitchen.zip", + "plane": "https://data.nerf.studio/nerfstudio/plane.zip", + "dozer": "https://data.nerf.studio/nerfstudio/dozer.zip", + "floating-tree": "https://data.nerf.studio/nerfstudio/floating-tree.zip", + "aspen": "https://data.nerf.studio/nerfstudio/aspen.zip", + "stump": "https://data.nerf.studio/nerfstudio/stump.zip", + "sculpture": "https://data.nerf.studio/nerfstudio/sculpture.zip", + "Giannini-Hall": "https://data.nerf.studio/nerfstudio/Giannini-Hall.zip", "all": None, + "nerfstudio-dataset": nerfstudio_dataset, } NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_ids.keys()) @@ -103,14 +126,33 @@ NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_id def download_capture_name(save_dir: Path, dataset_name: str, capture_name: str, capture_name_to_file_id: dict): """Download specific captures a given dataset and capture name.""" - url = f"https://drive.google.com/uc?id={capture_name_to_file_id[capture_name]}" - target_path = str(save_dir / f"{dataset_name}/{capture_name}") - os.makedirs(target_path, exist_ok=True) - download_path = Path(f"{target_path}.zip") - tmp_path = str(save_dir / ".temp") - shutil.rmtree(tmp_path, ignore_errors=True) - os.makedirs(tmp_path, exist_ok=True) - gdown.download(url, output=str(download_path)) + file_id_or_zip_url = capture_name_to_file_id[capture_name] + if file_id_or_zip_url.endswith(".zip"): + url = file_id_or_zip_url # zip url + target_path = str(save_dir / f"{dataset_name}/{capture_name}") + os.makedirs(target_path, exist_ok=True) + download_path = Path(f"{target_path}.zip") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + try: + os.remove(download_path) + except OSError: + pass + run_command(f"wget {url} -O {download_path}", verbose=True) + else: + url = f"https://drive.google.com/uc?id={file_id_or_zip_url}" # file id + target_path = str(save_dir / f"{dataset_name}/{capture_name}") + os.makedirs(target_path, exist_ok=True) + download_path = Path(f"{target_path}.zip") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + try: + os.remove(download_path) + except OSError: + pass + gdown.download(url, output=str(download_path)) with zipfile.ZipFile(download_path, "r") as zip_ref: zip_ref.extractall(tmp_path) inner_folders = os.listdir(tmp_path) @@ -132,7 +174,13 @@ class NerfstudioDownload(DatasetDownload): """Download the nerfstudio dataset.""" if self.capture_name == "all": for capture_name in nerfstudio_file_ids: - if capture_name != "all": + if capture_name not in ("all", "nerfstudio-dataset"): + download_capture_name(save_dir, "nerfstudio", capture_name, nerfstudio_file_ids) + return + + if self.capture_name == "nerfstudio-dataset": + for capture_name in nerfstudio_dataset: + if capture_name not in ("all", "nerfstudio-dataset"): download_capture_name(save_dir, "nerfstudio", capture_name, nerfstudio_file_ids) return From 29df5943343a0f7d6fa4bff2918880501c97d07b Mon Sep 17 00:00:00 2001 From: Nikhil Mohan <47728081+nikmo33@users.noreply.github.com> Date: Fri, 16 Dec 2022 20:16:20 +0000 Subject: [PATCH 07/24] Minimal disk dataparser (#1128) * wip * merge upstream * clkeanup * cleanup * cleanup * cleanup * more cleanup * cleanup * cleanup docs * lint * style fix --- .../data/datamanagers/base_datamanager.py | 2 + .../data/datamanagers/semantic_datamanager.py | 1 + .../data/dataparsers/minimal_dataparser.py | 97 ++++++++++++++++++ tests/data/minimal_parser/train.npz | Bin 0 -> 8422 bytes tests/data/minimal_parser/val.npz | Bin 0 -> 8422 bytes tests/test_train.py | 10 ++ 6 files changed, 110 insertions(+) create mode 100644 nerfstudio/data/dataparsers/minimal_dataparser.py create mode 100644 tests/data/minimal_parser/train.npz create mode 100644 tests/data/minimal_parser/val.npz diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 140d018ba..49b759a9d 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -42,6 +42,7 @@ from nerfstudio.data.dataparsers.friends_dataparser import FriendsDataParserConf from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( InstantNGPDataParserConfig, ) +from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConfig from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserConfig from nerfstudio.data.dataparsers.phototourism_dataparser import ( @@ -65,6 +66,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of tyro.extras.subcommand_type_from_defaults( { "nerfstudio-data": NerfstudioDataParserConfig(), + "minimal-parser": MinimalDataParserConfig(), "blender-data": BlenderDataParserConfig(), "friends-data": FriendsDataParserConfig(), "instant-ngp-data": InstantNGPDataParserConfig(), diff --git a/nerfstudio/data/datamanagers/semantic_datamanager.py b/nerfstudio/data/datamanagers/semantic_datamanager.py index 188c7f6c3..322b1d3bd 100644 --- a/nerfstudio/data/datamanagers/semantic_datamanager.py +++ b/nerfstudio/data/datamanagers/semantic_datamanager.py @@ -41,6 +41,7 @@ class SemanticDataManager(VanillaDataManager): # pylint: disable=abstract-metho """ def create_train_dataset(self) -> SemanticDataset: + self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") return SemanticDataset( dataparser_outputs=self.train_dataparser_outputs, scale_factor=self.config.camera_res_scale_factor, diff --git a/nerfstudio/data/dataparsers/minimal_dataparser.py b/nerfstudio/data/dataparsers/minimal_dataparser.py new file mode 100644 index 000000000..b6ffd8518 --- /dev/null +++ b/nerfstudio/data/dataparsers/minimal_dataparser.py @@ -0,0 +1,97 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Data parser for pre-prepared datasets for all cameras, with no additional processing needed +Optional fields - semantics, mask_filenames, cameras.distortion_params, cameras.times +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Type + +import numpy as np +import torch + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.data.dataparsers.base_dataparser import ( + DataParser, + DataParserConfig, + DataparserOutputs, + Semantics, +) +from nerfstudio.data.scene_box import SceneBox + + +@dataclass +class MinimalDataParserConfig(DataParserConfig): + """Minimal dataset config""" + + _target: Type = field(default_factory=lambda: MinimalDataParser) + """target class to instantiate""" + data: Path = Path("/home/nikhil/nerfstudio-main/tests/data/lego_test/minimal_parser") + + +@dataclass +class MinimalDataParser(DataParser): + """Minimal DatasetParser""" + + config: MinimalDataParserConfig + + def _generate_dataparser_outputs(self, split="train"): + filepath = self.config.data / f"{split}.npz" + data = np.load(filepath, allow_pickle=True) + + image_filenames = data["image_filenames"].tolist() + mask_filenames = data["mask_filenames"].tolist() if "mask_filenames" in data.keys() else None + + metadata = None + if "semantics" in data.keys(): + semantics = data["semantics"].item() + metadata = { + "semantics": Semantics( + filenames=semantics["filenames"].tolist(), + classes=semantics["classes"].tolist(), + colors=semantics["colors"].tolist(), + mask_classes=semantics["mask_classes"].tolist(), + ) + } + + scene_box_aabb = torch.from_numpy(data["scene_box"]) + scene_box = SceneBox(aabb=scene_box_aabb) + + camera_np = data["cameras"].item() + distortion_params = None + if "distortion_params" in camera_np.keys(): + distortion_params = torch.from_numpy(camera_np["distortion_params"]) + cameras = Cameras( + fx=torch.from_numpy(camera_np["fx"]), + fy=torch.from_numpy(camera_np["fy"]), + cx=torch.from_numpy(camera_np["cx"]), + cy=torch.from_numpy(camera_np["cy"]), + distortion_params=distortion_params, + height=torch.from_numpy(camera_np["height"]), + width=torch.from_numpy(camera_np["width"]), + camera_to_worlds=torch.from_numpy(camera_np["camera_to_worlds"])[:, :3, :4], + camera_type=torch.from_numpy(camera_np["camera_type"]), + times=torch.from_numpy(camera_np["times"]) if "times" in camera_np.keys() else None, + ) + + dataparser_outputs = DataparserOutputs( + image_filenames=image_filenames, + cameras=cameras, + scene_box=scene_box, + mask_filenames=mask_filenames, + metadata=metadata, + ) + return dataparser_outputs diff --git a/tests/data/minimal_parser/train.npz b/tests/data/minimal_parser/train.npz new file mode 100644 index 0000000000000000000000000000000000000000..9d8acbb15f7b0060ffaa046218d6aff7b79c0aff GIT binary patch literal 8422 zcmeHMdvH|M8Q(leAV3HZAPC5!LIQ;FNC*LvdoTg5ERVQ_&URWR>~1!>Ta#?g?hO#c zG2j5AqC8umh=72!+KPZRq9j@Yu>%OGQ&c)-%0oM&6cLr8)qcN|{npKrO>Tz%+0Xer z_nz~;zu!3fnm(~@fORyxZ)C1u_UPnEOM6UeHnu8pyz zsAZ@?477H}?CHfnkFm_NJeXeU3Ap{~6RhcrW~OIa)60B*!|$vr@%c+Vn7+qZ5%AD_ zKzCMq=sjah=Fm**L)KfNq`EkFRZV4eZI;{T_hePpR2W{T-|wtVD)D>VzN&!XuW=hm z($XZAR8`uvDr0Pxr82fC#wFvH%Xqu&kYS6l#V(ZzLBUd^w%U^<6CJUX62H$^DZ4v5(vs>#pd?53 zaHIt#OMW_f%#=~1OGYWm8<+K~>T=oBkg0arOHaM`Fd?8NABUi+TIpo+{tjG_wL`~X5 zh!4KlubEoHKI_3J@_GQ1CJ1q&9`FYnW{LelkMY9&w9*{vg!gh{otSMymg|Z7S&6X@ z43rXc?#P>U=rM#(#EL#xgLCX7TAd(^7-1JRfjndpmz5Z}KpgOe5pp2Z1@DkWJrScB zVqC)x@-a({>+qb-=)wnTf_&OT$cOgO)Gu}(c5pz~zt}S{34kwNg@@?f5uE#dv1A6?yIrxGu>oazv>0ll?z&>QH#Abi*QAfms-!m|Ln<2K3{sG@+VuW4J4>&bb zN9cf{$G=I47qu}%TzACCHq7YpUO--8#EP7)#Ed=E3i*fUj=r-jJ(?!i=JSO|V9iPl z`|t(Zk(kFC`VJjl3y+7UTO`Qf>~1f?2Vy}D*f*ZrBK~(3`hc^>Z12%$$g)0sn$cx^ z;ol3W8)}5OVGp&AM!*Q{BTulE82R!3Lcd$`kND67=<~kC83UjE8%d8h5#r)?*fvXC zd)P-kxh`$(3&=9Q+Vi`@eF*=1i_b!=Ar6iyJO-{k@@$Vk)DiWE9-htWqLy5fmik1? zEfVSiKd23!!SIb-@Enb}?!Xinc(W4oIe=Kg>p`!IBF{1_%|aithyk&;q{lTzY;&jpM`XM5DNR)YBA(k`Yd!TuDP?no&BZQwsmX47gwsae_g(;y|iak!QwiL#!^o` znJoDElS^gdgZJN8YZot+s(I+WbM&a1BRipI=&XF=iIAN)-S1X&}-ngN?T3@f7K6Sr(-|(A0;^6D~;?&9Fpl^!>xlkQxy-vJ+3wk&6~BC_vff*taDuv8}zSz84$Ft?c2Lo#j<=^f&jmJ;)J$l zb$79TU8-PN#PHgI3F64%DFQWt{tM3!SMrPN>Lz~t(MKB9N&E1OLp%BIO!4?*NlFGe zz;|OqgSL8AN&))#)ir}^y<2I@alN+%$6-w1H01Mw|5NnX216D>kCxQ zz&qij;D_(O*LFXbE5E56MuQ< zK7~)@{>|6(l{GAyZf-kER#KcJnws=fw;l? z@fx7_O{abzYG%i}%-G4-yw{6l8t$%|elKd}Z48or07-wl9H4Gk;N^C`SGd|hbx(od z+zX;yQN%)Ruu`MH!ivChh#}LPyt!*-274O{Z^P_zc*t93P%V+v5{u*rLuR=go#9O( zs03GwawO}Ig8pc`923&d4%Wn*Cv#Afv38jYP~TjQaa4_z7@&}dN112H@h(RSO|e5? z$$X!`f-V~;u-A$3ns1i{OBJHATp`x&jH=zE!PcsHwRUSpo;8ra;Owj^jyj? zIhY!c@*YFp>rxk5^Sz~pE`P!n?t_J?c4-S)n5Hb83tHe&PB-KXwe4p{ZhL)f-P{dS zJFvR@bbG2iB`)6rz5>Srh{qN;etua_OLyS2{526*;MVM0y#Gw1AI8QQ`f+SLsJQWG zQ_nXzrk>Y|Z85Q_aj8wgS|V@wgVQwBsoJz~-ydn9`K@rcfv&Wdn%xmcsy07M!&Q6q mW03=Ojps~1!>Ta#?g?hO#c zG2j5AqC8umh=72!+KPZRq9j@Yu>%OGQ&c)-%0oM&6cLr8)qcN|{npKrO>Tz%+0Xer z_nz~;zu!3fnm(~@fORyxZ)C1u_UPnEOM6UeHnu8pyz zsAZ@?477H}?CHfnkFm_NJeXeU3Ap{~6RhcrW~OIa)60B*!|$vr@%c+Vn7+qZ5%AD_ zKzCMq=sjah=Fm**L)KfNq`EkFRZV4eZI;{T_hePpR2W{T-|wtVD)D>VzN&!XuW=hm z($XZAR8`uvDr0Pxr82fC#wFvH%Xqu&kYS6l#V(ZzLBUd^w%U^<6CJUX62H$^DZ4v5(vs>#pd?53 zaHIt#OMW_f%#=~1OGYWm8<+K~>T=oBkg0arOHaM`Fd?8NABUi+TIpo+{tjG_wL`~X5 zh!4KlubEoHKI_3J@_GQ1CJ1q&9`FYnW{LelkMY9&w9*{vg!gh{otSMymg|Z7S&6X@ z43rXc?#P>U=rM#(#EL#xgLCX7TAd(^7-1JRfjndpmz5Z}KpgOe5pp2Z1@DkWJrScB zVqC)x@-a({>+qb-=)wnTf_&OT$cOgO)Gu}(c5pz~zt}S{34kwNg@@?f5uE#dv1A6?yIrxGu>oazv>0ll?z&>QH#Abi*QAfms-!m|Ln<2K3{sG@+VuW4J4>&bb zN9cf{$G=I47qu}%TzACCHq7YpUO--8#EP7)#Ed=E3i*fUj=r-jJ(?!i=JSO|V9iPl z`|t(Zk(kFC`VJjl3y+7UTO`Qf>~1f?2Vy}D*f*ZrBK~(3`hc^>Z12%$$g)0sn$cx^ z;ol3W8)}5OVGp&AM!*Q{BTulE82R!3Lcd$`kND67=<~kC83UjE8%d8h5#r)?*fvXC zd)P-kxh`$(3&=9Q+Vi`@eF*=1i_b!=Ar6iyJO-{k@@$Vk)DiWE9-htWqLy5fmik1? zEfVSiKd23!!SIb-@Enb}?!Xinc(W4oIe=Kg>p`!IBF{1_%|aithyk&;q{lTzY;&jpM`XM5DNR)YBA(k`Yd!TuDP?no&BZQwsmX47gwsae_g(;y|iak!QwiL#!^o` znJoDElS^gdgZJN8YZot+s(I+WbM&a1BRipI=&XF=iIAN)-S1X&}-ngN?T3@f7K6Sr(-|(A0;^6D~;?&9Fpl^!>xlkQxy-vJ+3wk&6~BC_vff*taDuv8}zSz84$Ft?c2Lo#j<=^f&jmJ;)J$l zb$79TU8-PN#PHgI3F64%DFQWt{tM3!SMrPN>Lz~t(MKB9N&E1OLp%BIO!4?*NlFGe zz;|OqgSL8AN&))#)ir}^y<2I@alN+%$6-w1H01Mw|5NnX216D>kCxQ zz&qij;D_(O*LFXbE5E56MuQ< zK7~)@{>|6(l{GAyZf-kER#KcJnws=fw;l? z@fx7_O{abzYG%i}%-G4-yw{6l8t$%|elKd}Z48or07-wl9H4Gk;N^C`SGd|hbx(od z+zX;yQN%)Ruu`MH!ivChh#}LPyt!*-274O{Z^P_zc*t93P%V+v5{u*rLuR=go#9O( zs03GwawO}Ig8pc`923&d4%Wn*Cv#Afv38jYP~TjQaa4_z7@&}dN112H@h(RSO|e5? z$$X!`f-V~;u-A$3ns1i{OBJHATp`x&jH=zE!PcsHwRUSpo;8ra;Owj^jyj? zIhY!c@*YFp>rxk5^Sz~pE`P!n?t_J?c4-S)n5Hb83tHe&PB-KXwe4p{ZhL)f-P{dS zJFvR@bbG2iB`)6rz5>Srh{qN;etua_OLyS2{526*;MVM0y#Gw1AI8QQ`f+SLsJQWG zQ_nXzrk>Y|Z85Q_aj8wgS|V@wgVQwBsoJz~-ydn9`K@rcfv&Wdn%xmcsy07M!&Q6q mW03=Ojps Date: Fri, 16 Dec 2022 12:42:45 -0800 Subject: [PATCH 08/24] Upgraded Gdown to fix data downloading. (#1140) upgraded gdown to fix some data downloads. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fe908f8c6..1127b0f7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "appdirs>=1.4", "av==9.2.0", "tyro>=0.3.31", - "gdown==4.5.1", + "gdown==4.6.0", "ninja==1.10.2.3", "functorch==0.2.1", "h5py>=2.9.0", From c338de4aa4b0509e3468508c88d7e5981990edc1 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 18 Dec 2022 10:46:39 -0800 Subject: [PATCH 09/24] Support non-pinhole rendering (#1141) --- nerfstudio/cameras/camera_paths.py | 30 ++++- nerfstudio/cameras/cameras.py | 2 +- nerfstudio/viewer/app/package.json | 2 +- .../SidePanel/CameraPanel/CameraPanel.jsx | 115 +++++++++++------- .../SidePanel/CameraPanel/CameraPropPanel.jsx | 22 ++++ .../src/modules/ViewerWindow/ViewerWindow.jsx | 91 +++++++------- nerfstudio/viewer/app/src/reducer.js | 1 + nerfstudio/viewer/server/utils.py | 15 ++- nerfstudio/viewer/server/viewer_utils.py | 13 +- 9 files changed, 191 insertions(+), 100 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index fb243917d..f5eb67501 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -23,7 +23,7 @@ import torch import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.camera_utils import get_interpolated_poses_many -from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length @@ -121,6 +121,19 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: image_height = camera_path["render_height"] image_width = camera_path["render_width"] + if "camera_type" in camera_path: + camera_type = camera_path["camera_type"] + else: + camera_type = "perspective" + + if "camera_type" not in camera_path: + camera_type = CameraType.PERSPECTIVE + elif camera_path["camera_type"] == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_path["camera_type"] == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE c2ws = [] fxs = [] @@ -129,11 +142,15 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: # pose c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3] c2ws.append(c2w) - # field of view - fov = camera["fov"] - focal_length = three_js_perspective_camera_focal_length(fov, image_height) - fxs.append(focal_length) - fys.append(focal_length) + if camera_type == CameraType.EQUIRECTANGULAR: + fxs.append(image_width / 2) + fys.append(image_height) + else: + # field of view + fov = camera["fov"] + focal_length = three_js_perspective_camera_focal_length(fov, image_height) + fxs.append(focal_length) + fys.append(focal_length) camera_to_worlds = torch.stack(c2ws, dim=0) fx = torch.tensor(fxs) @@ -144,4 +161,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: cx=image_width / 2, cy=image_height / 2, camera_to_worlds=camera_to_worlds, + camera_type=camera_type, ) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 0299fa4bb..eb10f414b 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -634,7 +634,7 @@ class Cameras(TensorDataclass): directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0] * sin_theta / theta, mask).float() directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1] * sin_theta / theta, mask).float() - directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask) + directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float() if CameraType.EQUIRECTANGULAR.value in cam_types: mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index b6183729e..78d4997f3 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "22-12-02-0", + "version": "22-12-16-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 9958ea7b1..4efab1b6f 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -112,12 +112,12 @@ function FovSelector(props) { camera.setFocalLength(val / camera.aspect); } } else if (fovLabel === FOV_LABELS.FOV) { + camera.fov = val; setGlobalFov(val); } else { - const old_fov = camera.fov; camera.setFocalLength(val / camera.aspect); const new_fov = camera.getEffectiveFOV(); - camera.fov = old_fov; + camera.fov = new_fov; setGlobalFov(new_fov); } @@ -216,6 +216,8 @@ function CameraList(props) { const [slider_value, set_slider_value] = React.useState(0); const [expanded, setExpanded] = React.useState(null); + const camera_type = useSelector((state) => state.renderingState.camera_type); + const handleChange = (cameraUUID: string) => (event: React.SyntheticEvent, isExpanded: boolean) => { @@ -399,7 +401,7 @@ function CameraList(props) { - {isAnimated('FOV') && ( + {isAnimated('FOV') && camera_type !== 'equirectangular' && ( state.renderingState.render_width, ); + const camera_type = useSelector((state) => state.renderingState.camera_type); const setRenderHeight = (value) => { dispatch({ @@ -501,6 +504,22 @@ export default function CameraPanel(props) { }); }; + const setCameraType = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/camera_type', + data: value, + }); + }; + + const setFieldOfView = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/field_of_view', + data: value, + }); + }; + // ui state const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); @@ -516,7 +535,7 @@ export default function CameraPanel(props) { // set slider and render camera back to 0 if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); - camera_render.fov = new_camera_list[0].fov; + setFieldOfView(new_camera_list[0].fov); set_slider_value(slider_min); } }; @@ -718,7 +737,7 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); } else { sceneTree.delete(['Camera Path', 'Curve']); } @@ -768,7 +787,7 @@ export default function CameraPanel(props) { fov = curve_object.curve_fovs.getPoint(point).z; const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); setGlobalFov(fov); } }, [slider_value, render_height, render_width]); @@ -828,6 +847,7 @@ export default function CameraPanel(props) { // const myData const camera_path_object = { keyframes, + camera_type, render_height, render_width, camera_path, @@ -871,6 +891,7 @@ export default function CameraPanel(props) { setRenderHeight(camera_path_object.render_height); setRenderWidth(camera_path_object.render_width); + setCameraType(camera_path_object.camera_type); setFps(camera_path_object.fps); setSeconds(camera_path_object.seconds); @@ -1024,52 +1045,54 @@ export default function CameraPanel(props) { />
-
- - - - -
+ sx={{ + mt: 1, + }} + > + + + + +
+ )}