From 85ef567682f83cb52f91f4e0f2f26780c38817c7 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Thu, 7 Sep 2023 22:10:00 -0700 Subject: [PATCH 01/19] Flesh out viewer beta (#2086) * Increase max_res of first proposal network * fix render statemachine logic for viewer beta * clicking on cameras in the viewer * fix fov calculation, update camera positions while training * cleanup * backup * fix configure theme * Fix theme configure for latest dev version of viser * Fix runtime errors * add set up direction button * add pause resume button * Export panel * Render tab skeleton * make responsiveness more snappy by stopping training * depth composite * start implementing viewercontrol in new viewer * refactor background img to use depth and add depth layering option * show/hide images * speed up rendering * fix crop color background * crop * rotated crops * rotated crops * transform handle on crop box * fix xyzw for crop * oriented crop export for pointclouds, not for meshes yet * Initial render panel * Fix export button * Theme tweaks * fix background issue in cropping * export rotated poisson meshe * double normals definition in export panel * Visual nits * remove tsdf from export panel * Add licenses * Fix some CI errors * Type fixes * Fix tests * Fix numpy broken * various bugfix --------- Co-authored-by: Brent Yi Co-authored-by: terrancewang --- .vscode/settings.json | 4 +- nerfstudio/cameras/cameras.py | 19 +- nerfstudio/data/scene_box.py | 38 +- nerfstudio/exporter/exporter_utils.py | 24 +- nerfstudio/model_components/renderers.py | 1 + nerfstudio/scripts/exporter.py | 28 +- nerfstudio/utils/math.py | 40 +- nerfstudio/viewer_beta/control_panel.py | 161 +++-- nerfstudio/viewer_beta/export_panel.py | 153 +++++ nerfstudio/viewer_beta/render_panel.py | 568 ++++++++++++++++++ .../viewer_beta/render_state_machine.py | 95 +-- nerfstudio/viewer_beta/utils.py | 60 +- nerfstudio/viewer_beta/viewer.py | 200 ++++-- nerfstudio/viewer_beta/viewer_elements.py | 129 +++- pyproject.toml | 3 +- tests/utils/test_visualization.py | 6 + 16 files changed, 1367 insertions(+), 162 deletions(-) create mode 100644 nerfstudio/viewer_beta/export_panel.py create mode 100644 nerfstudio/viewer_beta/render_panel.py diff --git a/.vscode/settings.json b/.vscode/settings.json index a02b23dd9..4c3a7447a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,7 +24,7 @@ "typescript.suggestionActions.enabled": false, "javascript.suggestionActions.enabled": false, "[python]": { - "editor.defaultFormatter": "ms-python.python", + "editor.defaultFormatter": "ms-python.black-formatter", "editor.codeActionsOnSave": { "source.organizeImports": true, "source.fixAll": true @@ -32,7 +32,7 @@ }, "editor.formatOnSave": true, "python.envFile": "${workspaceFolder}/.env", - "python.formatting.provider": "black", + "python.formatting.provider": "none", "python.formatting.blackArgs": ["--line-length=120"], "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": false, diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 8ad46e223..c2ef887a6 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -32,7 +32,7 @@ import nerfstudio.utils.math import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.rays import RayBundle -from nerfstudio.data.scene_box import SceneBox +from nerfstudio.data.scene_box import SceneBox, OrientedBox from nerfstudio.utils.tensor_dataclass import TensorDataclass TORCH_DEVICE = Union[torch.device, str] @@ -323,6 +323,7 @@ class Cameras(TensorDataclass): keep_shape: Optional[bool] = None, disable_distortion: bool = False, aabb_box: Optional[SceneBox] = None, + obb_box: Optional[OrientedBox] = None, ) -> RayBundle: """Generates rays for the given camera indices. @@ -465,20 +466,24 @@ class Cameras(TensorDataclass): if keep_shape is False: raybundle = raybundle.flatten() - if aabb_box: + if aabb_box is not None or obb_box is not None: with torch.no_grad(): - tensor_aabb = Parameter(aabb_box.aabb.flatten(), requires_grad=False) - rays_o = raybundle.origins.contiguous() rays_d = raybundle.directions.contiguous() - tensor_aabb = tensor_aabb.to(rays_o.device) shape = rays_o.shape rays_o = rays_o.reshape((-1, 3)) rays_d = rays_d.reshape((-1, 3)) - t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) + if aabb_box is not None: + tensor_aabb = Parameter(aabb_box.aabb.flatten(), requires_grad=False) + tensor_aabb = tensor_aabb.to(rays_o.device) + t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) + elif obb_box is not None: + t_min, t_max = nerfstudio.utils.math.intersect_obb(rays_o, rays_d, obb_box) + else: + assert False t_min = t_min.reshape([shape[0], shape[1], 1]) t_max = t_max.reshape([shape[0], shape[1], 1]) @@ -747,8 +752,6 @@ class Cameras(TensorDataclass): rotation = c2w[..., :3, :3] - -torch.pi * ((x - cx) / fx)[0] - # interocular axis of the VR180 camera vr180_x_axis = torch.tensor([1, 0, 0], device=c2w.device) diff --git a/nerfstudio/data/scene_box.py b/nerfstudio/data/scene_box.py index aaa055645..835c2b1cf 100644 --- a/nerfstudio/data/scene_box.py +++ b/nerfstudio/data/scene_box.py @@ -17,7 +17,8 @@ Dataset input structures. """ from dataclasses import dataclass -from typing import Union +from typing import Union, Tuple +import viser.transforms as vtf import torch from jaxtyping import Float @@ -76,3 +77,38 @@ class SceneBox: xyzs = poses[..., :3, -1] aabb = torch.stack([torch.min(xyzs, dim=0)[0], torch.max(xyzs, dim=0)[0]]) return SceneBox(aabb=aabb * scale_factor) + + +@dataclass +class OrientedBox: + R: Float[Tensor, "3 3"] + """R: rotation matrix.""" + T: Float[Tensor, "3"] + """T: translation vector.""" + S: Float[Tensor, "3"] + """S: scale vector.""" + + def within(self, pts: Float[Tensor, "n 3"]): + """Returns a boolean mask indicating whether each point is within the box.""" + R, T, S = self.R, self.T, self.S.to(pts) + H = torch.eye(4, device=pts.device, dtype=pts.dtype) + H[:3, :3] = R + H[:3, 3] = T + H_world2bbox = torch.inverse(H) + pts = torch.cat((pts, torch.ones_like(pts[..., :1])), dim=-1) + pts = torch.matmul(H_world2bbox, pts.T).T[..., :3] + + comp_l = torch.tensor(-S / 2) + comp_m = torch.tensor(S / 2) + mask = torch.all(torch.concat([pts > comp_l, pts < comp_m], dim=-1), dim=-1) + return mask + + @staticmethod + def from_params( + pos: Tuple[float, float, float], rpy: Tuple[float, float, float], scale: Tuple[float, float, float] + ): + """Construct a box from position, rotation, and scale parameters.""" + R = torch.tensor(vtf.SO3.from_rpy_radians(rpy[0], rpy[1], rpy[2]).as_matrix()) + T = torch.tensor(pos) + S = torch.tensor(scale) + return OrientedBox(R=R, T=T, S=S) diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py index 1baeeb2b5..66d658690 100644 --- a/nerfstudio/exporter/exporter_utils.py +++ b/nerfstudio/exporter/exporter_utils.py @@ -33,6 +33,7 @@ from torch import Tensor from nerfstudio.cameras.cameras import Cameras from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.data.scene_box import OrientedBox from nerfstudio.pipelines.base_pipeline import Pipeline, VanillaPipeline from nerfstudio.utils.rich_utils import CONSOLE, ItersPerSecColumn @@ -84,8 +85,9 @@ def generate_point_cloud( depth_output_name: str = "depth", normal_output_name: Optional[str] = None, use_bounding_box: bool = True, - bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0), - bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0), + bounding_box_min: Optional[Tuple[float, float, float]] = None, + bounding_box_max: Optional[Tuple[float, float, float]] = None, + crop_obb: Optional[OrientedBox] = None, std_ratio: float = 10.0, ) -> o3d.geometry.PointCloud: """Generate a point cloud from a nerf. @@ -119,7 +121,8 @@ def generate_point_cloud( rgbs = [] normals = [] view_directions = [] - + if use_bounding_box and (crop_obb is not None and bounding_box_max is not None): + CONSOLE.print("Provided aabb and crop_obb at the same time, using only the obb", style="bold yellow") with progress as progress_bar: task = progress_bar.add_task("Generating Point Cloud", total=num_points) while not progress_bar.finished: @@ -163,12 +166,15 @@ def generate_point_cloud( normal = normal[mask] if use_bounding_box: - comp_l = torch.tensor(bounding_box_min, device=point.device) - comp_m = torch.tensor(bounding_box_max, device=point.device) - assert torch.all( - comp_l < comp_m - ), f"Bounding box min {bounding_box_min} must be smaller than max {bounding_box_max}" - mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1) + if crop_obb is None: + comp_l = torch.tensor(bounding_box_min, device=point.device) + comp_m = torch.tensor(bounding_box_max, device=point.device) + assert torch.all( + comp_l < comp_m + ), f"Bounding box min {bounding_box_min} must be smaller than max {bounding_box_max}" + mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1) + else: + mask = crop_obb.within(point) point = point[mask] rgb = rgb[mask] view_direction = view_direction[mask] diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 8bc5b4d78..20a506b1e 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -219,6 +219,7 @@ class RGBRenderer(nn.Module): Returns: Outputs of rgb values. """ + if background_color is None: background_color = self.background_color diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index d5ccd82bb..2e4fc68ca 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -34,6 +34,7 @@ from typing_extensions import Annotated, Literal from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager +from nerfstudio.data.scene_box import OrientedBox from nerfstudio.exporter import texture_utils, tsdf_utils from nerfstudio.exporter.exporter_utils import ( collect_camera_poses, @@ -109,10 +110,17 @@ class ExportPointCloud(Exporter): """Name of the RGB output.""" use_bounding_box: bool = True """Only query points within the bounding box""" - bounding_box_min: Tuple[float, float, float] = (-1, -1, -1) + bounding_box_min: Optional[Tuple[float, float, float]] = (-1, -1, -1) """Minimum of the bounding box, used if use_bounding_box is True.""" - bounding_box_max: Tuple[float, float, float] = (1, 1, 1) + bounding_box_max: Optional[Tuple[float, float, float]] = (1, 1, 1) """Maximum of the bounding box, used if use_bounding_box is True.""" + + obb_center: Optional[Tuple[float, float, float]] = None + """Center of the oriented bounding box.""" + obb_rotation: Optional[Tuple[float, float, float]] = None + """Rotation of the oriented bounding box. Expressed as RPY Euler angles in radians""" + obb_scale: Optional[Tuple[float, float, float]] = None + """Scale of the oriented bounding box along each axis.""" num_rays_per_batch: int = 32768 """Number of rays to evaluate per batch. Decrease if you run out of memory.""" std_ratio: float = 10.0 @@ -135,7 +143,9 @@ class ExportPointCloud(Exporter): # Whether the normals should be estimated based on the point cloud. estimate_normals = self.normal_method == "open3d" - + crop_obb = None + if self.obb_center is not None and self.obb_rotation is not None and self.obb_scale is not None: + crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale) pcd = generate_point_cloud( pipeline=pipeline, num_points=self.num_points, @@ -148,6 +158,7 @@ class ExportPointCloud(Exporter): use_bounding_box=self.use_bounding_box, bounding_box_min=self.bounding_box_min, bounding_box_max=self.bounding_box_max, + crop_obb=crop_obb, std_ratio=self.std_ratio, ) torch.cuda.empty_cache() @@ -264,6 +275,12 @@ class ExportPoissonMesh(Exporter): """Minimum of the bounding box, used if use_bounding_box is True.""" bounding_box_max: Tuple[float, float, float] = (1, 1, 1) """Minimum of the bounding box, used if use_bounding_box is True.""" + obb_center: Optional[Tuple[float, float, float]] = None + """Center of the oriented bounding box.""" + obb_rotation: Optional[Tuple[float, float, float]] = None + """Rotation of the oriented bounding box. Expressed as RPY Euler angles in radians""" + obb_scale: Optional[Tuple[float, float, float]] = None + """Scale of the oriented bounding box along each axis.""" num_rays_per_batch: int = 32768 """Number of rays to evaluate per batch. Decrease if you run out of memory.""" texture_method: Literal["point_cloud", "nerf"] = "nerf" @@ -296,6 +313,10 @@ class ExportPoissonMesh(Exporter): # Whether the normals should be estimated based on the point cloud. estimate_normals = self.normal_method == "open3d" + if self.obb_center is not None and self.obb_rotation is not None and self.obb_scale is not None: + crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale) + else: + crop_obb = None pcd = generate_point_cloud( pipeline=pipeline, @@ -309,6 +330,7 @@ class ExportPoissonMesh(Exporter): use_bounding_box=self.use_bounding_box, bounding_box_min=self.bounding_box_min, bounding_box_max=self.bounding_box_max, + crop_obb=crop_obb, std_ratio=self.std_ratio, ) torch.cuda.empty_cache() diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 788a0520d..8567a8b28 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -21,7 +21,7 @@ import torch from jaxtyping import Bool, Float from torch import Tensor -from nerfstudio.utils.misc import torch_compile +from nerfstudio.data.scene_box import OrientedBox def components_from_spherical_harmonics( @@ -199,7 +199,7 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor: return torch.exp(-0.5 * x_vars) * torch.sin(x_means) -@torch_compile(dynamic=True, mode="reduce-overhead", backend="eager") +# @torch_compile(dynamic=True, mode="reduce-overhead", backend="eager") def intersect_aabb( origins: torch.Tensor, directions: torch.Tensor, @@ -240,6 +240,42 @@ def intersect_aabb( return t_min, t_max +def intersect_obb( + origins: torch.Tensor, + directions: torch.Tensor, + obb: OrientedBox, + max_bound: float = 1e10, + invalid_value: float = 1e10, +): + """ + Ray intersection with an oriented bounding box (OBB) + + Args: + origins: [N,3] tensor of 3d positions + directions: [N,3] tensor of normalized directions + R: [3,3] rotation matrix + T: [3] translation vector + S: [3] extents of the bounding box + max_bound: Maximum value of t_max + invalid_value: Value to return in case of no intersection + """ + # Transform ray to OBB space + R, T, S = obb.R, obb.T, obb.S.to(origins.device) + H = torch.eye(4, device=origins.device, dtype=origins.dtype) + H[:3, :3] = R + H[:3, 3] = T + H_world2bbox = torch.inverse(H) + origins = torch.cat((origins, torch.ones_like(origins[..., :1])), dim=-1) + origins = torch.matmul(H_world2bbox, origins.T).T[..., :3] + directions = torch.matmul(H_world2bbox[:3, :3], directions.T).T + + # Compute intersection with axis-aligned bounding box with min as -S and max as +S + aabb = torch.concat((-S / 2, S / 2)) + t_min, t_max = intersect_aabb(origins, directions, aabb, max_bound=max_bound, invalid_value=invalid_value) + + return t_min, t_max + + def safe_normalize( vectors: Float[Tensor, "*batch_dim N"], eps: float = 1e-10, diff --git a/nerfstudio/viewer_beta/control_panel.py b/nerfstudio/viewer_beta/control_panel.py index a56d4bb74..07f556dd4 100644 --- a/nerfstudio/viewer_beta/control_panel.py +++ b/nerfstudio/viewer_beta/control_panel.py @@ -16,11 +16,13 @@ from collections import defaultdict from typing import Callable, DefaultDict, List, Tuple, get_args +import numpy as np import torch -from viser import ViserServer - +import viser.transforms as vtf +from nerfstudio.data.scene_box import OrientedBox from nerfstudio.utils.colormaps import ColormapOptions, Colormaps from nerfstudio.viewer_beta.viewer_elements import ( # ViewerButtonGroup, + ViewerButton, ViewerButtonGroup, ViewerCheckbox, ViewerDropdown, @@ -30,6 +32,7 @@ from nerfstudio.viewer_beta.viewer_elements import ( # ViewerButtonGroup, ViewerSlider, ViewerVec3, ) +from viser import ViserServer class ControlPanel: @@ -47,21 +50,29 @@ class ControlPanel: self, viser_server: ViserServer, time_enabled: bool, + scale_ratio: float, rerender_cb: Callable, crop_update_cb: Callable, update_output_cb: Callable, update_split_output_cb: Callable, + toggle_training_state_cb: Callable, + camera_vis: Callable, ): + self.viser_scale_ratio = scale_ratio # elements holds a mapping from tag: [elements] self.viser_server = viser_server self._elements_by_tag: DefaultDict[str, List[ViewerElement]] = defaultdict(lambda: []) self._train_speed = ViewerButtonGroup( - name="Train Speed ", - default_value="Balanced", - options=["Slow", "Balanced", "Fast"], + name="Train Speed", + default_value="Mid", + options=["Slow", "Mid", "Fast"], cb_hook=lambda han: self._train_speed_cb(), ) + self._reset_camera = ViewerButton( + name="Reset Up Direction", + cb_hook=lambda han: self._reset_camera_cb(), + ) self._output_render = ViewerDropdown( "Output Render", "not set", @@ -118,6 +129,9 @@ class ControlPanel: step=0.05, hint="Target training utilization, 0.0 is slow, 1.0 is fast. Doesn't affect final render quality", ) + self._layer_depth = ViewerCheckbox( + "Composite Depth", True, cb_hook=rerender_cb, hint="Allow NeRF to occlude 3D browser objects" + ) self._max_res = ViewerSlider( "Max Res", 512, 64, 2048, 100, cb_hook=rerender_cb, hint="Maximum resolution to render in viewport" ) @@ -130,27 +144,76 @@ class ControlPanel: self._background_color = ViewerRGB( "Background color", (38, 42, 55), cb_hook=crop_update_cb, hint="Color of the background" ) - self._crop_min = ViewerVec3( - "Crop Min", (-1, -1, -1), 0.05, cb_hook=crop_update_cb, hint="Minimum value of the crop" + self._crop_handle = self.viser_server.add_transform_controls("Crop", depth_test=False, line_width=4.0) + + def update_center(han): + self._crop_handle.position = tuple(p * self.viser_scale_ratio for p in han.value) + + self._crop_center = ViewerVec3( + "Crop Center", + (0.0, 0.0, 0.0), + step=0.01, + cb_hook=lambda e: [crop_update_cb(e), update_center(e)], + hint="Center of the crop box", ) - self._crop_max = ViewerVec3( - "Crop Max", (1, 1, 1), 0.05, cb_hook=crop_update_cb, hint="Maximum value of the crop" + + def update_rot(han): + self._crop_handle.wxyz = vtf.SO3.from_rpy_radians(*han.value).wxyz + + self._crop_rot = ViewerVec3( + "Crop Rotation", + (0.0, 0.0, 0.0), + step=0.01, + cb_hook=lambda e: [crop_update_cb(e), update_rot(e)], + hint="Rotation of the crop box", ) + + self._crop_scale = ViewerVec3( + "Crop Scale", (1.0, 1.0, 1.0), step=0.01, cb_hook=crop_update_cb, hint="Scale of the crop box" + ) + + @self._crop_handle.on_update + def _update_crop_handle(han): + pos = self._crop_handle.position + self._crop_center.value = tuple(p / self.viser_scale_ratio for p in pos) + rpy = vtf.SO3(self._crop_handle.wxyz).as_rpy_radians() + self._crop_rot.value = (float(rpy.roll), float(rpy.pitch), float(rpy.yaw)) + self._time = ViewerSlider("Time", 0.0, 0.0, 1.0, 0.01, cb_hook=rerender_cb, hint="Time to render") self._time_enabled = time_enabled + self.stat_folder = self.viser_server.add_gui_folder("Stats") + with self.stat_folder: + self.markdown = self.viser_server.add_gui_markdown("Step: 0") + self.pause_train = viser_server.add_gui_button(label="Pause Training", disabled=False) + self.pause_train.on_click(lambda _: self.toggle_pause_button()) + self.pause_train.on_click(lambda han: toggle_training_state_cb(han)) + self.resume_train = viser_server.add_gui_button(label="Resume Training", disabled=False) + self.resume_train.on_click(lambda _: self.toggle_pause_button()) + self.resume_train.on_click(lambda han: toggle_training_state_cb(han)) + self.resume_train.visible = False + # Add buttons to toggle training image visibility + self.hide_images = viser_server.add_gui_button(label="Hide Train Cams", disabled=False) + self.hide_images.on_click(lambda _: camera_vis(False)) + self.hide_images.on_click(lambda _: self.toggle_cameravis_button()) + self.show_images = viser_server.add_gui_button(label="Show Train Cams", disabled=False) + self.show_images.on_click(lambda _: camera_vis(True)) + self.show_images.on_click(lambda _: self.toggle_cameravis_button()) + self.show_images.visible = False + self.add_element(self._train_speed) self.add_element(self._train_util) + self.add_element(self._reset_camera) with self.viser_server.add_gui_folder("Render Options"): self.add_element(self._max_res) self.add_element(self._output_render) self.add_element(self._colormap) + self.add_element(self._layer_depth) # colormap options - with self.viser_server.add_gui_folder(" "): - self.add_element(self._invert, additional_tags=("colormap",)) - self.add_element(self._normalize, additional_tags=("colormap",)) - self.add_element(self._min, additional_tags=("colormap",)) - self.add_element(self._max, additional_tags=("colormap",)) + self.add_element(self._invert, additional_tags=("colormap",)) + self.add_element(self._normalize, additional_tags=("colormap",)) + self.add_element(self._min, additional_tags=("colormap",)) + self.add_element(self._max, additional_tags=("colormap",)) # split options with self.viser_server.add_gui_folder("Split Screen"): @@ -159,19 +222,19 @@ class ControlPanel: self.add_element(self._split_percentage, additional_tags=("split",)) self.add_element(self._split_output_render, additional_tags=("split",)) self.add_element(self._split_colormap, additional_tags=("split",)) - with self.viser_server.add_gui_folder(" "): - self.add_element(self._split_invert, additional_tags=("split_colormap",)) - self.add_element(self._split_normalize, additional_tags=("split_colormap",)) - self.add_element(self._split_min, additional_tags=("split_colormap",)) - self.add_element(self._split_max, additional_tags=("split_colormap",)) + + self.add_element(self._split_invert, additional_tags=("split_colormap",)) + self.add_element(self._split_normalize, additional_tags=("split_colormap",)) + self.add_element(self._split_min, additional_tags=("split_colormap",)) + self.add_element(self._split_max, additional_tags=("split_colormap",)) with self.viser_server.add_gui_folder("Crop Viewport"): self.add_element(self._crop_viewport) - # Crop options self.add_element(self._background_color, additional_tags=("crop",)) - self.add_element(self._crop_min, additional_tags=("crop",)) - self.add_element(self._crop_max, additional_tags=("crop",)) + self.add_element(self._crop_center, additional_tags=("crop",)) + self.add_element(self._crop_scale, additional_tags=("crop",)) + self.add_element(self._crop_rot, additional_tags=("crop",)) self.add_element(self._time, additional_tags=("time",)) @@ -182,13 +245,36 @@ class ControlPanel: if self._train_speed.value == "Fast": self._train_util.value = 0.95 self._max_res.value = 256 - elif self._train_speed.value == "Balanced": + elif self._train_speed.value == "Mid": self._train_util.value = 0.85 self._max_res.value = 512 elif self._train_speed.value == "Slow": self._train_util.value = 0.5 self._max_res.value = 1024 + def _reset_camera_cb(self) -> None: + for client in self.viser_server.get_clients().values(): + client.camera.up_direction = vtf.SO3(client.camera.wxyz) @ np.array([0.0, -1.0, 0.0]) + + def toggle_pause_button(self) -> None: + self.pause_train.visible = not self.pause_train.visible + self.resume_train.visible = not self.resume_train.visible + + def toggle_cameravis_button(self) -> None: + self.hide_images.visible = not self.hide_images.visible + self.show_images.visible = not self.show_images.visible + + def update_step(self, step): + """ + Args: + step: the train step to set the model to + """ + with self.viser_server.atomic(), self.stat_folder: + # TODO change to a .value call instead of remove() and add, this makes it jittery + with self.viser_server.atomic(): + self.markdown.remove() + self.markdown = self.viser_server.add_gui_markdown(f"Step: {step}") + def update_output_options(self, new_options: List[str]): """ Args: @@ -226,6 +312,7 @@ class ControlPanel: self._split_output_render.set_hidden(not self._split.value) self._split_colormap.set_hidden(not self._split.value) self._split_colormap.set_disabled(self.split_output_render == "rgb") + self._crop_handle.visible = self.crop_viewport def update_colormap_options(self, dimensions: int, dtype: type) -> None: """update the colormap options based on the current render @@ -286,24 +373,12 @@ class ControlPanel: self._crop_viewport.value = value @property - def crop_min(self) -> Tuple[float, float, float]: - """Returns the current crop min setting""" - return self._crop_min.value - - @crop_min.setter - def crop_min(self, value: Tuple[float, float, float]): - """Sets the crop min setting""" - self._crop_min.value = value - - @property - def crop_max(self) -> Tuple[float, float, float]: - """Returns the current crop max setting""" - return self._crop_max.value - - @crop_max.setter - def crop_max(self, value: Tuple[float, float, float]): - """Sets the crop max setting""" - self._crop_max.value = value + def crop_obb(self): + """Returns the current crop obb setting""" + rxyz = self._crop_rot.value + R = torch.tensor(vtf.SO3.from_rpy_radians(rxyz[0], rxyz[1], rxyz[2]).as_matrix()) + obb = OrientedBox(R, torch.tensor(self._crop_center.value), torch.tensor(self._crop_scale.value)) + return obb @property def background_color(self) -> Tuple[int, int, int]: @@ -347,6 +422,10 @@ class ControlPanel: invert=self._split_invert.value, ) + @property + def layer_depth(self): + return self._layer_depth.value + def _get_colormap_options(dimensions: int, dtype: type) -> List[Colormaps]: """ diff --git a/nerfstudio/viewer_beta/export_panel.py b/nerfstudio/viewer_beta/export_panel.py new file mode 100644 index 000000000..425e10a19 --- /dev/null +++ b/nerfstudio/viewer_beta/export_panel.py @@ -0,0 +1,153 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path + +import viser +import viser.transforms as vtf +from nerfstudio.data.scene_box import OrientedBox +from nerfstudio.viewer_beta.control_panel import ControlPanel +from typing_extensions import Literal + + +def populate_export_tab(server: viser.ViserServer, control_panel: ControlPanel, config_path: Path) -> None: + crop_output = server.add_gui_checkbox("Use Crop", False) + + @crop_output.on_update + def _(_) -> None: + control_panel.crop_viewport = crop_output.value + + with server.add_gui_folder("Point Cloud"): + populate_point_cloud_tab(server, control_panel, config_path) + with server.add_gui_folder("Mesh"): + populate_mesh_tab(server, control_panel, config_path) + + +def show_command_modal(client: viser.ClientHandle, what: Literal["mesh", "point cloud"], command: str) -> None: + """Show a modal to each currently connected client. + + In the future, we should only show the modal to the client that pushes the + generation button. + """ + with client.add_gui_modal(what.title() + " Export") as modal: + client.add_gui_markdown( + "\n".join( + [ + f"To export a {what}, run the following from the command line:", + "", + "```", + command, + "```", + ] + ) + ) + close_button = client.add_gui_button("Close") + + @close_button.on_click + def _(_) -> None: + modal.close() + + +def get_crop_string(obb: OrientedBox): + """Takes in an oriented bounding box and returns a string of the form "--obb_{center,rotation,scale} + and each arg formatted with spaces around it + """ + rpy = vtf.SO3.from_matrix(obb.R.numpy(force=True)).as_rpy_radians() + pos = obb.T.squeeze().tolist() + scale = obb.S.squeeze().tolist() + rpystring = " ".join([str(x) for x in rpy]) + posstring = " ".join([str(x) for x in pos]) + scalestring = " ".join([str(x) for x in scale]) + return f"--obb_center {posstring} --obb_rotation {rpystring} --obb_scale {scalestring}" + + +def populate_point_cloud_tab( + server: viser.ViserServer, + control_panel: ControlPanel, + config_path: Path, +) -> None: + server.add_gui_markdown("Render depth, project to an oriented point cloud, and filter. ") + num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1) + remove_outliers = server.add_gui_checkbox("Remove outliers", True) + normals = server.add_gui_dropdown( + "Normals", + # TODO: options here could depend on what's available to the model. + ("open3d", "model_output"), + initial_value="open3d", + hint="Normal map source.", + ) + output_dir = server.add_gui_text("Output Directory", initial_value="exports/pcd/") + generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2) + + @generate_command.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client is not None + command = " ".join( + [ + "ns-export pointcloud", + f"--load-config {config_path}", + f"--output-dir {output_dir.value}", + f"--num-points {num_points.value}", + f"--remove-outliers {remove_outliers.value}", + f"--normal-method {normals.value}", + f"--use_bounding_box {control_panel.crop_viewport}", + get_crop_string(control_panel.crop_obb), + ] + ) + show_command_modal(event.client, "point cloud", command) + + +def populate_mesh_tab( + server: viser.ViserServer, + control_panel: ControlPanel, + config_path: Path, +) -> None: + server.add_gui_markdown( + "Render depth, project to an oriented point cloud, and run Poisson surface reconstruction." + ) + + normals = server.add_gui_dropdown( + "Normals", + ("open3d", "model_output"), + initial_value="open3d", + hint="Source for normal maps.", + ) + num_faces = server.add_gui_number("# Faces", initial_value=50_000, min=1) + texture_resolution = server.add_gui_number("Texture Resolution", min=8, initial_value=2048) + output_directory = server.add_gui_text("Output Directory", initial_value="exports/mesh/") + num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1) + remove_outliers = server.add_gui_checkbox("Remove outliers", True) + + generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2) + + @generate_command.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client is not None + command = " ".join( + [ + "ns-export poisson", + f"--load-config {config_path}", + f"--output-dir {output_directory.value}", + f"--target-num-faces {num_faces.value}", + f"--num-pixels-per-side {texture_resolution.value}", + f"--num-points {num_points.value}", + f"--remove-outliers {remove_outliers.value}", + f"--normal-method {normals.value}", + f"--use_bounding_box {control_panel.crop_viewport}", + get_crop_string(control_panel.crop_obb), + ] + ) + show_command_modal(event.client, "mesh", command) diff --git a/nerfstudio/viewer_beta/render_panel.py b/nerfstudio/viewer_beta/render_panel.py new file mode 100644 index 000000000..02d4d6ce0 --- /dev/null +++ b/nerfstudio/viewer_beta/render_panel.py @@ -0,0 +1,568 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import colorsys +import dataclasses +import threading +import time +from typing import Dict, List, Optional, Tuple + +import numpy as onp +import splines +import splines.quaternion +import viser +import viser.transforms as tf + + +@dataclasses.dataclass +class Keyframe: + position: onp.ndarray + wxyz: onp.ndarray + override_fov_enabled: bool + override_fov_value: float + aspect: float + + @staticmethod + def from_camera(camera: viser.CameraHandle, aspect: float) -> Keyframe: + return Keyframe( + camera.position, + camera.wxyz, + override_fov_enabled=False, + override_fov_value=camera.fov, + aspect=aspect, + ) + + +class CameraPath: + def __init__(self, server: viser.ViserServer): + self._server = server + self._keyframes: Dict[int, Tuple[Keyframe, viser.CameraFrustumHandle]] = {} + self._keyframe_counter: int = 0 + self._spline: Optional[viser.SceneNodeHandle] = None + self._camera_edit_panel: Optional[viser.Gui3dContainerHandle] = None + + self._orientation_spline: Optional[splines.quaternion.KochanekBartels] = None + self._position_spline: Optional[splines.KochanekBartels] = None + self._fov_spline: Optional[splines.KochanekBartels] = None + self._keyframes_visible: bool = True + + # These parameters should be overridden externally. + self.loop: bool = False + self.smoothness: float = 0.5 # Tension / alpha term. + self.default_fov: float = 0.0 + + def set_keyframes_visible(self, visible: bool) -> None: + self._keyframes_visible = visible + for keyframe in self._keyframes.values(): + keyframe[1].visible = visible + + def add_camera(self, keyframe: Keyframe, keyframe_index: Optional[int] = None) -> None: + """Add a new camera, or replace an old one if `keyframe_index` is passed in.""" + server = self._server + + # Add a keyframe if we aren't replacing an existing one. + if keyframe_index is None: + keyframe_index = self._keyframe_counter + self._keyframe_counter += 1 + + frustum_handle = server.add_camera_frustum( + f"/render_cameras/{keyframe_index}", + fov=keyframe.override_fov_value if keyframe.override_fov_enabled else self.default_fov, + aspect=keyframe.aspect, + scale=0.1, + color=(127, 127, 127), + wxyz=keyframe.wxyz, + position=keyframe.position, + visible=self._keyframes_visible, + ) + + @frustum_handle.on_click + def _(_) -> None: + with server.add_3d_gui_container( + "/camera_edit_panel", + wxyz=keyframe.wxyz, + position=keyframe.position, + ) as camera_edit_panel: + self._camera_edit_panel = camera_edit_panel + override_fov = server.add_gui_checkbox("Override FOV", initial_value=keyframe.override_fov_enabled) + override_fov_degrees = server.add_gui_slider( + "Override FOV (degrees)", + 5.0, + 175.0, + step=0.1, + initial_value=keyframe.override_fov_value * 180.0 / onp.pi, + disabled=not keyframe.override_fov_enabled, + ) + delete_button = server.add_gui_button("Delete", color="red", icon=viser.Icon.TRASH) + go_to_button = server.add_gui_button("Go to") + close_button = server.add_gui_button("Close") + + @override_fov.on_update + def _(_) -> None: + keyframe.override_fov_enabled = override_fov.value + override_fov_degrees.disabled = not override_fov.value + self.add_camera(keyframe, keyframe_index) + + @override_fov_degrees.on_update + def _(_) -> None: + keyframe.override_fov_value = override_fov_degrees.value / 180.0 * onp.pi + self.add_camera(keyframe, keyframe_index) + + @delete_button.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client is not None + with event.client.add_gui_modal("Confirm") as modal: + event.client.add_gui_markdown("Delete keyframe?") + confirm_button = event.client.add_gui_button("Yes", color="red", icon=viser.Icon.TRASH) + exit_button = event.client.add_gui_button("Cancel") + + @confirm_button.on_click + def _(_) -> None: + assert camera_edit_panel is not None + + keyframe_id = None + for i, keyframe_tuple in self._keyframes.items(): + if keyframe_tuple[1] is frustum_handle: + keyframe_id = i + break + assert keyframe_id is not None + + self._keyframes.pop(keyframe_id) + frustum_handle.remove() + camera_edit_panel.remove() + modal.close() + self.update_spline() + + @exit_button.on_click + def _(_) -> None: + modal.close() + + @go_to_button.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client is not None + client = event.client + T_world_current = tf.SE3.from_rotation_and_translation( + tf.SO3(client.camera.wxyz), client.camera.position + ) + T_world_target = tf.SE3.from_rotation_and_translation( + tf.SO3(keyframe.wxyz), keyframe.position + ) @ tf.SE3.from_translation(onp.array([0.0, 0.0, -0.5])) + + T_current_target = T_world_current.inverse() @ T_world_target + + for j in range(10): + T_world_set = T_world_current @ tf.SE3.exp(T_current_target.log() * j / 9.0) + + # Important bit: we atomically set both the orientation and the position + # of the camera. + with client.atomic(): + client.camera.wxyz = T_world_set.rotation().wxyz + client.camera.position = T_world_set.translation() + time.sleep(1.0 / 30.0) + + @close_button.on_click + def _(_) -> None: + assert camera_edit_panel is not None + camera_edit_panel.remove() + + self._keyframes[keyframe_index] = (keyframe, frustum_handle) + + def update_aspect(self, aspect: float) -> None: + for keyframe_index, frame in self._keyframes.items(): + frame = dataclasses.replace(frame[0], aspect=aspect) + self.add_camera(frame, keyframe_index=keyframe_index) + + def reset(self) -> None: + for frame in self._keyframes.values(): + frame[1].remove() + self._keyframes.clear() + self.update_spline() + + def interpolate_pose_and_fov(self, normalized_t: float) -> Optional[Tuple[tf.SE3, float]]: + if len(self._keyframes) < 2: + return None + # TODO: this doesn't need to be constantly re-instantiated. + self._fov_spline = splines.KochanekBartels( + [ + keyframe[0].override_fov_value if keyframe[0].override_fov_enabled else self.default_fov + for keyframe in self._keyframes.values() + ], + tcb=(self.smoothness, 0.0, 0.0), + endconditions="closed" if self.loop else "natural", + ) + + assert self._orientation_spline is not None + assert self._position_spline is not None + assert self._fov_spline is not None + max_t = len(self._keyframes) if self.loop else len(self._keyframes) - 1 + t = max_t * normalized_t + quat = self._orientation_spline.evaluate(t) + assert isinstance(quat, splines.quaternion.UnitQuaternion) + return ( + tf.SE3.from_rotation_and_translation( + tf.SO3(onp.array([quat.scalar, *quat.vector])), + self._position_spline.evaluate(t), + ), + float(self._fov_spline.evaluate(t)), + ) + + def update_spline(self) -> None: + keyframes = list(self._keyframes.values()) + if len(keyframes) <= 1: + if self._spline is not None: + self._spline.remove() + self._spline = None + return + + # Update internal splines. + self._orientation_spline = splines.quaternion.KochanekBartels( + [ + splines.quaternion.UnitQuaternion.from_unit_xyzw(onp.roll(keyframe[0].wxyz, shift=-1)) + for keyframe in keyframes + ], + tcb=(self.smoothness, 0.0, 0.0), + endconditions="closed" if self.loop else "natural", + ) + self._position_spline = splines.KochanekBartels( + [keyframe[0].position for keyframe in keyframes], + tcb=(self.smoothness, 0.0, 0.0), + endconditions="closed" if self.loop else "natural", + ) + + # Update visualized spline. + num_keyframes = len(keyframes) + 1 if self.loop else len(keyframes) + points_array = onp.array( + [self._position_spline.evaluate(t) for t in onp.linspace(0, num_keyframes - 1, num_keyframes * 100)] + ) + colors_array = onp.array([colorsys.hls_to_rgb(h, 0.5, 1.0) for h in onp.linspace(0.0, 1.0, len(points_array))]) + self._spline = self._server.add_point_cloud( + "/render_camera_spline", + points=points_array, + colors=colors_array, + point_size=0.035, + ) + + +def populate_render_tab(server: viser.ViserServer) -> None: + fov_degrees = server.add_gui_slider( + "FOV", + initial_value=90.0, + min=0.1, + max=175.0, + step=0.01, + hint="Field-of-view for rendering, which can also be overridden on a per-keyframe basis.", + ) + + @fov_degrees.on_update + def _(_) -> None: + fov_radians = fov_degrees.value / 180.0 * onp.pi + for client in server.get_clients().values(): + client.camera.fov = fov_radians + camera_path.default_fov = fov_radians + + # Updating the aspect ratio will also re-render the camera frustums. + # Could rethink this. + camera_path.update_aspect(resolution.value[0] / resolution.value[1]) + + resolution = server.add_gui_vector2( + "Resolution", + initial_value=(1920, 1080), + min=(50, 50), + max=(10_000, 10_000), + step=1, + hint="Tension parameter for adjusting smoothness of spline interpolation.", + ) + + @resolution.on_update + def _(_) -> None: + """Update the aspect ratio for all cameras when the resolution changes.""" + camera_path.update_aspect(resolution.value[0] / resolution.value[1]) + + add_button = server.add_gui_button( + "Add keyframe", + icon=viser.Icon.PLUS, + hint="Add a new keyframe at the current pose.", + ) + + @add_button.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client_id is not None + camera = server.get_clients()[event.client_id].camera + + # Add this camera to the path. + camera_path.add_camera( + Keyframe.from_camera(camera, aspect=resolution.value[0] / resolution.value[1]), + ) + camera_path.update_spline() + + reset_up_button = server.add_gui_button( + "Reset up direction", + icon=viser.Icon.ARROW_AUTOFIT_UP, + hint="Reset the orbit up direction.", + ) + + @reset_up_button.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client is not None + event.client.camera.up_direction = tf.SO3(event.client.camera.wxyz) @ onp.array([0.0, -1.0, 0.0]) + + clear_keyframes_button = server.add_gui_button( + "Clear keyframes", + icon=viser.Icon.TRASH, + hint="Remove all keyframes from the render path.", + ) + + @clear_keyframes_button.on_click + def _(event: viser.GuiEvent) -> None: + assert event.client_id is not None + client = server.get_clients()[event.client_id] + with client.add_gui_modal("Confirm") as modal: + client.add_gui_markdown("Clear all keyframes?") + confirm_button = client.add_gui_button("Yes", color="red", icon=viser.Icon.TRASH) + exit_button = client.add_gui_button("Cancel") + + @confirm_button.on_click + def _(_) -> None: + camera_path.reset() + modal.close() + + # Clear move handles. + if len(transform_controls) > 0: + for t in transform_controls: + t.remove() + transform_controls.clear() + return + + @exit_button.on_click + def _(_) -> None: + modal.close() + + loop = server.add_gui_checkbox("Loop", False) + + @loop.on_update + def _(_) -> None: + camera_path.loop = loop.value + camera_path.update_spline() + + smoothness = server.add_gui_slider( + "Spline Tension", + min=0.0, + max=1.0, + initial_value=0.0, + step=0.01, + hint="Tension parameter for adjusting smoothness of spline interpolation.", + ) + + @smoothness.on_update + def _(_) -> None: + camera_path.smoothness = smoothness.value + camera_path.update_spline() + + move_checkbox = server.add_gui_checkbox( + "Move keyframes", + initial_value=False, + hint="Toggle move handles for keyframes in the scene.", + ) + + @move_checkbox.on_update + def _(event: viser.GuiEvent) -> None: + # Clear move handles when toggled off. + if move_checkbox.value is False: + for t in transform_controls: + t.remove() + transform_controls.clear() + return + + def _make_transform_controls_callback( + keyframe: Tuple[Keyframe, viser.SceneNodeHandle], + controls: viser.TransformControlsHandle, + ) -> None: + @controls.on_update + def _(_) -> None: + keyframe[0].wxyz = controls.wxyz + keyframe[0].position = controls.position + + keyframe[1].wxyz = controls.wxyz + keyframe[1].position = controls.position + + camera_path.update_spline() + + # Show move handles. + assert event.client is not None + for keyframe_index, keyframe in camera_path._keyframes.items(): + controls = event.client.add_transform_controls( + f"/keyframe_move/{keyframe_index}", + scale=0.4, + wxyz=keyframe[0].wxyz, + position=keyframe[0].position, + ) + transform_controls.append(controls) + _make_transform_controls_callback(keyframe, controls) + + playback_folder = server.add_gui_folder("Playback") + with playback_folder: + duration_number = server.add_gui_number("Duration (sec)", min=0.0, max=1e8, step=0.0001, initial_value=4.0) + framerate_slider = server.add_gui_slider("FPS", min=1.0, max=240.0, step=1e-8, initial_value=30.0) + framerate_buttons = server.add_gui_button_group("", ("24", "30", "60")) + + @framerate_buttons.on_click + def _(_) -> None: + framerate_slider.value = float(framerate_buttons.value) + + play_button = server.add_gui_button("Play", icon=viser.Icon.PLAYER_PLAY) + pause_button = server.add_gui_button("Pause", icon=viser.Icon.PLAYER_PAUSE, visible=False) + attach_viewport_checkbox = server.add_gui_checkbox("Attach viewport", initial_value=False) + show_checkbox = server.add_gui_checkbox( + "Show keyframes", + initial_value=True, + hint="Show keyframes in the scene.", + ) + + @show_checkbox.on_update + def _(_: viser.GuiEvent) -> None: + camera_path.set_keyframes_visible(show_checkbox.value) + + def add_preview_frame_slider() -> Optional[viser.GuiInputHandle[int]]: + """Helper for creating the current frame # slider. This is removed and + re-added anytime the `max` value changes.""" + max_frame_index = int(framerate_slider.value * duration_number.value) - 1 + + if max_frame_index <= 0: + return None + with playback_folder: + preview_frame_slider = server.add_gui_slider( + "Preview frame", + min=0, + max=max_frame_index, + step=1, + initial_value=0, + # Place right after the pause button. + order=pause_button.order + 0.01, + ) + + @preview_frame_slider.on_update + def _(_) -> None: + max_frame_index = int(framerate_slider.value * duration_number.value) - 1 + maybe_pose_and_fov = camera_path.interpolate_pose_and_fov( + preview_frame_slider.value / max_frame_index if max_frame_index > 0 else 0 + ) + if maybe_pose_and_fov is None: + return + pose, fov = maybe_pose_and_fov + server.add_camera_frustum( + "/preview_camera", + fov=fov, + aspect=resolution.value[0] / resolution.value[1], + scale=0.35, + wxyz=pose.rotation().wxyz, + position=pose.translation(), + color=(10, 200, 30), + # Hack: hide green frustum if the viewport is attached. + # This is a waste of bandwidth, but will ensure that any old + # frustums are removed/aren't rendered. + # + # Easy to fix with a global variable. + visible=not attach_viewport_checkbox.value, + ) + if attach_viewport_checkbox.value: + for client in server.get_clients().values(): + client.camera.wxyz = pose.rotation().wxyz + client.camera.position = pose.translation() + client.camera.fov = fov + + return preview_frame_slider + + @attach_viewport_checkbox.on_update + def _(_) -> None: + if not attach_viewport_checkbox.value: + for client in server.get_clients().values(): + client.camera.fov = fov_degrees.value + + preview_frame_slider = add_preview_frame_slider() + + @duration_number.on_update + @framerate_slider.on_update + def _(_) -> None: + nonlocal preview_frame_slider + old = preview_frame_slider + assert old is not None + + preview_frame_slider = add_preview_frame_slider() + if preview_frame_slider is not None: + old.remove() + else: + preview_frame_slider = old + + # Play the camera trajectory when the play button is pressed. + @play_button.on_click + def _(_) -> None: + play_button.visible = False + pause_button.visible = True + + def play() -> None: + while not play_button.visible: + max_frame = int(framerate_slider.value * duration_number.value) + if max_frame > 0: + assert preview_frame_slider is not None + preview_frame_slider.value = (preview_frame_slider.value + 1) % max_frame + time.sleep(1.0 / framerate_slider.value) + + threading.Thread(target=play).start() + + # Play the camera trajectory when the play button is pressed. + @pause_button.on_click + def _(_) -> None: + play_button.visible = True + pause_button.visible = False + + render_button = server.add_gui_button( + "Generate Command", + color="green", + icon=viser.Icon.FILE_EXPORT, + hint="Generate the ns-render command for rendering the camera path.", + ) + + @render_button.on_click + def _(event: viser.GuiEvent) -> None: + """TODO: write the render JSON and show the render command.""" + assert event.client is not None + with event.client.add_gui_modal("TODO") as modal: + event.client.add_gui_markdown("TODO") + close_button = event.client.add_gui_button("Close") + + @close_button.on_click + def _(_) -> None: + modal.close() + + camera_path = CameraPath(server) + camera_path.default_fov = fov_degrees.value / 180.0 * onp.pi + + transform_controls: List[viser.SceneNodeHandle] = [] + + +def main() -> None: + """Launch a GUI with just the render panel, for development purposes.""" + server = viser.ViserServer() + server.configure_theme(dark_mode=True, control_layout="collapsible") + server.world_axes.visible = True + + populate_render_tab(server) + while True: + time.sleep(10.0) + + +if __name__ == "__main__": + main() diff --git a/nerfstudio/viewer_beta/render_state_machine.py b/nerfstudio/viewer_beta/render_state_machine.py index 6be9b3403..6355a5ba6 100644 --- a/nerfstudio/viewer_beta/render_state_machine.py +++ b/nerfstudio/viewer_beta/render_state_machine.py @@ -17,17 +17,14 @@ from __future__ import annotations import contextlib import threading -import time from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, get_args import torch - from nerfstudio.model_components.renderers import background_color_override_context from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils -from nerfstudio.viewer_beta import utils from nerfstudio.viewer_beta.utils import CameraState, get_camera if TYPE_CHECKING: @@ -55,7 +52,7 @@ class RenderStateMachine(threading.Thread): viewer: the viewer state """ - def __init__(self, viewer: Viewer): + def __init__(self, viewer: Viewer, viser_scale_ratio: float): threading.Thread.__init__(self) self.transitions: Dict[RenderStates, Dict[RenderActions, RenderStates]] = { s: {} for s in get_args(RenderStates) @@ -74,11 +71,12 @@ class RenderStateMachine(threading.Thread): self.next_action: Optional[RenderAction] = None self.state: RenderStates = "low_static" self.render_trigger = threading.Event() - self.target_fps = 24 + self.target_fps = 30 self.viewer = viewer self.interrupt_render_flag = False self.daemon = True self.output_keys = {} + self.viser_scale_ratio = viser_scale_ratio def action(self, action: RenderAction): """Takes an action and updates the state machine @@ -88,9 +86,7 @@ class RenderStateMachine(threading.Thread): """ if self.next_action is None: self.next_action = action - elif action.action == "step" and ( - self.state == "low_move" or self.next_action.action in ("move", "static", "rerender") - ): + elif action.action == "step" and (self.state == "low_move" or self.next_action.action in ("move", "rerender")): # ignore steps if: # 1. we are in low_moving state # 2. the current next_action is move, static, or rerender @@ -107,7 +103,6 @@ class RenderStateMachine(threading.Thread): # handle interrupt logic if self.state == "high" and self.next_action.action in ("move", "rerender"): - print("interrupting render", self.next_action.action) self.interrupt_render_flag = True self.render_trigger.set() @@ -117,14 +112,11 @@ class RenderStateMachine(threading.Thread): Args: camera_state: the current camera state """ - # initialize the camera ray bundle - utils.update_render_aabb( - crop_viewport=self.viewer.control_panel.crop_viewport, - crop_min=self.viewer.control_panel.crop_min, - crop_max=self.viewer.control_panel.crop_max, - model=self.viewer.get_model(), - ) + if self.viewer.control_panel.crop_viewport: + obb = self.viewer.control_panel.crop_obb + else: + obb = None image_height, image_width = self._calculate_image_res(camera_state.aspect) @@ -132,28 +124,39 @@ class RenderStateMachine(threading.Thread): camera = camera.to(self.viewer.get_model().device) assert camera is not None, "render called before viewer connected" - with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): - camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) - - with TimeWriter(None, None, write=False) as vis_t: + with TimeWriter(None, None, write=False) as vis_t: + with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): + camera_ray_bundle = camera.generate_rays(camera_indices=0, obb_box=obb) self.viewer.get_model().eval() step = self.viewer.step - if self.viewer.control_panel.crop_viewport: - color = self.viewer.control_panel.background_color - if color is None: - background_color = torch.tensor([0.0, 0.0, 0.0], device=self.viewer.pipeline.model.device) + try: + if self.viewer.control_panel.crop_viewport: + color = self.viewer.control_panel.background_color + if color is None: + background_color = torch.tensor([0.0, 0.0, 0.0], device=self.viewer.pipeline.model.device) + else: + background_color = torch.tensor( + [color[0] / 255.0, color[1] / 255.0, color[2] / 255.0], + device=self.viewer.get_model().device, + ) + with background_color_override_context( + background_color + ), torch.no_grad(), viewer_utils.SetTrace(self.check_interrupt): + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) else: - background_color = torch.tensor( - [color[0] / 255.0, color[1] / 255.0, color[2] / 255.0], - device=self.viewer.get_model().device, - ) - with background_color_override_context(background_color), torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) - else: - with torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + with torch.no_grad(), viewer_utils.SetTrace(self.check_interrupt): + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + except viewer_utils.IOChangeException: + self.viewer.get_model().train() + raise self.viewer.get_model().train() - num_rays = len(camera_ray_bundle) + num_rays = len(camera_ray_bundle) + if self.viewer.control_panel.layer_depth: + # convert to z_depth if depth compositing is enabled + R = camera.camera_to_worlds[0:3, 0:3].T + pts = camera_ray_bundle.directions * outputs["depth"] + pts = (R @ (pts.view(-1, 3).T)).T.view(*camera_ray_bundle.directions.shape) + outputs["gl_z_buf_depth"] = -pts[..., 2:3] # negative z axis is the coordinate convention render_time = vis_t.duration if writer.is_initialized(): writer.put_time( @@ -164,28 +167,25 @@ class RenderStateMachine(threading.Thread): def run(self): """Main loop for the render thread""" while True: - self.render_trigger.wait() - self.render_trigger.clear() + if not self.render_trigger.wait(0.2): + # if we haven't received a trigger in a while, send a static action + if self.viewer.camera_state is not None: + self.action(RenderAction(action="static", camera_state=self.viewer.camera_state)) action = self.next_action - assert action is not None, "Action should never be None at this point" + self.render_trigger.clear() + if action is None: + continue self.next_action = None if self.state == "high" and action.action == "static": # if we are in high res and we get a static action, we don't need to do anything continue self.state = self.transitions[self.state][action.action] try: - with viewer_utils.SetTrace(self.check_interrupt): - outputs = self._render_img(action.camera_state) + outputs = self._render_img(action.camera_state) except viewer_utils.IOChangeException: # if we got interrupted, don't send the output to the viewer continue self._send_output_to_viewer(outputs) - # if we rendered a static low res, we need to self-trigger a static high-res - if self.next_action is None: - # if there hasn't been an action, wait for 1/target_fps seconds in case we get another move command - time.sleep(1 / self.target_fps) - if self.state in ["low_static", "low_move"]: - self.action(RenderAction("static", action.camera_state)) def check_interrupt(self, frame, event, arg): """Raises interrupt when flag has been set and not already on lowest resolution. @@ -234,11 +234,14 @@ class RenderStateMachine(threading.Thread): selected_output[:, split_index] = torch.tensor([0.133, 0.157, 0.192], device=selected_output.device) selected_output = (selected_output * 255).type(torch.uint8) - + depth = ( + outputs["gl_z_buf_depth"].cpu().numpy() * self.viser_scale_ratio if "gl_z_buf_depth" in outputs else None + ) self.viewer.viser_server.set_background_image( selected_output.cpu().numpy(), format=self.viewer.config.image_format, jpeg_quality=self.viewer.config.jpeg_quality, + depth=depth, ) def _calculate_image_res(self, aspect_ratio: float) -> Tuple[int, int]: diff --git a/nerfstudio/viewer_beta/utils.py b/nerfstudio/viewer_beta/utils.py index 0dd435b88..a3dd7b1c2 100644 --- a/nerfstudio/viewer_beta/utils.py +++ b/nerfstudio/viewer_beta/utils.py @@ -12,16 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Any, List, Optional, Tuple, Union import numpy as np import torch from jaxtyping import Float - from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.data.scene_box import SceneBox from nerfstudio.models.base_model import Model +from torch import nn @dataclass @@ -53,7 +55,7 @@ def get_camera( pp_w = image_width / 2.0 pp_h = image_height / 2.0 focal_length = pp_h / np.tan(fov / 2.0) - intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float() + intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]], dtype=torch.float32) camera_type = CameraType.PERSPECTIVE @@ -63,7 +65,7 @@ def get_camera( cx=pp_w, cy=pp_h, camera_type=camera_type, - camera_to_worlds=camera_state.c2w, + camera_to_worlds=camera_state.c2w.to(torch.float32), times=torch.tensor([0.0], dtype=torch.float32), ) return camera @@ -93,3 +95,53 @@ def update_render_aabb( model.render_aabb = SceneBox(aabb=torch.stack([crop_min_tensor, crop_max_tensor], dim=0)) else: model.render_aabb = None + + +def parse_object( + obj: Any, + type_check, + tree_stub: str, +) -> List[Tuple[str, Any]]: + """ + obj: the object to parse + type_check: recursively adds instances of this type to the output + tree_stub: the path down the object tree to this object + + Returns: + a list of (path/to/object, obj), which represents the path down the object tree + along with the object itself + """ + + def add(ret: List[Tuple[str, Any]], ts: str, v: Any): + """ + helper that adds to ret, and if v exists already keeps the tree stub with + the shortest path + """ + for i, (t, o) in enumerate(ret): + if o == v: + if len(t.split("/")) > len(ts.split("/")): + ret[i] = (ts, v) + return + ret.append((ts, v)) + + if not hasattr(obj, "__dict__"): + return [] + ret = [] + # get a list of the properties of the object, sorted by whether things are instances of type_check + obj_props = [(k, getattr(obj, k)) for k in dir(obj)] + for k, v in obj_props: + if k[0] == "_": + continue + new_tree_stub = f"{tree_stub}/{k}" + if isinstance(v, type_check): + add(ret, new_tree_stub, v) + elif isinstance(v, nn.Module): + if v is obj: + # some nn.Modules might contain infinite references, e.g. consider foo = nn.Module(), foo.bar = foo + # to stop infinite recursion, we skip such attributes + continue + lower_rets = parse_object(v, type_check, new_tree_stub) + # check that the values aren't already in the tree + for ts, o in lower_rets: + add(ret, ts, o) + return ret diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py index a5271571d..9e2314ad4 100644 --- a/nerfstudio/viewer_beta/viewer.py +++ b/nerfstudio/viewer_beta/viewer.py @@ -16,15 +16,16 @@ from __future__ import annotations import threading +import time from pathlib import Path -from typing import TYPE_CHECKING, Literal, Optional +from typing import TYPE_CHECKING, Dict, List, Literal, Optional import numpy as np import torch import torchvision import viser +import viser.theme import viser.transforms as vtf - from nerfstudio.configs import base_config as cfg from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.models.base_model import Model @@ -33,13 +34,19 @@ from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName from nerfstudio.viewer.server import viewer_utils from nerfstudio.viewer_beta.control_panel import ControlPanel +from nerfstudio.viewer_beta.export_panel import populate_export_tab +from nerfstudio.viewer_beta.render_panel import populate_render_tab from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStateMachine -from nerfstudio.viewer_beta.utils import CameraState +from nerfstudio.viewer_beta.utils import CameraState, parse_object +from nerfstudio.viewer_beta.viewer_elements import ViewerControl, ViewerElement + +from nerfstudio.cameras.camera_optimizers import CameraOptimizer if TYPE_CHECKING: from nerfstudio.engine.trainer import Trainer -VISER_NERFSTUDIO_SCALE_RATIO: int = 10 + +VISER_NERFSTUDIO_SCALE_RATIO: float = 10.0 @decorate_all([check_main_thread]) @@ -95,57 +102,148 @@ class Viewer: self.train_btn_state: Literal["training", "paused", "completed"] = "training" self._prev_train_state: Literal["training", "paused", "completed"] = "training" - self.camera_message = None - + self.client: Optional[viser.ClientHandle] = None self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port) - - self.viser_server.on_client_connect(self.handle_new_client) - - self.control_panel = ControlPanel( - self.viser_server, - self.include_time, - self._interrupt_render, - self._crop_params_update, - self._output_type_change, - self._output_split_type_change, + buttons = ( + viser.theme.TitlebarButton( + text="Getting Started", + icon=None, + href="https://nerf.studio", + ), + viser.theme.TitlebarButton( + text="Github", + icon="GitHub", + href="https://github.com/nerfstudio-project/nerfstudio", + ), + viser.theme.TitlebarButton( + text="Documentation", + icon="Description", + href="https://docs.nerf.studio", + ), + ) + image = viser.theme.TitlebarImage( + image_url_light="https://docs.nerf.studio/en/latest/_static/imgs/logo.png", + image_url_dark="https://docs.nerf.studio/en/latest/_static/imgs/logo-dark.png", + image_alt="NerfStudio Logo", + href="https://docs.nerf.studio/", + ) + titlebar_theme = viser.theme.TitlebarConfig(buttons=buttons, image=image) + self.viser_server.configure_theme( + titlebar_content=titlebar_theme, + control_layout="collapsible", + dark_mode=True, + brand_color=(255, 211, 105), ) - self.render_statemachine = RenderStateMachine(self) + self.render_statemachine = RenderStateMachine(self, VISER_NERFSTUDIO_SCALE_RATIO) + self.viser_server.on_client_connect(self.handle_new_client) + + tabs = self.viser_server.add_gui_tab_group() + control_tab = tabs.add_tab("Control", viser.Icon.SETTINGS) + with control_tab: + self.control_panel = ControlPanel( + self.viser_server, + self.include_time, + VISER_NERFSTUDIO_SCALE_RATIO, + self._interrupt_render, + self._crop_params_update, + self._output_type_change, + self._output_split_type_change, + self._toggle_training_state, + self.set_camera_visibility, + ) + with tabs.add_tab("Render", viser.Icon.CAMERA): + populate_render_tab(self.viser_server) + + with tabs.add_tab("Export", viser.Icon.PACKAGE_EXPORT): + config_path = self.log_filename.parents[0] / "config.yml" + populate_export_tab(self.viser_server, self.control_panel, config_path) + + def nested_folder_install(folder_labels: List[str], element: ViewerElement): + if len(folder_labels) == 0: + element.install(self.viser_server) + # also rewire the hook to rerender + prev_cb = element.cb_hook + element.cb_hook = lambda element: [prev_cb(element), self._interrupt_render(element)] + else: + with self.viser_server.add_gui_folder(folder_labels[0]): + nested_folder_install(folder_labels[1:], element) + + with control_tab: + self.viewer_elements = [] + self.viewer_elements.extend(parse_object(pipeline, ViewerElement, "Custom Elements")) + for param_path, element in self.viewer_elements: + folder_labels = param_path.split("/")[:-1] + nested_folder_install(folder_labels, element) + + # scrape the trainer/pipeline for any ViewerControl objects to initialize them + self.viewer_controls: List[ViewerControl] = [ + e for (_, e) in parse_object(self.trainer, ViewerControl, "Custom Elements") + ] + for c in self.viewer_controls: + c._setup(self) self.render_statemachine.start() def handle_new_client(self, client: viser.ClientHandle) -> None: + self.client = client + self.last_move_time = 0 + @client.camera.on_update - def _(_: viser.CameraHandle) -> None: - R = vtf.SO3(wxyz=client.camera.wxyz) + def _(cam: viser.CameraHandle) -> None: + assert self.client is not None + self.last_move_time = time.time() + R = vtf.SO3(wxyz=self.client.camera.wxyz) R = R @ vtf.SO3.from_x_radians(np.pi) - R = torch.tensor(R.as_matrix(), dtype=torch.float32) - pos = torch.tensor(client.camera.position, dtype=torch.float32) / VISER_NERFSTUDIO_SCALE_RATIO + R = torch.tensor(R.as_matrix()) + pos = torch.tensor(self.client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO c2w = torch.concatenate([R, pos[:, None]], dim=1) - self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w) + self.camera_state = CameraState(fov=self.client.camera.fov, aspect=self.client.camera.aspect, c2w=c2w) self.render_statemachine.action(RenderAction("move", self.camera_state)) + def set_camera_visibility(self, visible: bool) -> None: + """Toggle the visibility of the training cameras.""" + with self.viser_server.atomic(): + for idx in self.camera_handles: + self.camera_handles[idx].visible = visible + + def update_camera_poses(self): + # Update the train camera locations based on optimization + assert self.camera_handles is not None + idxs = list(self.camera_handles.keys()) + if hasattr(self.pipeline.datamanager, "train_camera_optimizer"): + camera_optimizer = self.pipeline.datamanager.train_camera_optimizer + else: + camera_optimizer = self.pipeline.model.camera_optimizer + with torch.no_grad(): + assert isinstance(camera_optimizer, CameraOptimizer) + c2ws_delta = camera_optimizer(torch.tensor(idxs, device=camera_optimizer.device)).cpu().numpy() + for idx in idxs: + # both are numpy arrays + c2w_orig = self.original_c2w[idx] + c2w_delta = c2ws_delta[idx, ...] + c2w = c2w_orig @ np.concatenate((c2w_delta, np.array([[0, 0, 0, 1]])), axis=0) + R = vtf.SO3.from_matrix(c2w[:3, :3]) # type: ignore + R = R @ vtf.SO3.from_x_radians(np.pi) + self.camera_handles[idx].position = c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO + self.camera_handles[idx].wxyz = R.wxyz + def _interrupt_render(self, _) -> None: """Interrupt current render.""" if self.camera_state is not None: self.render_statemachine.action(RenderAction("rerender", self.camera_state)) + def _toggle_training_state(self, _) -> None: + """Toggle the trainer's training state.""" + if self.trainer is not None: + if self.trainer.training_state == "training": + self.trainer.training_state = "paused" + elif self.trainer.training_state == "paused": + self.trainer.training_state = "training" + def _crop_params_update(self, _) -> None: """Update crop parameters""" - print("Crop params not set up") - # crop_min = torch.tensor(self.control_panel.crop_min, dtype=torch.float32) - # crop_max = torch.tensor(self.control_panel.crop_max, dtype=torch.float32) - # scene_box = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0)) - # self.viser_server.update_scene_box(scene_box) - # crop_scale = crop_max - crop_min - # crop_center = (crop_max + crop_min) / 2.0 - # self.viser_server.send_crop_params( - # crop_enabled=self.control_panel.crop_viewport, - # crop_bg_color=self.control_panel.background_color, - # crop_scale=tuple(crop_scale.tolist()), - # crop_center=tuple(crop_center.tolist()), - # ) - # if self.camera_message is not None: - # self.render_statemachine.action(RenderAction("rerender", self.camera_message)) + if self.camera_state is not None: + self.render_statemachine.action(RenderAction("move", self.camera_state)) def _output_type_change(self, _): self.output_type_changed = True @@ -181,8 +279,9 @@ class Viewer: dataset: dataset to render in the scene train_state: Current status of training """ - # draw the training cameras and images + self.camera_handles: Dict[int, viser.CameraFrustumHandle] = {} + self.original_c2w: Dict[int, np.ndarray] = {} image_indices = self._pick_drawn_image_idxs(len(train_dataset)) for idx in image_indices: image = train_dataset[idx]["image"] @@ -195,16 +294,26 @@ class Viewer: c2w = camera.camera_to_worlds.cpu().numpy() R = vtf.SO3.from_matrix(c2w[:3, :3]) R = R @ vtf.SO3.from_x_radians(np.pi) - self.viser_server.add_camera_frustum( - name=f"Camera {idx}", - fov=float(camera.fx[0]), - scale=0.2, + camera_handle = self.viser_server.add_camera_frustum( + name=f"/cameras/camera_{idx:05d}", + fov=float(2 * np.arctan(camera.cx / camera.fx[0])), + scale=0.1, aspect=float(camera.cx[0] / camera.cy[0]), image=image_uint8, wxyz=R.wxyz, position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO, ) + @camera_handle.on_click + def _(event: viser.ClickEvent[viser.CameraFrustumHandle]) -> None: + assert self.client is not None + with self.client.atomic(): + self.client.camera.position = event.target.position + self.client.camera.wxyz = event.target.wxyz + + self.camera_handles[idx] = camera_handle + self.original_c2w[idx] = c2w + self.train_state = train_state self.train_util = 0.9 @@ -219,7 +328,10 @@ class Viewer: if self.camera_state is None: return - + # this stops training while moving to make the response smoother + while time.time() - self.last_move_time < 0.1: + time.sleep(0.05) + # self.render_statemachine.action(RenderAction("static", self.camera_state)) if self.trainer is not None and self.trainer.training_state == "training" and self.train_util != 1: if ( EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"] @@ -239,6 +351,8 @@ class Viewer: if step > self.last_step + render_freq: self.last_step = step self.render_statemachine.action(RenderAction("step", self.camera_state)) + self.update_camera_poses() + self.control_panel.update_step(step) def update_colormap_options(self, dimensions: int, dtype: type) -> None: """update the colormap options based on the current render diff --git a/nerfstudio/viewer_beta/viewer_elements.py b/nerfstudio/viewer_beta/viewer_elements.py index f9fe0e071..916566d5d 100644 --- a/nerfstudio/viewer_beta/viewer_elements.py +++ b/nerfstudio/viewer_beta/viewer_elements.py @@ -19,8 +19,12 @@ from __future__ import annotations from abc import abstractmethod -from typing import Any, Callable, Generic, List, Optional, Tuple, Union +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Generic, List, Optional, Tuple, Union +import numpy as np +import torch +import viser.transforms as vtf from typing_extensions import LiteralString, TypeVar from viser import ( GuiButtonGroupHandle, @@ -30,10 +34,131 @@ from viser import ( ViserServer, ) +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.viewer_beta.utils import CameraState, get_camera + +if TYPE_CHECKING: + from nerfstudio.viewer_beta.viewer import Viewer + TValue = TypeVar("TValue") TString = TypeVar("TString", default=str, bound=str) +@dataclass +class ViewerClick: + """ + Class representing a click in the viewer as a ray. + """ + + # the information here matches the information in the ClickMessage, + # but we implement a wrapper as an abstraction layer + origin: Tuple[float, float, float] + """The origin of the click in world coordinates (center of camera)""" + direction: Tuple[float, float, float] + """ + The direction of the click if projected from the camera through the clicked pixel, + in world coordinates + """ + + +class ViewerControl: + """ + class for exposing non-gui controls of the viewer to the user + """ + + def __init__(self): + # this should be a user-facing constructor, since it will be used inside the model/pipeline class + self.click_cbs = [] + + def _setup(self, viewer: Viewer): + """ + Internal use only, setup the viewer control with the viewer state object + + Args: + viewer: The viewer object (viewer.py) + """ + self.viewer: Viewer = viewer + self.viser_server: ViserServer = viewer.viser_server + + def set_pose( + self, + position: Optional[Tuple[float, float, float]] = None, + look_at: Optional[Tuple[float, float, float]] = None, + instant: bool = False, + ): + """ + Set the camera position of the viewer camera. + + Args: + position: The new position of the camera in world coordinates + look_at: The new look_at point of the camera in world coordinates + instant: If the camera should move instantly or animate to the new position + """ + raise NotImplementedError() + + def set_fov(self, fov): + """ + Set the FOV of the viewer camera + + Args: + fov: The new FOV of the camera in degrees + + """ + raise NotImplementedError() + + def set_crop(self, min_point: Tuple[float, float, float], max_point: Tuple[float, float, float]): + """ + Set the scene crop box of the viewer to the specified min,max point + + Args: + min_point: The minimum point of the crop box + max_point: The maximum point of the crop box + + """ + raise NotImplementedError() + + def get_camera(self, img_height: int, img_width: int) -> Optional[Cameras]: + """ + Returns the Cameras object representing the current camera for the viewer, or None if the viewer + is not connected yet + + Args: + img_height: The height of the image to get camera intrinsics for + img_width: The width of the image to get camera intrinsics for + """ + assert self.viewer.client is not None + + from nerfstudio.viewer_beta.viewer import VISER_NERFSTUDIO_SCALE_RATIO + + R = vtf.SO3(wxyz=self.viewer.client.camera.wxyz) + R = R @ vtf.SO3.from_x_radians(np.pi) + R = torch.tensor(R.as_matrix()) + pos = torch.tensor(self.viewer.client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO + c2w = torch.concatenate([R, pos[:, None]], dim=1) + camera_state = CameraState(fov=self.viewer.client.camera.fov, aspect=self.viewer.client.camera.aspect, c2w=c2w) + return get_camera(camera_state, img_height, img_width) + + def register_click_cb(self, cb: Callable): + """ + Add a callback which will be called when a click is detected in the viewer. + + Args: + cb: The callback to call when a click is detected. + The callback should take a ViewerClick object as an argument + """ + self.click_cbs.append(cb) + + def on_click(self, msg): + """ + Internal use only, register a click in the viewer which propagates to all self.click_cbs + """ + raise NotImplementedError() + + @property + def server(self): + return self.viser_server + + class ViewerElement(Generic[TValue]): """Base class for all viewer elements @@ -100,7 +225,7 @@ class ViewerButton(ViewerElement[bool]): super().__init__(name, disabled=disabled, cb_hook=cb_hook) def _create_gui_handle(self, viser_server: ViserServer) -> None: - self.gui_handle = viser_server.add_gui_button(self.name, disabled=self.disabled) + self.gui_handle = viser_server.add_gui_button(label=self.name, disabled=self.disabled) def install(self, viser_server: ViserServer) -> None: self._create_gui_handle(viser_server) diff --git a/pyproject.toml b/pyproject.toml index 6d8fb7804..1347f954b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,12 +46,13 @@ dependencies = [ "requests", "rich>=12.5.1", "scikit-image>=0.19.3", + "splines==0.3.0", "tensorboard>=2.13.0", "torch>=1.13.1", "torchvision>=0.14.1", "torchmetrics[image]>=1.0.1", "typing_extensions>=4.4.0", - "viser==0.1.0", + "viser==0.1.2", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas", diff --git a/tests/utils/test_visualization.py b/tests/utils/test_visualization.py index 8fad19579..b3d9f53d1 100644 --- a/tests/utils/test_visualization.py +++ b/tests/utils/test_visualization.py @@ -42,6 +42,9 @@ def test_apply_boolean_colormap(): def test_cube_center(): """Test adding a cube""" cube = plotly_utils.get_cube(1.0, torch.Tensor([2.0, 3.0, 4.0])) + assert cube.x is not None + assert cube.y is not None + assert cube.z is not None assert cube.x[0] == 1.5 assert cube.y[0] == 2.5 assert cube.z[-1] == 4.5 @@ -50,6 +53,9 @@ def test_cube_center(): def test_aabb_center(): """Test adding a cube with different dsid length""" cube = plotly_utils.get_cube(torch.Tensor([2.0, 1.0, 3.0])) + assert cube.x is not None + assert cube.y is not None + assert cube.z is not None assert cube.x[0] == -1 assert cube.y[0] == -0.5 assert cube.z[-1] == 1.5 From 6af0bcfbf867584f357082a80877c9ce640f9867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Fri, 8 Sep 2023 10:26:41 +0200 Subject: [PATCH 02/19] Unify dataparser and method discovery (#2398) * Unify external dataparser registration * Unify dataparser and method discovery --- docs/developer_guides/new_methods.md | 9 +++ nerfstudio/configs/dataparser_configs.py | 2 +- nerfstudio/plugins/registry_dataparser.py | 42 +++++++++++- tests/plugins/test_registry.py | 78 +++++++++++++++++++++++ 4 files changed, 127 insertions(+), 4 deletions(-) diff --git a/docs/developer_guides/new_methods.md b/docs/developer_guides/new_methods.md index a8f8bd423..4b56d8581 100644 --- a/docs/developer_guides/new_methods.md +++ b/docs/developer_guides/new_methods.md @@ -132,6 +132,15 @@ finally run the following to register the dataparser. pip install -e . ``` +Similarly to the method develomement, you can also use environment variables to register dataparsers. +Use the `NERFSTUDIO_DATAPARSER_CONFIGS` environment variable: + +``` +export NERFSTUDIO_DATAPARSER_CONFIGS="my-dataparser=my_package.my_config:MyDataParser" +``` + +Same as with custom methods, `NERFSTUDIO_DATAPARSER_CONFIGS` environment variable additionally accepts a function or derived class to temporarily register your custom method. + ## Running custom method After registering your method you should be able to run the method with, diff --git a/nerfstudio/configs/dataparser_configs.py b/nerfstudio/configs/dataparser_configs.py index 46a728141..f60092a46 100644 --- a/nerfstudio/configs/dataparser_configs.py +++ b/nerfstudio/configs/dataparser_configs.py @@ -54,7 +54,7 @@ dataparsers = { "colmap": ColmapDataParserConfig(), } -external_dataparsers = discover_dataparsers() +external_dataparsers, _ = discover_dataparsers() all_dataparsers = {**dataparsers, **external_dataparsers} if TYPE_CHECKING: diff --git a/nerfstudio/plugins/registry_dataparser.py b/nerfstudio/plugins/registry_dataparser.py index f15fdc78c..e35aeaf48 100644 --- a/nerfstudio/plugins/registry_dataparser.py +++ b/nerfstudio/plugins/registry_dataparser.py @@ -16,6 +16,8 @@ Module that keeps all registered plugins and allows for plugin discovery. """ +import os +import importlib import sys import typing as t from dataclasses import dataclass @@ -41,21 +43,55 @@ class DataParserSpecification: config: DataParserConfig """Dataparser configuration""" + description: t.Optional[str] = None + """Description of the dataparser""" -def discover_dataparsers() -> t.Dict[str, DataParserConfig]: + +def discover_dataparsers() -> t.Tuple[t.Dict[str, DataParserConfig], t.Dict[str, str]]: """ Discovers all dataparsers registered using the `nerfstudio.dataparser_configs` entrypoint. + And also dataparsers in the NERFSTUDIO_DATAPARSER_CONFIGS environment variable. """ dataparsers = {} + descriptions = {} discovered_entry_points = entry_points(group="nerfstudio.dataparser_configs") for name in discovered_entry_points.names: spec = discovered_entry_points[name].load() if not isinstance(spec, DataParserSpecification): CONSOLE.print( - f"[bold yellow]Warning: Could not entry point {spec} as it is an instance of DataParserSpecification" + f"[bold yellow]Warning: Could not entry point {spec} as it is not an instance of DataParserSpecification" ) continue spec = t.cast(DataParserSpecification, spec) dataparsers[name] = spec.config + descriptions[name] = spec.description - return dataparsers + if "NERFSTUDIO_DATAPARSER_CONFIGS" in os.environ: + try: + strings = os.environ["NERFSTUDIO_DATAPARSER_CONFIGS"].split(",") + for definition in strings: + if not definition: + continue + name, path = definition.split("=") + CONSOLE.print(f"[bold green]Info: Loading method {name} from environment variable") + module, config_name = path.split(":") + dataparser_config = getattr(importlib.import_module(module), config_name) + + # method_config specified as function or class -> instance + if callable(dataparser_config): + dataparser_config = dataparser_config() + + # check for valid instance type + if not isinstance(dataparser_config, DataParserSpecification): + raise TypeError("Method is not an instance of DataParserSpecification") + + # save to methods + dataparsers[name] = dataparser_config.config + descriptions[name] = dataparser_config.description + except Exception: # pylint: disable=broad-except + CONSOLE.print_exception() + CONSOLE.print( + "[bold red]Error: Could not load methods from environment variable NERFSTUDIO_DATAPARSER_CONFIGS" + ) + + return dataparsers, descriptions diff --git a/tests/plugins/test_registry.py b/tests/plugins/test_registry.py index 08805b2d9..7263a79c0 100644 --- a/tests/plugins/test_registry.py +++ b/tests/plugins/test_registry.py @@ -9,6 +9,8 @@ from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.plugins import registry from nerfstudio.plugins.types import MethodSpecification +from nerfstudio.plugins import registry_dataparser +from nerfstudio.plugins.registry_dataparser import DataParserSpecification, discover_dataparsers, DataParserConfig if sys.version_info < (3, 10): import importlib_metadata @@ -94,3 +96,79 @@ def test_discover_methods_from_environment_variable_function(): def test_discover_methods_from_environment_variable_instance(): """Tests if a custom method instance from env variable gets properly registered using the discover_methods method""" _test_discover_methods_from_environment_variable("TestConfig") + + +@dataclass +class TestDataparserConfigClass(DataParserSpecification): + config: DataParserConfig = DataParserConfig() + description: str = "Test description" + + +def TestDataparserConfigFunc(): + return TestDataparserConfigClass() + + +TestDataparserConfig = TestDataparserConfigClass() + + +def test_discover_dataparser(): + """Tests if a custom method gets properly registered using the discover_dataparsers method""" + entry_points_backup = registry_dataparser.entry_points + + def entry_points(group=None): + return importlib_metadata.EntryPoints( + [ + importlib_metadata.EntryPoint( + name="test", value="test_registry:TestDataparserConfig", group="nerfstudio.dataparser_configs" + ) + ] + ).select(group=group) + + try: + # Mock importlib entry_points + registry_dataparser.entry_points = entry_points + + # Discover plugins + dataparsers, _ = discover_dataparsers() + assert "test" in dataparsers + config = dataparsers["test"] + assert isinstance(config, DataParserConfig) + finally: + # Revert mock + registry_dataparser.entry_points = entry_points_backup + + +def _test_discover_dataparsers_from_environment_variable(method): + """Tests if a custom method from env variable gets properly registered using the discover_dataparsers method""" + old_env = None + try: + old_env = os.environ.get("NERFSTUDIO_DATAPARSER_CONFIGS", None) + method_key = f"{method}-env" + os.environ["NERFSTUDIO_DATAPARSER_CONFIGS"] = f"{method_key}=test_registry:{method}" + + # Discover plugins + dataparsers, _ = discover_dataparsers() + assert method_key in dataparsers + config = dataparsers[method_key] + assert isinstance(config, DataParserConfig) + finally: + # Revert mock + if old_env is not None: + os.environ["NERFSTUDIO_DATAPARSER_CONFIGS"] = old_env + else: + del os.environ["NERFSTUDIO_DATAPARSER_CONFIGS"] + + +def test_discover_dataparsers_from_environment_variable_class(): + """Tests if a custom dataparser class from env variable gets properly registered using the discover_methods method""" + _test_discover_dataparsers_from_environment_variable("TestDataparserConfigClass") + + +def test_discover_dataparsers_from_environment_variable_function(): + """Tests if a custom dataparser function from env variable gets properly registered using the discover_methods method""" + _test_discover_dataparsers_from_environment_variable("TestDataparserConfigFunc") + + +def test_discover_dataparsers_from_environment_variable_instance(): + """Tests if a custom dataparser instance from env variable gets properly registered using the discover_methods method""" + _test_discover_dataparsers_from_environment_variable("TestDataparserConfig") From 06e59ed1ec147c1b6357b9a2195cb12822ebdd33 Mon Sep 17 00:00:00 2001 From: drewgilliam Date: Fri, 8 Sep 2023 12:14:16 -0400 Subject: [PATCH 03/19] camera/plugin docs (#2412) * api docs for cameras & plugins * fix docstrings --- docs/reference/api/cameras.rst | 46 ++++++++++++++++++++++++++++++ docs/reference/api/index.rst | 2 ++ docs/reference/api/plugins.rst | 25 ++++++++++++++++ nerfstudio/cameras/camera_utils.py | 15 ++++++---- nerfstudio/cameras/cameras.py | 28 +++++++++--------- 5 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 docs/reference/api/cameras.rst create mode 100644 docs/reference/api/plugins.rst diff --git a/docs/reference/api/cameras.rst b/docs/reference/api/cameras.rst new file mode 100644 index 000000000..7653852c7 --- /dev/null +++ b/docs/reference/api/cameras.rst @@ -0,0 +1,46 @@ +.. _cameras: + +Cameras +============ + +Cameras +---------------- + +.. automodule:: nerfstudio.cameras.cameras + :members: + :show-inheritance: + +Camera Optimizers +----------------- + +.. automodule:: nerfstudio.cameras.camera_optimizers + :members: + :show-inheritance: + +Camera Paths +---------------- + +.. automodule:: nerfstudio.cameras.camera_paths + :members: + :show-inheritance: + +Camera Utils +---------------- + +.. automodule:: nerfstudio.cameras.camera_utils + :members: + :show-inheritance: + +Lie Groups +---------------- + +.. automodule:: nerfstudio.cameras.lie_groups + :members: + :show-inheritance: + +Rays +---------------- + +.. automodule:: nerfstudio.cameras.rays + :members: + :show-inheritance: diff --git a/docs/reference/api/index.rst b/docs/reference/api/index.rst index 01c7d7fe7..a6691feca 100644 --- a/docs/reference/api/index.rst +++ b/docs/reference/api/index.rst @@ -7,6 +7,7 @@ TODO: Explanation of each component .. toctree:: + cameras config data/index fields @@ -14,5 +15,6 @@ TODO: Explanation of each component models model_components/index optimizers + plugins utils/index viewer diff --git a/docs/reference/api/plugins.rst b/docs/reference/api/plugins.rst new file mode 100644 index 000000000..bb8504559 --- /dev/null +++ b/docs/reference/api/plugins.rst @@ -0,0 +1,25 @@ +.. _plugins: + +Plugins +============ + +Method Registry +---------------------- + +.. automodule:: nerfstudio.plugins.registry + :members: + :show-inheritance: + +DataParser Registry +---------------------- + +.. automodule:: nerfstudio.plugins.registry_dataparser + :members: + :show-inheritance: + +Types +---------------------- + +.. automodule:: nerfstudio.plugins.types + :members: + :show-inheritance: diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index 2c0b746d1..8c527b9db 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -519,21 +519,24 @@ def auto_orient_and_center_poses( method: Literal["pca", "up", "vertical", "none"] = "up", center_method: Literal["poses", "focus", "none"] = "poses", ) -> Tuple[Float[Tensor, "*num_poses 3 4"], Float[Tensor, "3 4"]]: - """Orients and centers the poses. We provide two methods for orientation: pca and up. + """Orients and centers the poses. - pca: Orient the poses so that the principal directions of the camera centers are aligned + We provide three methods for orientation: + + - pca: Orient the poses so that the principal directions of the camera centers are aligned with the axes, Z corresponding to the smallest principal component. This method works well when all of the cameras are in the same plane, for example when images are taken using a mobile robot. - up: Orient the poses so that the average up vector is aligned with the z axis. + - up: Orient the poses so that the average up vector is aligned with the z axis. This method works well when images are not at arbitrary angles. - vertical: Orient the poses so that the Z 3D direction projects close to the + - vertical: Orient the poses so that the Z 3D direction projects close to the y axis in images. This method works better if cameras are not all looking in the same 3D direction, which may happen in camera arrays or in LLFF. There are two centering methods: - poses: The poses are centered around the origin. - focus: The origin is set to the focus of attention of all cameras (the + + - poses: The poses are centered around the origin. + - focus: The origin is set to the focus of attention of all cameras (the closest point to cameras optical axes). Recommended for inward-looking camera configurations. diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index c2ef887a6..68f7b186f 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -329,25 +329,27 @@ class Cameras(TensorDataclass): This function will standardize the input arguments and then call the _generate_rays_from_coords function to generate the rays. Our goal is to parse the arguments and then get them into the right shape: - - camera_indices: (num_rays:..., num_cameras_batch_dims) - - coords: (num_rays:..., 2) - - camera_opt_to_camera: (num_rays:..., 3, 4) or None - - distortion_params_delta: (num_rays:..., 6) or None + + - camera_indices: (num_rays:..., num_cameras_batch_dims) + - coords: (num_rays:..., 2) + - camera_opt_to_camera: (num_rays:..., 3, 4) or None + - distortion_params_delta: (num_rays:..., 6) or None Read the docstring for _generate_rays_from_coords for more information on how we generate the rays after we have standardized the arguments. We are only concerned about different combinations of camera_indices and coords matrices, and the following are the 4 cases we have to deal with: - 1. isinstance(camera_indices, int) and coords == None - - In this case we broadcast our camera_indices / coords shape (h, w, 1 / 2 respectively) - 2. isinstance(camera_indices, int) and coords != None - - In this case, we broadcast camera_indices to the same batch dim as coords - 3. not isinstance(camera_indices, int) and coords == None - - In this case, we will need to set coords so that it is of shape (h, w, num_rays, 2), and broadcast - all our other args to match the new definition of num_rays := (h, w) + num_rays - 4. not isinstance(camera_indices, int) and coords != None - - In this case, we have nothing to do, only check that the arguments are of the correct shape + + 1. isinstance(camera_indices, int) and coords == None + - In this case we broadcast our camera_indices / coords shape (h, w, 1 / 2 respectively) + 2. isinstance(camera_indices, int) and coords != None + - In this case, we broadcast camera_indices to the same batch dim as coords + 3. not isinstance(camera_indices, int) and coords == None + - In this case, we will need to set coords so that it is of shape (h, w, num_rays, 2), and broadcast + all our other args to match the new definition of num_rays := (h, w) + num_rays + 4. not isinstance(camera_indices, int) and coords != None + - In this case, we have nothing to do, only check that the arguments are of the correct shape There is one more edge case we need to be careful with: when we have "jagged cameras" (ie: different heights and widths for each camera). This isn't problematic when we specify coords, since coords is already a tensor. From 77ac7d0cf85f0d4b76ddf9ca3d60082a4cec53e2 Mon Sep 17 00:00:00 2001 From: Sebastiaan <751205+SharkWipf@users.noreply.github.com> Date: Sat, 9 Sep 2023 01:00:10 +0200 Subject: [PATCH 04/19] Fix issue when extracting all frames from a video. (#2413) * Remove superfluous num_frames * process_data video: Always use -vsync vfr, even when extracting all images. --- nerfstudio/process_data/process_data_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index 6db731e4f..334dab4de 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -167,7 +167,6 @@ def convert_video_to_images( start_y = crop_factor[0] crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}," - num_frames = get_num_frames_in_video(video_path) spacing = num_frames // num_frames_target downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}[out{i}]" for i in range(num_downscales + 1)] @@ -184,9 +183,10 @@ def convert_video_to_images( + ";".join(downscale_chains) ) + ffmpeg_cmd += " -vsync vfr" + if spacing > 1: CONSOLE.print("Number of frames to extract:", math.ceil(num_frames / spacing)) - ffmpeg_cmd += " -vsync vfr" select_cmd = f"thumbnail={spacing},setpts=N/TB," else: CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.") From ead9b37c658708af7971871839c6fbf613d8092d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Mon, 11 Sep 2023 09:50:18 +0200 Subject: [PATCH 05/19] Allow colmap parser to load 3D points (#2408) --- .../data/dataparsers/colmap_dataparser.py | 78 ++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py index a8a993d30..046f6d5d3 100644 --- a/nerfstudio/data/dataparsers/colmap_dataparser.py +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -71,6 +71,10 @@ class ColmapDataParserConfig(DataParserConfig): """Path to depth maps directory. If not set, depths are not loaded.""" colmap_path: Path = Path("sparse/0") """Path to the colmap reconstruction directory relative to the data path.""" + load_3D_points: bool = True + """Whether to load the 3D points from the colmap reconstruction.""" + max_2D_matches_per_3D_point: int = -1 + """Maximum number of 2D matches per 3D point. If set to -1, all 2D matches are loaded. If set to 0, no 2D matches are loaded.""" class ColmapDataParser(DataParser): @@ -202,7 +206,7 @@ class ColmapDataParser(DataParser): raise ValueError(f"Unknown dataparser split {split}") return indices - def _generate_dataparser_outputs(self, split: str = "train"): + def _generate_dataparser_outputs(self, split: str = "train", **kwargs): assert self.config.data.exists(), f"Data directory {self.config.data} does not exist." colmap_path = self.config.data / self.config.colmap_path assert colmap_path.exists(), f"Colmap path {colmap_path} does not exist." @@ -328,6 +332,11 @@ class ColmapDataParser(DataParser): applied_scale = float(meta["applied_scale"]) scale_factor *= applied_scale + metadata = {} + if self.config.load_3D_points: + # Load 3D points + metadata.update(self._load_3D_points(colmap_path, transform_matrix, scale_factor)) + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, @@ -338,10 +347,77 @@ class ColmapDataParser(DataParser): metadata={ "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, "depth_unit_scale_factor": self.config.depth_unit_scale_factor, + **metadata, }, ) return dataparser_outputs + def _load_3D_points(self, colmap_path: Path, transform_matrix: torch.Tensor, scale_factor: float): + if (colmap_path / "points3D.bin").exists(): + colmap_points = colmap_utils.read_points3D_binary(colmap_path / "points3D.bin") + elif (colmap_path / "points3D.txt").exists(): + colmap_points = colmap_utils.read_points3D_text(colmap_path / "points3D.txt") + else: + raise ValueError(f"Could not find points3D.txt or points3D.bin in {colmap_path}") + points3D = torch.from_numpy(np.array([p.xyz for p in colmap_points.values()], dtype=np.float32)) + points3D = ( + torch.cat( + ( + points3D, + torch.ones_like(points3D[..., :1]), + ), + -1, + ) + @ transform_matrix.T + ) + points3D *= scale_factor + + # Load point colours + points3D_rgb = torch.from_numpy(np.array([p.rgb for p in colmap_points.values()], dtype=np.uint8)) + points3D_num_points = torch.tensor([len(p.image_ids) for p in colmap_points.values()], dtype=torch.int64) + out = { + "points3D_xyz": points3D, + "points3D_rgb": points3D_rgb, + "points3D_error": torch.from_numpy(np.array([p.error for p in colmap_points.values()], dtype=np.float32)), + "points3D_num_points": points3D_num_points, + } + if self.config.max_2D_matches_per_3D_point != 0: + if (colmap_path / "images.txt").exists(): + im_id_to_image = colmap_utils.read_images_text(colmap_path / "images.txt") + elif (colmap_path / "images.bin").exists(): + im_id_to_image = colmap_utils.read_images_binary(colmap_path / "images.bin") + else: + raise ValueError(f"Could not find images.txt or images.bin in {colmap_path}") + downscale_factor = self._downscale_factor + max_num_points = int(torch.max(points3D_num_points).item()) + if self.config.max_2D_matches_per_3D_point > 0: + max_num_points = min(max_num_points, self.config.max_2D_matches_per_3D_point) + points3D_image_ids = [] + points3D_image_xy = [] + for p in colmap_points.values(): + nids = np.array(p.image_ids, dtype=np.int64) + nxy_ids = np.array(p.point2D_idxs, dtype=np.int32) + if self.config.max_2D_matches_per_3D_point != -1: + # Randomly sample 2D matches + idxs = np.argsort(p.error)[: self.config.max_2D_matches_per_3D_point] + nids = nids[idxs] + nxy_ids = nxy_ids[idxs] + nxy = [im_id_to_image[im_id].xys[pt_idx] for im_id, pt_idx in zip(nids, nxy_ids)] + nxy = torch.from_numpy(np.stack(nxy).astype(np.float32)) + nids = torch.from_numpy(nids) + assert len(nids.shape) == 1 + assert len(nxy.shape) == 2 + points3D_image_ids.append( + torch.cat((nids, torch.full((max_num_points - len(nids),), -1, dtype=torch.int64))) + ) + points3D_image_xy.append( + torch.cat((nxy, torch.full((max_num_points - len(nxy), nxy.shape[-1]), 0, dtype=torch.float32))) + / downscale_factor + ) + out["points3D_image_ids"] = torch.stack(points3D_image_ids, dim=0) + out["points3D_image_xy"] = torch.stack(points3D_image_xy, dim=0) + return out + def _setup_downscale_factor( self, image_filenames: List[Path], mask_filenames: List[Path], depth_filenames: List[Path] ): From 570ccf3c0a298160caad72745d13db0c23d45a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Mon, 11 Sep 2023 18:00:44 +0200 Subject: [PATCH 06/19] Rename COLMAP 3D point features (#2417) --- nerfstudio/data/dataparsers/colmap_dataparser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py index 046f6d5d3..2c15258d6 100644 --- a/nerfstudio/data/dataparsers/colmap_dataparser.py +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -379,7 +379,7 @@ class ColmapDataParser(DataParser): "points3D_xyz": points3D, "points3D_rgb": points3D_rgb, "points3D_error": torch.from_numpy(np.array([p.error for p in colmap_points.values()], dtype=np.float32)), - "points3D_num_points": points3D_num_points, + "points3D_num_points2D": points3D_num_points, } if self.config.max_2D_matches_per_3D_point != 0: if (colmap_path / "images.txt").exists(): @@ -415,7 +415,7 @@ class ColmapDataParser(DataParser): / downscale_factor ) out["points3D_image_ids"] = torch.stack(points3D_image_ids, dim=0) - out["points3D_image_xy"] = torch.stack(points3D_image_xy, dim=0) + out["points3D_points2D_xy"] = torch.stack(points3D_image_xy, dim=0) return out def _setup_downscale_factor( From cc98fb6bd4eb5d14a2a75bfa3dc46a6bc0d2666e Mon Sep 17 00:00:00 2001 From: Maxim Bonnaerens Date: Mon, 11 Sep 2023 20:07:52 +0200 Subject: [PATCH 07/19] Fix benchmarking: apply get_background_color in renderer and set profiler to none (#2397) * apply get_background_color and set profiler to none * Apply override first --------- Co-authored-by: Brent Yi --- nerfstudio/model_components/renderers.py | 17 +++++++---------- .../benchmarking/launch_train_blender.sh | 6 +++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 20a506b1e..1fde0d693 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -102,21 +102,18 @@ class RGBRenderer(nn.Module): comp_rgb = torch.sum(weights * rgb, dim=-2) accumulated_weight = torch.sum(weights, dim=-2) if BACKGROUND_COLOR_OVERRIDE is not None: - # This case must be before the others or the override is not properly applied - background_color = cls.get_background_color( - BACKGROUND_COLOR_OVERRIDE, shape=comp_rgb.shape, device=comp_rgb.device - ) - elif background_color == "random": + background_color = BACKGROUND_COLOR_OVERRIDE + if background_color == "random": # If background color is random, the predicted color is returned without blending, # as if the background color was black. return comp_rgb - elif background_color == "last_sample": # Note, this is only supported for non-packed samples. background_color = rgb[..., -1, :] + background_color = cls.get_background_color(background_color, shape=comp_rgb.shape, device=comp_rgb.device) assert isinstance(background_color, torch.Tensor) - comp_rgb = comp_rgb + background_color.to(comp_rgb.device) * (1.0 - accumulated_weight) + comp_rgb = comp_rgb + background_color * (1.0 - accumulated_weight) return comp_rgb @classmethod @@ -124,13 +121,13 @@ class RGBRenderer(nn.Module): cls, background_color: BackgroundColor, shape: Tuple[int, ...], device: torch.device ) -> Union[Float[Tensor, "3"], Float[Tensor, "*bs 3"]]: """Returns the RGB background color for a specified background color. - Note: This function CANNOT be called for background_color being either "last_sample" or "random". Args: - rgb: RGB for each sample. - background_color: The background color specification. + background_color: The background color specification. If a string is provided, it must be a valid color name. + shape: Shape of the output tensor. + device: Device on which to create the tensor. Returns: Background color as RGB. diff --git a/nerfstudio/scripts/benchmarking/launch_train_blender.sh b/nerfstudio/scripts/benchmarking/launch_train_blender.sh index cbdd7d754..7b9db727d 100644 --- a/nerfstudio/scripts/benchmarking/launch_train_blender.sh +++ b/nerfstudio/scripts/benchmarking/launch_train_blender.sh @@ -17,7 +17,7 @@ while getopts "m:v:s" opt; do m ) method_name="$OPTARG" ;; v ) vis="$OPTARG" ;; s ) single=true ;; - ? ) helpFunction ;; + ? ) helpFunction ;; esac done @@ -83,12 +83,12 @@ for dataset in "${DATASETS[@]}"; do --steps-per-save=1000 \ --max-num-iterations=16500 \ --logging.local-writer.enable=False \ - --logging.enable-profiler=False \ + --logging.profiler="none" \ --vis "${vis}" \ --timestamp "$timestamp" \ ${dataparser} & GPU_PID[$idx]=$! echo "Launched ${method_name} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}" - + # update gpu ((idx=(idx+1)%len)) done From e9330d515d6c7c81905e0b61ca3bd1e018d7ecdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Wed, 13 Sep 2023 19:55:24 +0200 Subject: [PATCH 08/19] Colmap parser: fix downscale for complex paths (#2425) --- .../data/dataparsers/colmap_dataparser.py | 89 ++++++++++++++----- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py index 2c15258d6..851a30367 100644 --- a/nerfstudio/data/dataparsers/colmap_dataparser.py +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -19,6 +19,7 @@ import math import sys from dataclasses import dataclass, field from pathlib import Path +from functools import partial from typing import List, Literal, Optional, Type import numpy as np @@ -32,8 +33,8 @@ from nerfstudio.data.dataparsers.base_dataparser import DataParser, DataParserCo from nerfstudio.data.scene_box import SceneBox from nerfstudio.data.utils import colmap_parsing_utils as colmap_utils from nerfstudio.process_data.colmap_utils import parse_colmap_camera_params -from nerfstudio.process_data.process_data_utils import downscale_images -from nerfstudio.utils.rich_utils import CONSOLE +from nerfstudio.utils.scripts import run_command +from nerfstudio.utils.rich_utils import CONSOLE, status MAX_AUTO_RESOLUTION = 1600 @@ -139,15 +140,19 @@ class ColmapDataParser(DataParser): c2w[2, :] *= -1 frame = { - "file_path": (self.config.images_path / im_data.name).as_posix(), + "file_path": (self.config.data / self.config.images_path / im_data.name).as_posix(), "transform_matrix": c2w, "colmap_im_id": im_id, } frame.update(cameras[im_data.camera_id]) if self.config.masks_path is not None: - frame["mask_path"] = ((self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),) + frame["mask_path"] = ( + (self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix(), + ) if self.config.depths_path is not None: - frame["depth_path"] = ((self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),) + frame["depth_path"] = ( + (self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix(), + ) frames.append(frame) if camera_model is not None: assert camera_model == frame["camera_model"], "Multiple camera models are not supported" @@ -175,7 +180,7 @@ class ColmapDataParser(DataParser): with (self.config.data / f"{split}_list.txt").open("r", encoding="utf8") as f: filenames = f.read().splitlines() # Validate split first - split_filenames = set(self.config.images_path / x for x in filenames) + split_filenames = set(self.config.data / self.config.images_path / x for x in filenames) unmatched_filenames = split_filenames.difference(image_filenames) if unmatched_filenames: raise RuntimeError( @@ -418,6 +423,25 @@ class ColmapDataParser(DataParser): out["points3D_points2D_xy"] = torch.stack(points3D_image_xy, dim=0) return out + def _downscale_images(self, paths, get_fname, downscale_factor: int, nearest_neighbor: bool = False): + with status(msg="[bold yellow]Downscaling images...", spinner="growVertical"): + assert downscale_factor > 1 + assert isinstance(downscale_factor, int) + # Using %05d ffmpeg commands appears to be unreliable (skips images). + for path in paths: + nn_flag = "" if not nearest_neighbor else ":flags=neighbor" + path_out = get_fname(path) + path_out.parent.mkdir(parents=True, exist_ok=True) + ffmpeg_cmd = [ + f'ffmpeg -y -noautorotate -i "{path}" ', + f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor}{nn_flag} ", + f'"{path_out}"', + ] + ffmpeg_cmd = " ".join(ffmpeg_cmd) + run_command(ffmpeg_cmd) + + CONSOLE.log("[bold green]:tada: Done downscaling images.") + def _setup_downscale_factor( self, image_filenames: List[Path], mask_filenames: List[Path], depth_filenames: List[Path] ): @@ -425,17 +449,16 @@ class ColmapDataParser(DataParser): Setup the downscale factor for the dataset. This is used to downscale the images and cameras. """ - def get_fname(filepath: Path) -> Path: + def get_fname(parent: Path, filepath: Path) -> Path: """Returns transformed file name when downscale factor is applied""" - parts = list(filepath.parts) - parts[-2] += f"_{self._downscale_factor}" - filepath = Path(*parts) - return self.config.data / filepath + rel_part = filepath.relative_to(parent) + base_part = parent.parent / (str(parent.name) + f"_{self._downscale_factor}") + return base_part / rel_part filepath = next(iter(image_filenames)) if self._downscale_factor is None: if self.config.downscale_factor is None: - test_img = Image.open(self.config.data / filepath) + test_img = Image.open(filepath) h, w = test_img.size max_res = max(h, w) df = 0 @@ -448,7 +471,9 @@ class ColmapDataParser(DataParser): CONSOLE.log(f"Using image downscale factor of {self._downscale_factor}") else: self._downscale_factor = self.config.downscale_factor - if self._downscale_factor > 1 and not all(get_fname(fp).parent.exists() for fp in image_filenames): + if self._downscale_factor > 1 and not all( + get_fname(self.config.data / self.config.images_path, fp).parent.exists() for fp in image_filenames + ): # Downscaled images not found # Ask if user wants to downscale the images automatically here CONSOLE.print( @@ -456,23 +481,39 @@ class ColmapDataParser(DataParser): ) if Confirm.ask("\nWould you like to downscale the images now?", default=False, console=CONSOLE): # Install the method - image_dir = self.config.data / image_filenames[0].parent - num_downscales = int(math.log2(self._downscale_factor)) - assert 2**num_downscales == self._downscale_factor, "Downscale factor must be a power of 2" - downscale_images(image_dir, num_downscales, folder_name=image_dir.name, nearest_neighbor=False) + self._downscale_images( + image_filenames, + partial(get_fname, self.config.data / self.config.images_path), + self._downscale_factor, + nearest_neighbor=False, + ) if len(mask_filenames) > 0: - mask_dir = mask_filenames[0].parent - downscale_images(mask_dir, num_downscales, folder_name=mask_dir.name, nearest_neighbor=True) + assert self.config.masks_path is not None + self._downscale_images( + mask_filenames, + partial(get_fname, self.config.data / self.config.masks_path), + self._downscale_factor, + nearest_neighbor=True, + ) if len(depth_filenames) > 0: - depth_dir = depth_filenames[0].parent - downscale_images(depth_dir, num_downscales, folder_name=depth_dir.name, nearest_neighbor=False) + assert self.config.depths_path is not None + self._downscale_images( + depth_filenames, + partial(get_fname, self.config.data / self.config.depths_path), + self._downscale_factor, + nearest_neighbor=True, + ) else: sys.exit(1) # Return transformed filenames if self._downscale_factor > 1: - image_filenames = [get_fname(fp) for fp in image_filenames] - mask_filenames = [get_fname(fp) for fp in mask_filenames] - depth_filenames = [get_fname(fp) for fp in depth_filenames] + image_filenames = [get_fname(self.config.data / self.config.images_path, fp) for fp in image_filenames] + if len(mask_filenames) > 0: + assert self.config.masks_path is not None + mask_filenames = [get_fname(self.config.data / self.config.masks_path, fp) for fp in mask_filenames] + if len(depth_filenames) > 0: + assert self.config.depths_path is not None + depth_filenames = [get_fname(self.config.data / self.config.depths_path, fp) for fp in depth_filenames] assert isinstance(self._downscale_factor, int) return image_filenames, mask_filenames, depth_filenames, self._downscale_factor From 805e6e702d328bf6fc2bf9721cbb0a4eedf03a35 Mon Sep 17 00:00:00 2001 From: anc2001 <71151378+anc2001@users.noreply.github.com> Date: Thu, 14 Sep 2023 09:11:15 -0700 Subject: [PATCH 09/19] fix pixel sampling with masks issue when data is list (#2369) Co-authored-by: Brent Yi --- nerfstudio/data/pixel_samplers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index b852c31b1..463d08294 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -215,7 +215,7 @@ class PixelSampler: num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch indices = self.sample_method( - num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i], device=device + num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i].unsqueeze(0), device=device ) indices[:, 0] = i all_indices.append(indices) From 7269a7ee1639cdb6d287988a648f53075ca8e1ea Mon Sep 17 00:00:00 2001 From: Sid Mehta Date: Fri, 15 Sep 2023 09:21:09 -0700 Subject: [PATCH 10/19] Add Comet Logging (#2431) * initial commit * fixed naming * added comet_ml as depedency * added comet to docs * Update custom_gui.md * Update models.md * Update benchmarking.md * added Comet to ReadMe * added decorator * fixed link and added comet examples * fixed link and added comet example * fixed format * fixed type ignore error * fixed readme typo * fixed typo --- README.md | 2 +- .../debugging_tools/benchmarking.md | 2 +- docs/developer_guides/pipelines/models.md | 2 +- docs/developer_guides/viewer/custom_gui.md | 2 +- docs/quickstart/first_nerf.md | 4 +- nerfstudio/configs/experiment_config.py | 15 ++++--- nerfstudio/engine/trainer.py | 11 ++--- nerfstudio/utils/decorators.py | 2 +- nerfstudio/utils/writer.py | 42 ++++++++++++++++--- pyproject.toml | 1 + 10 files changed, 57 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index aa5825a23..b6f05f9be 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ ns-train nerfacto --help ### Tensorboard / WandB / Viewer -We support three different methods to track training progress, using the viewer, [tensorboard](https://www.tensorflow.org/tensorboard), and [Weights and Biases](https://wandb.ai/site). You can specify which visualizer to use by appending `--vis {viewer, tensorboard, wandb, viewer+wandb, viewer+tensorboard}` to the training command. Simultaneously utilizing the viewer alongside wandb or tensorboard may cause stuttering issues during evaluation steps. The viewer only works for methods that are fast (ie. nerfacto, instant-ngp), for slower methods like NeRF, use the other loggers. +We support four different methods to track training progress, using the viewer[tensorboard](https://www.tensorflow.org/tensorboard), [Weights and Biases](https://wandb.ai/site), and ,[Comet](https://comet.com/?utm_source=nerf&utm_medium=referral&utm_content=github). You can specify which visualizer to use by appending `--vis {viewer, tensorboard, wandb, comet viewer+wandb, viewer+tensorboard, viewer+comet}` to the training command. Simultaneously utilizing the viewer alongside wandb or tensorboard may cause stuttering issues during evaluation steps. The viewer only works for methods that are fast (ie. nerfacto, instant-ngp), for slower methods like NeRF, use the other loggers. # Learn More diff --git a/docs/developer_guides/debugging_tools/benchmarking.md b/docs/developer_guides/debugging_tools/benchmarking.md index b38c09277..bf97db4b5 100644 --- a/docs/developer_guides/debugging_tools/benchmarking.md +++ b/docs/developer_guides/debugging_tools/benchmarking.md @@ -16,7 +16,7 @@ Simply replace the arguments in brackets with the correct arguments. - `-m {METHOD_NAME}`: Name of the method you want to benchmark (e.g. `nerfacto`, `mipnerf`). - `-s`: Launch a single job per GPU. -- `-v {VIS}`: Use another visualization than wandb, which is the default. Only other option is tensorboard. +- `-v {VIS}`: Use another visualization than wandb, which is the default. Other options are comet & tensorboard. - `{GPU_LIST}`: (optional) Specify the list of gpus you want to use on your machine space separated. for instance, if you want to use GPU's 0-3, you will need to pass in `0 1 2 3`. If left empty, the script will automatically find available GPU's and distribute training jobs on the available GPUs. :::{admonition} Tip diff --git a/docs/developer_guides/pipelines/models.md b/docs/developer_guides/pipelines/models.md index 8afa76d3a..4fb1cb3f2 100644 --- a/docs/developer_guides/pipelines/models.md +++ b/docs/developer_guides/pipelines/models.md @@ -55,7 +55,7 @@ class Model: """Process a RayBundle object and return RayOutputs describing quanties for each ray.""" def get_metrics_dict(self, outputs, batch): - """Returns metrics dictionary which will be plotted with wandb or tensorboard.""" + """Returns metrics dictionary which will be plotted with comet, wandb or tensorboard.""" def get_loss_dict(self, outputs, batch, metrics_dict=None): """Returns a dictionary of losses to be summed which will be your loss.""" diff --git a/docs/developer_guides/viewer/custom_gui.md b/docs/developer_guides/viewer/custom_gui.md index e11381d02..2ae41fa6a 100644 --- a/docs/developer_guides/viewer/custom_gui.md +++ b/docs/developer_guides/viewer/custom_gui.md @@ -59,7 +59,7 @@ class MyModel(Model): **Writing to the element** -You can write to a viewer element in Python, which provides a convenient way to track values in your code without the need for wandb/tensorboard or relying on `print` statements. +You can write to a viewer element in Python, which provides a convenient way to track values in your code without the need for comet/wandb/tensorboard or relying on `print` statements. ```python self.custom_value.value = x diff --git a/docs/quickstart/first_nerf.md b/docs/quickstart/first_nerf.md index a88183d9e..137bd0b23 100644 --- a/docs/quickstart/first_nerf.md +++ b/docs/quickstart/first_nerf.md @@ -96,9 +96,9 @@ ns-train nerfacto nerfstudio-data --help Each script will have some other minor quirks (like the training script dataparser subcommand needing to come after the model subcommand), read up on them [here](../reference/cli/index.md). -## Tensorboard / WandB / Viewer +## Comet / Tensorboard / WandB / Viewer -We support three different methods to track training progress, using the viewer, [tensorboard](https://www.tensorflow.org/tensorboard), and [Weights and Biases](https://wandb.ai/site). You can specify which visualizer to use by appending `--vis {viewer, tensorboard, wandb, viewer+wandb, viewer+tensorboard}` to the training command. Simultaneously utilizing the viewer alongside wandb or tensorboard may cause stuttering issues during evaluation steps. The viewer only works for methods that are fast (ie. nerfacto, instant-ngp), for slower methods like NeRF, use the other loggers. +We support four different methods to track training progress, using the viewer [tensorboard](https://www.tensorflow.org/tensorboard), [Weights and Biases](https://wandb.ai/site), and [Comet](https://comet.com/?utm_source=nerf&utm_medium=referral&utm_content=nerf_docs). You can specify which visualizer to use by appending `--vis {viewer, tensorboard, wandb, viewer+wandb, viewer+tensorboard, viewer+comet}` to the training command. Simultaneously utilizing the viewer alongside wandb or tensorboard may cause stuttering issues during evaluation steps. The viewer only works for methods that are fast (ie. nerfacto, instant-ngp), for slower methods like NeRF, use the other loggers. ## Evaluating Runs diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py index 1fa3598f6..0da8893ed 100644 --- a/nerfstudio/configs/experiment_config.py +++ b/nerfstudio/configs/experiment_config.py @@ -22,13 +22,7 @@ from pathlib import Path from typing import Any, Dict, Literal, Optional import yaml - -from nerfstudio.configs.base_config import ( - InstantiateConfig, - LoggingConfig, - MachineConfig, - ViewerConfig, -) +from nerfstudio.configs.base_config import InstantiateConfig, LoggingConfig, MachineConfig, ViewerConfig from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.engine.optimizers import OptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig @@ -68,7 +62,9 @@ class ExperimentConfig(InstantiateConfig): } ) """Dictionary of optimizer groups and their schedulers""" - vis: Literal["viewer", "wandb", "tensorboard", "viewer+wandb", "viewer+tensorboard", "viewer_beta"] = "wandb" + vis: Literal[ + "viewer", "wandb", "tensorboard", "comet", "viewer+wandb", "viewer+tensorboard", "viewer+comet", "viewer_beta" + ] = "wandb" """Which visualizer to use.""" data: Optional[Path] = None """Alias for --pipeline.datamanager.data""" @@ -95,6 +91,9 @@ class ExperimentConfig(InstantiateConfig): """Checks if tensorboard is enabled.""" return ("tensorboard" == self.vis) | ("viewer+tensorboard" == self.vis) + def is_comet_enabled(self) -> bool: + return ("comet" == self.vis) | ("viewer+comet" == self.vis) + def set_timestamp(self) -> None: """Dynamically set the experiment timestamp""" if self.timestamp == "{timestamp}": diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 94256809c..c8ae50043 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -27,11 +27,6 @@ from threading import Lock from typing import Dict, List, Literal, Optional, Tuple, Type, cast import torch -from rich import box, style -from rich.panel import Panel -from rich.table import Table -from torch.cuda.amp.grad_scaler import GradScaler - from nerfstudio.configs.experiment_config import ExperimentConfig from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation @@ -44,6 +39,10 @@ from nerfstudio.utils.rich_utils import CONSOLE from nerfstudio.utils.writer import EventName, TimeWriter from nerfstudio.viewer.server.viewer_state import ViewerState from nerfstudio.viewer_beta.viewer import Viewer as ViewerBetaState +from rich import box, style +from rich.panel import Panel +from rich.table import Table +from torch.cuda.amp.grad_scaler import GradScaler TRAIN_INTERATION_OUTPUT = Tuple[torch.Tensor, Dict[str, torch.Tensor], Dict[str, torch.Tensor]] TORCH_DEVICE = str @@ -197,6 +196,7 @@ class Trainer: writer.setup_event_writer( self.config.is_wandb_enabled(), self.config.is_tensorboard_enabled(), + self.config.is_comet_enabled(), log_dir=writer_log_path, experiment_name=self.config.experiment_name, project_name=self.config.project_name, @@ -326,6 +326,7 @@ class Trainer: (self.config.is_viewer_enabled() or self.config.is_viewer_beta_enabled()) and not self.config.is_tensorboard_enabled() and not self.config.is_wandb_enabled() + and not self.config.is_comet_enabled() ): string: str = ( "[NOTE] Not running eval iterations since only viewer is enabled.\n" diff --git a/nerfstudio/utils/decorators.py b/nerfstudio/utils/decorators.py index 06f28f60a..d6fd3f417 100644 --- a/nerfstudio/utils/decorators.py +++ b/nerfstudio/utils/decorators.py @@ -66,7 +66,7 @@ def check_eval_enabled(func: Callable) -> Callable: def wrapper(self, *args, **kwargs): ret = None - if self.config.is_wandb_enabled() or self.config.is_tensorboard_enabled(): + if self.config.is_wandb_enabled() or self.config.is_tensorboard_enabled() or self.config.is_comet_enabled(): ret = func(self, *args, **kwargs) return ret diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py index 8d3b07b1a..59e6c5a71 100644 --- a/nerfstudio/utils/writer.py +++ b/nerfstudio/utils/writer.py @@ -24,16 +24,16 @@ from pathlib import Path from time import time from typing import Any, Dict, List, Optional, Union +import comet_ml import torch import wandb from jaxtyping import Float -from torch import Tensor -from torch.utils.tensorboard import SummaryWriter - from nerfstudio.configs import base_config as cfg from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.printing import human_format from nerfstudio.utils.rich_utils import CONSOLE +from torch import Tensor +from torch.utils.tensorboard import SummaryWriter def to8b(x): @@ -150,7 +150,7 @@ def put_time(name: str, duration: float, step: int, avg_over_steps: bool = True, put_scalar(name, duration, step) if update_eta: - ## NOTE: eta should be called with avg train iteration time + # NOTE: eta should be called with avg train iteration time remain_iter = GLOBAL_BUFFER["max_iter"] - step remain_time = remain_iter * GLOBAL_BUFFER["events"][name]["avg"] put_scalar(EventName.ETA, remain_time, step) @@ -185,7 +185,7 @@ def setup_local_writer(config: cfg.LoggingConfig, max_iter: int, banner_messages else: CONSOLE.log("disabled local writer") - ## configure all the global buffer basic information + # configure all the global buffer basic information GLOBAL_BUFFER["max_iter"] = max_iter GLOBAL_BUFFER["max_buffer_size"] = config.max_buffer_size GLOBAL_BUFFER["steps_per_log"] = config.steps_per_log @@ -203,6 +203,7 @@ def is_initialized(): def setup_event_writer( is_wandb_enabled: bool, is_tensorboard_enabled: bool, + is_comet_enabled: bool, log_dir: Path, experiment_name: str, project_name: str = "nerfstudio-project", @@ -214,6 +215,11 @@ def setup_event_writer( banner_messages: list of messages to always display at bottom of screen """ using_event_writer = False + + if is_comet_enabled: + curr_writer = CometWriter(log_dir=log_dir, experiment_name=experiment_name, project_name=project_name) + EVENT_WRITERS.append(curr_writer) + using_event_writer = True if is_wandb_enabled: curr_writer = WandbWriter(log_dir=log_dir, experiment_name=experiment_name, project_name=project_name) EVENT_WRITERS.append(curr_writer) @@ -225,7 +231,7 @@ def setup_event_writer( if using_event_writer: string = f"logging events to: {log_dir}" else: - string = "Disabled tensorboard/wandb event writers" + string = "Disabled comet/tensorboard/wandb event writers" CONSOLE.print(f"[bold yellow]{string}") @@ -346,6 +352,30 @@ class TensorboardWriter(Writer): self.tb_writer.add_text("config", str(config_dict)) +@decorate_all([check_main_thread]) +class CometWriter(Writer): + """Comet_ML Writer Class""" + + def __init__(self, log_dir: Path, experiment_name: str, project_name: str = "nerfstudio-project"): + self.experiment = comet_ml.Experiment(project_name=project_name) + if experiment_name != "unnamed": + self.experiment.set_name(experiment_name) + + def write_image(self, name: str, image: Float[Tensor, "H W C"], step: int) -> None: + self.experiment.log_image(image, name, step=step) + + def write_scalar(self, name: str, scalar: Union[float, torch.Tensor], step: int) -> None: + self.experiment.log_metric(name, scalar, step) + + def write_config(self, name: str, config_dict: Dict[str, Any], step: int): + """Function that writes out the config to Comet + + Args: + config: config dictionary to write out + """ + self.experiment.log_parameters(config_dict, step=step) + + def _cursorup(x: int): """utility tool to move the cursor up on the terminal diff --git a/pyproject.toml b/pyproject.toml index 1347f954b..add21cf6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ classifiers = [ dependencies = [ "appdirs>=1.4", "av>=9.2.0", + "comet_ml>=3.33.8", "cryptography>=38", "tyro>=0.5.3", "gdown>=4.6.0", From ef2fd3dbe1d5b6f35781716b747ea142fe21f5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Fri, 15 Sep 2023 23:18:23 +0200 Subject: [PATCH 11/19] Fix import error when tcnn installed but CUDA runtime not available (#2429) --- Dockerfile | 6 ++- nerfstudio/field_components/encodings.py | 8 +--- nerfstudio/field_components/mlp.py | 8 +--- nerfstudio/fields/sdf_field.py | 7 +-- nerfstudio/utils/external.py | 58 ++++++++++++++++++++++++ tests/field_components/test_fields.py | 7 ++- 6 files changed, 68 insertions(+), 26 deletions(-) create mode 100644 nerfstudio/utils/external.py diff --git a/Dockerfile b/Dockerfile index b48b0dd87..d73c48734 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,6 +168,8 @@ RUN cd nerfstudio && \ # Change working directory WORKDIR /workspace -# Install nerfstudio cli auto completion and enter shell if no command was provided. -CMD ns-install-cli --mode install && /bin/bash +# Install nerfstudio cli auto completion +RUN ns-install-cli --mode install +# Bash as default entrypoint. +CMD /bin/bash -l diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py index 7145ae599..437ee805f 100644 --- a/nerfstudio/field_components/encodings.py +++ b/nerfstudio/field_components/encodings.py @@ -29,13 +29,7 @@ from torch import Tensor, nn from nerfstudio.field_components.base_field_component import FieldComponent from nerfstudio.utils.math import components_from_spherical_harmonics, expected_sin from nerfstudio.utils.printing import print_tcnn_speed_warning - -try: - import tinycudann as tcnn - - TCNN_EXISTS = True -except ModuleNotFoundError: - TCNN_EXISTS = False +from nerfstudio.utils.external import tcnn, TCNN_EXISTS class Encoding(FieldComponent): diff --git a/nerfstudio/field_components/mlp.py b/nerfstudio/field_components/mlp.py index 52eabd29d..2585a49e4 100644 --- a/nerfstudio/field_components/mlp.py +++ b/nerfstudio/field_components/mlp.py @@ -25,13 +25,7 @@ from nerfstudio.field_components.base_field_component import FieldComponent from nerfstudio.utils.printing import print_tcnn_speed_warning from nerfstudio.utils.rich_utils import CONSOLE - -try: - import tinycudann as tcnn - - TCNN_EXISTS = True -except ModuleNotFoundError: - TCNN_EXISTS = False +from nerfstudio.utils.external import TCNN_EXISTS, tcnn def activation_to_tcnn_string(activation: Union[nn.Module, None]) -> str: diff --git a/nerfstudio/fields/sdf_field.py b/nerfstudio/fields/sdf_field.py index 2f730d2c5..ed936d4b2 100644 --- a/nerfstudio/fields/sdf_field.py +++ b/nerfstudio/fields/sdf_field.py @@ -33,12 +33,7 @@ from nerfstudio.field_components.encodings import NeRFEncoding from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SpatialDistortion from nerfstudio.fields.base_field import Field, FieldConfig - -try: - import tinycudann as tcnn -except ModuleNotFoundError: - # tinycudann module doesn't exist - pass +from nerfstudio.utils.external import tcnn class LearnedVariance(nn.Module): diff --git a/nerfstudio/utils/external.py b/nerfstudio/utils/external.py new file mode 100644 index 000000000..f14219bd5 --- /dev/null +++ b/nerfstudio/utils/external.py @@ -0,0 +1,58 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + + +class _LazyError: + def __init__(self, data): + self.__data = data # pylint: disable=unused-private-member + + class LazyErrorObj: + def __init__(self, data): + self.__data = data # pylint: disable=unused-private-member + + def __call__(self, *args, **kwds): + name, exc = object.__getattribute__(self, "__data") + raise RuntimeError(f"Could not load package {name}.") from exc + + def __getattr__(self, __name: str): + name, exc = object.__getattribute__(self, "__data") + raise RuntimeError(f"Could not load package {name}") from exc + + def __getattr__(self, __name: str): + return _LazyError.LazyErrorObj(object.__getattribute__(self, "__data")) + + +TCNN_EXISTS = False +tcnn_import_exception = None +tcnn = None +try: + import tinycudann + + tcnn = tinycudann + del tinycudann + TCNN_EXISTS = True +except ModuleNotFoundError as _exp: + tcnn_import_exception = _exp +except ImportError as _exp: + tcnn_import_exception = _exp +except EnvironmentError as _exp: + if "Unknown compute capability" not in _exp.args[0]: + raise _exp + print("Could not load tinycudann: " + str(_exp), file=sys.stderr) + tcnn_import_exception = _exp + +if tcnn_import_exception is not None: + tcnn = _LazyError(tcnn_import_exception) diff --git a/tests/field_components/test_fields.py b/tests/field_components/test_fields.py index 74d7d6c3a..fd5332776 100644 --- a/tests/field_components/test_fields.py +++ b/tests/field_components/test_fields.py @@ -5,15 +5,14 @@ import torch from nerfstudio.cameras.rays import Frustums, RaySamples from nerfstudio.fields.nerfacto_field import NerfactoField +from nerfstudio.utils.external import TCNN_EXISTS, tcnn_import_exception def test_nerfacto_field(): """Test the Nerfacto field""" - try: - import tinycudann as tcnn # noqa: F401 - except ModuleNotFoundError as e: + if not TCNN_EXISTS: # tinycudann module doesn't exist - print(e) + print(tcnn_import_exception) return device = torch.device("cuda" if torch.cuda.is_available() else "cpu") aabb_scale = 1.0 From b0e700137bd8bde21fbc7867f433dd36b2905e1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=95=E3=83=A9=E3=83=8F=E3=83=86=E3=82=A3=E3=80=80?= =?UTF-8?q?=E4=BB=81?= Date: Tue, 19 Sep 2023 13:13:48 +0900 Subject: [PATCH 12/19] Allow the option for timestamp to be generated for `ns-train --load-config` (#2442) * Fix timestamp for ns-train --load-config * Remove whitespace --- nerfstudio/scripts/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nerfstudio/scripts/train.py b/nerfstudio/scripts/train.py index 9ca18c71a..fde0a5c37 100644 --- a/nerfstudio/scripts/train.py +++ b/nerfstudio/scripts/train.py @@ -226,7 +226,6 @@ def launch( def main(config: TrainerConfig) -> None: """Main function.""" - config.set_timestamp() if config.data: CONSOLE.log("Using --data alias for --data.pipeline.datamanager.data") config.pipeline.datamanager.data = config.data @@ -239,6 +238,8 @@ def main(config: TrainerConfig) -> None: CONSOLE.log(f"Loading pre-set config from: {config.load_config}") config = yaml.load(config.load_config.read_text(), Loader=yaml.Loader) + config.set_timestamp() + # print and save config config.print_to_terminal() config.save_config() From 3e1d4af8217134fc34b42690345c5847147bc732 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 19 Sep 2023 16:22:55 -0700 Subject: [PATCH 13/19] Add share link support for beta viewer (#2445) * Add share link support for beta viewer * More consistent CLI experience * fix viewer jitter by adding atomic update on camera pose * change to make_share_url * fix * remove incorrect assert --------- Co-authored-by: Justin Kerr --- nerfstudio/configs/base_config.py | 2 ++ nerfstudio/configs/experiment_config.py | 10 ++++++++-- nerfstudio/engine/trainer.py | 1 + nerfstudio/scripts/viewer/run_viewer.py | 8 ++++++-- nerfstudio/viewer_beta/viewer.py | 25 ++++++++++++++----------- pyproject.toml | 2 +- 6 files changed, 32 insertions(+), 16 deletions(-) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 315659acb..fc906fae9 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -146,3 +146,5 @@ class ViewerConfig(PrintableConfig): """Image format viewer should use; jpeg is lossy compression, while png is lossless.""" jpeg_quality: int = 90 """Quality tradeoff to use for jpeg compression.""" + make_share_url: bool = False + """Viewer beta feature: print a shareable URL. `vis` must be set to viewer_beta; this flag is otherwise ignored.""" diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py index 0da8893ed..42cc4399e 100644 --- a/nerfstudio/configs/experiment_config.py +++ b/nerfstudio/configs/experiment_config.py @@ -22,7 +22,13 @@ from pathlib import Path from typing import Any, Dict, Literal, Optional import yaml -from nerfstudio.configs.base_config import InstantiateConfig, LoggingConfig, MachineConfig, ViewerConfig + +from nerfstudio.configs.base_config import ( + InstantiateConfig, + LoggingConfig, + MachineConfig, + ViewerConfig, +) from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.engine.optimizers import OptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig @@ -73,7 +79,7 @@ class ExperimentConfig(InstantiateConfig): relative_model_dir: Path = Path("nerfstudio_models/") """Relative path to save all checkpoints.""" load_scheduler: bool = True - """Whether to load the scheduler state_dict to resume training, if exists""" + """Whether to load the scheduler state_dict to resume training, if it exists.""" def is_viewer_enabled(self) -> bool: """Checks if a viewer is enabled.""" diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index c8ae50043..04fa8e05b 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -177,6 +177,7 @@ class Trainer: pipeline=self.pipeline, trainer=self, train_lock=self.train_lock, + share=self.config.viewer.make_share_url, ) banner_messages = [f"Viewer Beta at: {self.viewer_state.viewer_url}"] self._check_viewer_warnings() diff --git a/nerfstudio/scripts/viewer/run_viewer.py b/nerfstudio/scripts/viewer/run_viewer.py index 56a712026..de210a30d 100644 --- a/nerfstudio/scripts/viewer/run_viewer.py +++ b/nerfstudio/scripts/viewer/run_viewer.py @@ -55,6 +55,8 @@ class RunViewer: """Viewer configuration""" vis: Literal["viewer", "viewer_beta"] = "viewer" """Type of viewer""" + make_share_url: bool = False + """Viewer beta feature: print a shareable URL. `vis` must be set to viewer_beta; this flag is otherwise ignored.""" def main(self) -> None: """Main function.""" @@ -66,6 +68,7 @@ class RunViewer: num_rays_per_chunk = config.viewer.num_rays_per_chunk assert self.viewer.num_rays_per_chunk == -1 config.vis = self.vis + config.viewer.make_share_url = self.make_share_url config.viewer = self.viewer.as_viewer_config() config.viewer.num_rays_per_chunk = num_rays_per_chunk @@ -103,6 +106,7 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int): log_filename=viewer_log_path, datapath=base_dir, pipeline=pipeline, + share=config.viewer.make_share_url, ) banner_messages = [f"Viewer Beta at: {viewer_state.viewer_url}"] @@ -126,11 +130,11 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int): def entrypoint(): """Entrypoint for use with pyproject scripts.""" tyro.extras.set_accent_color("bright_yellow") - tyro.cli(RunViewer).main() + tyro.cli(tyro.conf.FlagConversionOff[RunViewer]).main() if __name__ == "__main__": entrypoint() # For sphinx docs -get_parser_fn = lambda: tyro.extras.get_parser(RunViewer) # noqa +get_parser_fn = lambda: tyro.extras.get_parser(tyro.conf.FlagConversionOff[RunViewer]) # noqa diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py index 9e2314ad4..d2a53b198 100644 --- a/nerfstudio/viewer_beta/viewer.py +++ b/nerfstudio/viewer_beta/viewer.py @@ -26,6 +26,8 @@ import torchvision import viser import viser.theme import viser.transforms as vtf + +from nerfstudio.cameras.camera_optimizers import CameraOptimizer from nerfstudio.configs import base_config as cfg from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.models.base_model import Model @@ -40,8 +42,6 @@ from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStat from nerfstudio.viewer_beta.utils import CameraState, parse_object from nerfstudio.viewer_beta.viewer_elements import ViewerControl, ViewerElement -from nerfstudio.cameras.camera_optimizers import CameraOptimizer - if TYPE_CHECKING: from nerfstudio.engine.trainer import Trainer @@ -59,6 +59,7 @@ class Viewer: datapath: path to data pipeline: pipeline object to use trainer: trainer object to use + share: print a shareable URL Attributes: viewer_url: url to open viewer @@ -77,6 +78,7 @@ class Viewer: pipeline: Pipeline, trainer: Optional[Trainer] = None, train_lock: Optional[threading.Lock] = None, + share: bool = False, ): self.config = config self.trainer = trainer @@ -103,7 +105,7 @@ class Viewer: self._prev_train_state: Literal["training", "paused", "completed"] = "training" self.client: Optional[viser.ClientHandle] = None - self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port) + self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port, share=share) buttons = ( viser.theme.TitlebarButton( text="Getting Started", @@ -191,14 +193,15 @@ class Viewer: @client.camera.on_update def _(cam: viser.CameraHandle) -> None: assert self.client is not None - self.last_move_time = time.time() - R = vtf.SO3(wxyz=self.client.camera.wxyz) - R = R @ vtf.SO3.from_x_radians(np.pi) - R = torch.tensor(R.as_matrix()) - pos = torch.tensor(self.client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO - c2w = torch.concatenate([R, pos[:, None]], dim=1) - self.camera_state = CameraState(fov=self.client.camera.fov, aspect=self.client.camera.aspect, c2w=c2w) - self.render_statemachine.action(RenderAction("move", self.camera_state)) + with client.atomic(): + self.last_move_time = time.time() + R = vtf.SO3(wxyz=self.client.camera.wxyz) + R = R @ vtf.SO3.from_x_radians(np.pi) + R = torch.tensor(R.as_matrix()) + pos = torch.tensor(self.client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO + c2w = torch.concatenate([R, pos[:, None]], dim=1) + self.camera_state = CameraState(fov=self.client.camera.fov, aspect=self.client.camera.aspect, c2w=c2w) + self.render_statemachine.action(RenderAction("move", self.camera_state)) def set_camera_visibility(self, visible: bool) -> None: """Toggle the visibility of the training cameras.""" diff --git a/pyproject.toml b/pyproject.toml index add21cf6d..c64152675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "torchvision>=0.14.1", "torchmetrics[image]>=1.0.1", "typing_extensions>=4.4.0", - "viser==0.1.2", + "viser==0.1.3", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas", From e0d8f0426752dc0f97bd332fd6b84e95e8377210 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Tue, 19 Sep 2023 16:48:54 -0700 Subject: [PATCH 14/19] Render oriented crops in viewer beta (#2447) * add obb to the rendering pipeline * useless line * lint * remove todo * pyright * more pyright --- nerfstudio/engine/trainer.py | 5 +- nerfstudio/scripts/render.py | 33 +++--- nerfstudio/viewer_beta/render_panel.py | 135 +++++++++++++++++++++---- nerfstudio/viewer_beta/viewer.py | 4 +- 4 files changed, 139 insertions(+), 38 deletions(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 04fa8e05b..b4ccba9ee 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -170,10 +170,13 @@ class Trainer: ) banner_messages = [f"Viewer at: {self.viewer_state.viewer_url}"] if self.config.is_viewer_beta_enabled() and self.local_rank == 0: + datapath = self.config.data + if datapath is None: + datapath = self.base_dir self.viewer_state = ViewerBetaState( self.config.viewer, log_filename=viewer_log_path, - datapath=self.base_dir, + datapath=datapath, pipeline=self.pipeline, trainer=self, train_lock=self.train_lock, diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 5a3ca78bd..99a7cae01 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -54,7 +54,7 @@ from nerfstudio.cameras.camera_paths import ( ) from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager -from nerfstudio.data.scene_box import SceneBox +from nerfstudio.data.scene_box import OrientedBox from nerfstudio.model_components import renderers from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils import colormaps, install_checks @@ -127,12 +127,10 @@ def _render_trajectory_video( with progress: for camera_idx in progress.track(range(cameras.size), description=""): - aabb_box = None + obb_box = None if crop_data is not None: - bounding_box_min = crop_data.center - crop_data.scale / 2.0 - bounding_box_max = crop_data.center + crop_data.scale / 2.0 - aabb_box = SceneBox(torch.stack([bounding_box_min, bounding_box_max]).to(pipeline.device)) - camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx, aabb_box=aabb_box) + obb_box = crop_data.obb + camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx, obb_box=obb_box) if crop_data is not None: with renderers.background_color_override_context( @@ -289,10 +287,17 @@ class CropData: background_color: Float[Tensor, "3"] = torch.Tensor([0.0, 0.0, 0.0]) """background color""" - center: Float[Tensor, "3"] = torch.Tensor([0.0, 0.0, 0.0]) - """center of the crop""" - scale: Float[Tensor, "3"] = torch.Tensor([2.0, 2.0, 2.0]) - """scale of the crop""" + obb: OrientedBox = OrientedBox(R=torch.eye(3), T=torch.zeros(3), S=torch.ones(3) * 2) + """Oriented box representing the crop region""" + + # properties for backwards-compatibility interface + @property + def center(self): + return self.obb.T + + @property + def scale(self): + return self.obb.S def get_crop_from_json(camera_json: Dict[str, Any]) -> Optional[CropData]: @@ -305,13 +310,13 @@ def get_crop_from_json(camera_json: Dict[str, Any]) -> Optional[CropData]: """ if "crop" not in camera_json or camera_json["crop"] is None: return None - bg_color = camera_json["crop"]["crop_bg_color"] - + center = camera_json["crop"]["crop_center"] + scale = camera_json["crop"]["crop_scale"] + rot = (0.0, 0.0, 0.0) if "crop_rot" not in camera_json["crop"] else tuple(camera_json["crop"]["crop_rot"]) return CropData( background_color=torch.Tensor([bg_color["r"] / 255.0, bg_color["g"] / 255.0, bg_color["b"] / 255.0]), - center=torch.Tensor(camera_json["crop"]["crop_center"]), - scale=torch.Tensor(camera_json["crop"]["crop_scale"]), + obb=OrientedBox.from_params(center, rot, scale), ) diff --git a/nerfstudio/viewer_beta/render_panel.py b/nerfstudio/viewer_beta/render_panel.py index 02d4d6ce0..7342b0f38 100644 --- a/nerfstudio/viewer_beta/render_panel.py +++ b/nerfstudio/viewer_beta/render_panel.py @@ -13,17 +13,19 @@ # limitations under the License. from __future__ import annotations - +from pathlib import Path import colorsys import dataclasses import threading import time from typing import Dict, List, Optional, Tuple - +import datetime +from nerfstudio.viewer_beta.control_panel import ControlPanel import numpy as onp import splines import splines.quaternion import viser +import json import viser.transforms as tf @@ -256,7 +258,11 @@ class CameraPath: ) -def populate_render_tab(server: viser.ViserServer) -> None: +def populate_render_tab( + server: viser.ViserServer, config_path: Path, datapath: Path, control_panel: ControlPanel +) -> None: + from nerfstudio.viewer_beta.viewer import VISER_NERFSTUDIO_SCALE_RATIO + fov_degrees = server.add_gui_slider( "FOV", initial_value=90.0, @@ -528,6 +534,11 @@ def populate_render_tab(server: viser.ViserServer) -> None: play_button.visible = True pause_button.visible = False + # set the initial value to the current date-time string + now = datetime.datetime.now() + render_name_text = server.add_gui_text( + "Render Name", initial_value=now.strftime("%Y-%m-%d-%H-%M-%S"), hint="Name of the render" + ) render_button = server.add_gui_button( "Generate Command", color="green", @@ -537,10 +548,107 @@ def populate_render_tab(server: viser.ViserServer) -> None: @render_button.on_click def _(event: viser.GuiEvent) -> None: - """TODO: write the render JSON and show the render command.""" assert event.client is not None - with event.client.add_gui_modal("TODO") as modal: - event.client.add_gui_markdown("TODO") + num_frames = int(framerate_slider.value * duration_number.value) + json_data = {} + # json data has the properties: + # keyframes: list of keyframes with + # matrix : flattened 4x4 matrix + # fov: float in degrees + # aspect: float + # camera_type: string of camera type + # render_height: int + # render_width: int + # fps: int + # seconds: float + # is_cycle: bool + # smoothness_value: float + # camera_path: list of frames with properties + # camera_to_world: flattened 4x4 matrix + # fov: float in degrees + # aspect: float + # first populate the keyframes: + keyframes = [] + for keyframe, dummy in camera_path._keyframes.values(): + pose = tf.SE3.from_rotation_and_translation( + tf.SO3(keyframe.wxyz) @ tf.SO3.from_x_radians(onp.pi), + keyframe.position / VISER_NERFSTUDIO_SCALE_RATIO, + ) + keyframes.append( + { + "matrix": pose.as_matrix().flatten().tolist(), + "fov": onp.rad2deg(keyframe.override_fov_value) + if keyframe.override_fov_enabled + else fov_degrees.value, + "aspect": keyframe.aspect, + } + ) + json_data["keyframes"] = keyframes + json_data["camera_type"] = "perspective" + json_data["render_height"] = resolution.value[1] + json_data["render_width"] = resolution.value[0] + json_data["fps"] = framerate_slider.value + json_data["seconds"] = duration_number.value + json_data["is_cycle"] = loop.value + json_data["smoothness_value"] = smoothness.value + # now populate the camera path: + camera_path_list = [] + for i in range(num_frames): + maybe_pose_and_fov = camera_path.interpolate_pose_and_fov(i / num_frames) + if maybe_pose_and_fov is None: + return + pose, fov = maybe_pose_and_fov + # rotate the axis of the camera 180 about x axis + pose = tf.SE3.from_rotation_and_translation( + pose.rotation() @ tf.SO3.from_x_radians(onp.pi), + pose.translation() / VISER_NERFSTUDIO_SCALE_RATIO, + ) + camera_path_list.append( + { + "camera_to_world": pose.as_matrix().flatten().tolist(), + "fov": onp.rad2deg(fov), + "aspect": resolution.value[0] / resolution.value[1], + } + ) + json_data["camera_path"] = camera_path_list + # finally add crop data if crop is enabled + if control_panel.crop_viewport: + obb = control_panel.crop_obb + rpy = tf.SO3.from_matrix(obb.R.numpy()).as_rpy_radians() + color = control_panel.background_color + json_data["crop"] = { + "crop_center": obb.T.tolist(), + "crop_scale": obb.S.tolist(), + "crop_rot": [rpy.roll, rpy.pitch, rpy.yaw], + "crop_bg_color": {"r": color[0], "g": color[1], "b": color[2]}, + } + + # now write the json file + json_outfile = datapath / "camera_paths" / f"{render_name_text.value}.json" + with open(json_outfile.absolute(), "w") as outfile: + json.dump(json_data, outfile) + # now show the command + with event.client.add_gui_modal("Render Command") as modal: + dataname = datapath.name + command = " ".join( + [ + "ns-render camera-path", + f"--load-config {config_path}", + f"--camera-path-filename {json_outfile.absolute()}", + f"--output-path renders/{dataname}/{render_name_text.value}.mp4", + ] + ) + event.client.add_gui_markdown( + "\n".join( + [ + "To render the trajectory, run the following from the command line:", + "", + "```", + command, + "```", + ] + ) + ) close_button = event.client.add_gui_button("Close") @close_button.on_click @@ -551,18 +659,3 @@ def populate_render_tab(server: viser.ViserServer) -> None: camera_path.default_fov = fov_degrees.value / 180.0 * onp.pi transform_controls: List[viser.SceneNodeHandle] = [] - - -def main() -> None: - """Launch a GUI with just the render panel, for development purposes.""" - server = viser.ViserServer() - server.configure_theme(dark_mode=True, control_layout="collapsible") - server.world_axes.visible = True - - populate_render_tab(server) - while True: - time.sleep(10.0) - - -if __name__ == "__main__": - main() diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py index d2a53b198..6e6e02a29 100644 --- a/nerfstudio/viewer_beta/viewer.py +++ b/nerfstudio/viewer_beta/viewer.py @@ -154,11 +154,11 @@ class Viewer: self._toggle_training_state, self.set_camera_visibility, ) + config_path = self.log_filename.parents[0] / "config.yml" with tabs.add_tab("Render", viser.Icon.CAMERA): - populate_render_tab(self.viser_server) + populate_render_tab(self.viser_server, config_path, self.datapath, self.control_panel) with tabs.add_tab("Export", viser.Icon.PACKAGE_EXPORT): - config_path = self.log_filename.parents[0] / "config.yml" populate_export_tab(self.viser_server, self.control_panel, config_path) def nested_folder_install(folder_labels: List[str], element: ViewerElement): From 76f8bcf953dbc001fe3db32839db1448bcd6ad85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Thu, 21 Sep 2023 22:51:35 +0200 Subject: [PATCH 15/19] Disable loading colmap points by default (#2451) --- nerfstudio/data/dataparsers/colmap_dataparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py index 851a30367..c68a9cf6a 100644 --- a/nerfstudio/data/dataparsers/colmap_dataparser.py +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -72,7 +72,7 @@ class ColmapDataParserConfig(DataParserConfig): """Path to depth maps directory. If not set, depths are not loaded.""" colmap_path: Path = Path("sparse/0") """Path to the colmap reconstruction directory relative to the data path.""" - load_3D_points: bool = True + load_3D_points: bool = False """Whether to load the 3D points from the colmap reconstruction.""" max_2D_matches_per_3D_point: int = -1 """Maximum number of 2D matches per 3D point. If set to -1, all 2D matches are loaded. If set to 0, no 2D matches are loaded.""" From c87ebe34ba8b11172971ce48e44b6a8e8eb7a6fc Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 21 Sep 2023 18:38:22 -0700 Subject: [PATCH 16/19] v0.3.4 (#2452) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c64152675..c041858ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nerfstudio" -version = "0.3.3" +version = "0.3.4" description = "All-in-one repository for state-of-the-art NeRFs" readme = "README.md" license = { text="Apache 2.0"} From 2fd8b59d20e73f55106012623d188b227a976df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=95=E3=83=A9=E3=83=8F=E3=83=86=E3=82=A3=E3=80=80?= =?UTF-8?q?=E4=BB=81?= Date: Fri, 22 Sep 2023 23:40:20 +0900 Subject: [PATCH 17/19] Add `background_color` config to keep consistent with other models (#2455) * add backgorund_color config * removed unused color import --- nerfstudio/models/generfacto.py | 7 ++++--- nerfstudio/models/mipnerf.py | 4 ++-- nerfstudio/models/tensorf.py | 4 +++- nerfstudio/models/vanilla_nerf.py | 8 +++++--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/nerfstudio/models/generfacto.py b/nerfstudio/models/generfacto.py index bf34eb335..c07f904c7 100644 --- a/nerfstudio/models/generfacto.py +++ b/nerfstudio/models/generfacto.py @@ -57,7 +57,7 @@ from nerfstudio.model_components.renderers import ( from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider from nerfstudio.model_components.shaders import LambertianShader, NormalsShader from nerfstudio.models.base_model import Model, ModelConfig -from nerfstudio.utils import colormaps, colors, math, misc +from nerfstudio.utils import colormaps, math, misc @dataclass @@ -68,7 +68,8 @@ class GenerfactoModelConfig(ModelConfig): """target class to instantiate""" prompt: str = "a high quality photo of a ripe pineapple" """prompt for stable dreamfusion""" - + background_color: Literal["random", "last_sample", "black", "white"] = "white" + """Whether to randomize the background color.""" orientation_loss_mult: Tuple[float, float] = (0.001, 10.0) """Orientation loss multipier on computed normals.""" orientation_loss_mult_range: Tuple[int, int] = (0, 15000) @@ -251,7 +252,7 @@ class GenerfactoModel(Model): ) # renderers - self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer() self.renderer_normals = NormalsRenderer() diff --git a/nerfstudio/models/mipnerf.py b/nerfstudio/models/mipnerf.py index a3a783461..67ca63251 100644 --- a/nerfstudio/models/mipnerf.py +++ b/nerfstudio/models/mipnerf.py @@ -38,7 +38,7 @@ from nerfstudio.model_components.renderers import ( ) from nerfstudio.models.base_model import Model from nerfstudio.models.vanilla_nerf import VanillaModelConfig -from nerfstudio.utils import colormaps, colors, misc +from nerfstudio.utils import colormaps, misc class MipNerfModel(Model): @@ -81,7 +81,7 @@ class MipNerfModel(Model): self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples, include_original=False) # renderers - self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer() diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index a2cd9901d..a0380d42e 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -89,6 +89,8 @@ class TensoRFModelConfig(ModelConfig): tensorf_encoding: Literal["triplane", "vm", "cp"] = "vm" regularization: Literal["none", "l1", "tv"] = "l1" """Regularization method used in tensorf paper""" + background_color: Literal["random", "last_sample", "black", "white"] = "white" + """Whether to randomize the background color.""" class TensoRFModel(Model): @@ -234,7 +236,7 @@ class TensoRFModel(Model): self.sampler_pdf = PDFSampler(num_samples=self.config.num_samples, single_jitter=True, include_original=False) # renderers - self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer() diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py index ea5344fb8..e95c8dfa4 100644 --- a/nerfstudio/models/vanilla_nerf.py +++ b/nerfstudio/models/vanilla_nerf.py @@ -19,7 +19,7 @@ Implementation of vanilla nerf. from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, List, Tuple, Type +from typing import Any, Dict, List, Tuple, Type, Literal import torch from torch.nn import Parameter @@ -41,7 +41,7 @@ from nerfstudio.model_components.renderers import ( RGBRenderer, ) from nerfstudio.models.base_model import Model, ModelConfig -from nerfstudio.utils import colormaps, colors, misc +from nerfstudio.utils import colormaps, misc @dataclass @@ -58,6 +58,8 @@ class VanillaModelConfig(ModelConfig): """Specifies whether or not to include ray warping based on time.""" temporal_distortion_params: Dict[str, Any] = to_immutable_dict({"kind": TemporalDistortionKind.DNERF}) """Parameters to instantiate temporal distortion with""" + background_color: Literal["random", "last_sample", "black", "white"] = "white" + """Whether to randomize the background color.""" class NeRFModel(Model): @@ -110,7 +112,7 @@ class NeRFModel(Model): self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples) # renderers - self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer() From 52f26e82b3adb9ba89d61e7ca938a00faa7994dd Mon Sep 17 00:00:00 2001 From: Maxim Bonnaerens Date: Fri, 22 Sep 2023 17:36:51 +0200 Subject: [PATCH 18/19] Do not materialize mlp_head if no field_heads (#2457) In case it is used in a proposal network we avoid initializing the head. Co-authored-by: Matthew Tancik --- nerfstudio/fields/vanilla_nerf_field.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nerfstudio/fields/vanilla_nerf_field.py b/nerfstudio/fields/vanilla_nerf_field.py index c9d2bbb78..22e11a25b 100644 --- a/nerfstudio/fields/vanilla_nerf_field.py +++ b/nerfstudio/fields/vanilla_nerf_field.py @@ -74,15 +74,15 @@ class NeRFField(Field): skip_connections=skip_connections, out_activation=nn.ReLU(), ) - - self.mlp_head = MLP( - in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim(), - num_layers=head_mlp_num_layers, - layer_width=head_mlp_layer_width, - out_activation=nn.ReLU(), - ) - self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim()) + + if field_heads: + self.mlp_head = MLP( + in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim(), + num_layers=head_mlp_num_layers, + layer_width=head_mlp_layer_width, + out_activation=nn.ReLU(), + ) self.field_heads = nn.ModuleList([field_head() for field_head in field_heads] if field_heads else []) # type: ignore for field_head in self.field_heads: field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore From 8812d7a5b092beb822ddd2b191c5e82cd3f66ed5 Mon Sep 17 00:00:00 2001 From: Maxim Bonnaerens Date: Fri, 22 Sep 2023 19:45:05 +0200 Subject: [PATCH 19/19] .vscode settings match move to ms-python.black-formatter (#2456) Co-authored-by: Brent Yi --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4c3a7447a..8b0f1d750 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,7 +33,7 @@ "editor.formatOnSave": true, "python.envFile": "${workspaceFolder}/.env", "python.formatting.provider": "none", - "python.formatting.blackArgs": ["--line-length=120"], + "black-formatter.args": ["--line-length=120"], "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": false, "python.linting.enabled": true,