Files
nerfstudio/nerfstudio/viewer_beta/control_panel.py
T
Brent Yi 7a8e31d84e Improved render panel for beta viewer (#2667)
* Continuous velocities for render splines

* Configure ruff line length

* Improved render preview

* No longer need to update camera FOV in render preview

* Details

* Polishing

* Cleanup

* Details

* Address pyright errors

* type: ignore for broken (?) Aria stub
2023-12-20 17:20:41 -08:00

424 lines
17 KiB
Python

# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Control panel for the viewer """
from collections import defaultdict
from typing import Callable, DefaultDict, List, Tuple, get_args
import numpy as np
import torch
import viser
import viser.transforms as vtf
from nerfstudio.data.scene_box import OrientedBox
from nerfstudio.utils.colormaps import ColormapOptions, Colormaps
from nerfstudio.viewer_beta.viewer_elements import ( # ViewerButtonGroup,
ViewerButtonGroup,
ViewerCheckbox,
ViewerDropdown,
ViewerElement,
ViewerNumber,
ViewerRGB,
ViewerSlider,
ViewerVec3,
)
from viser import ViserServer
class ControlPanel:
"""
Initializes the control panel with all the elements
Args:
time_enabled: whether or not the time slider should be enabled
rerender_cb: a callback that will be called when the user changes a parameter that requires a rerender
(eg train speed, max res, etc)
update_output_cb: a callback that will be called when the user changes the output render
default_composite_depth: whether to default to compositing depth or not
"""
def __init__(
self,
viser_server: ViserServer,
time_enabled: bool,
scale_ratio: float,
rerender_cb: Callable[[], None],
update_output_cb: Callable,
update_split_output_cb: Callable,
default_composite_depth: bool = True,
):
self.viser_scale_ratio = scale_ratio
# elements holds a mapping from tag: [elements]
self.viser_server = viser_server
self._elements_by_tag: DefaultDict[str, List[ViewerElement]] = defaultdict(lambda: [])
self.default_composite_depth = default_composite_depth
self._train_speed = ViewerButtonGroup(
name="Train Speed",
default_value="Mid",
options=["Slow", "Mid", "Fast"],
cb_hook=lambda han: self._train_speed_cb(),
)
self._output_render = ViewerDropdown(
"Output type",
"not set",
["not set"],
cb_hook=lambda han: [self.update_control_panel(), update_output_cb(han), rerender_cb()],
hint="The output to render",
)
self._colormap = ViewerDropdown[Colormaps](
"Colormap", "default", ["default"], cb_hook=lambda _: rerender_cb(), hint="The colormap to use"
)
self._invert = ViewerCheckbox("Invert", False, cb_hook=lambda _: rerender_cb(), hint="Invert the colormap")
self._normalize = ViewerCheckbox(
"Normalize", True, cb_hook=lambda _: rerender_cb(), hint="Normalize the colormap"
)
self._min = ViewerNumber("Min", 0.0, cb_hook=lambda _: rerender_cb(), hint="Min value of the colormap")
self._max = ViewerNumber("Max", 1.0, cb_hook=lambda _: rerender_cb(), hint="Max value of the colormap")
self._split = ViewerCheckbox(
"Enable",
False,
cb_hook=lambda han: [self.update_control_panel(), rerender_cb()],
hint="Render two outputs",
)
self._split_percentage = ViewerSlider(
"Split percentage", 0.5, 0.0, 1.0, 0.01, cb_hook=lambda _: rerender_cb(), hint="Where to split"
)
self._split_output_render = ViewerDropdown(
"Output render split",
"not set",
["not set"],
cb_hook=lambda han: [self.update_control_panel(), update_split_output_cb(han), rerender_cb()],
hint="The second output",
)
# Hack: spaces are after at the end of the names to make them unique
self._split_colormap = ViewerDropdown[Colormaps](
"Colormap ", "default", ["default"], cb_hook=lambda _: rerender_cb(), hint="Colormap of the second output"
)
self._split_invert = ViewerCheckbox(
"Invert ", False, cb_hook=lambda _: rerender_cb(), hint="Invert the colormap of the second output"
)
self._split_normalize = ViewerCheckbox(
"Normalize ", True, cb_hook=lambda _: rerender_cb(), hint="Normalize the colormap of the second output"
)
self._split_min = ViewerNumber(
"Min ", 0.0, cb_hook=lambda _: rerender_cb(), hint="Min value of the colormap of the second output"
)
self._split_max = ViewerNumber(
"Max ", 1.0, cb_hook=lambda _: rerender_cb(), hint="Max value of the colormap of the second output"
)
self._train_util = ViewerSlider(
"Train Util",
default_value=0.85,
min_value=0.0,
max_value=1,
step=0.05,
hint="Target training utilization, 0.0 is slow, 1.0 is fast. Doesn't affect final render quality",
)
self._layer_depth = ViewerCheckbox(
"Composite depth",
self.default_composite_depth,
cb_hook=lambda _: rerender_cb(),
hint="Allow NeRF to occlude 3D browser objects",
)
self._max_res = ViewerSlider(
"Max res",
512,
64,
2048,
100,
cb_hook=lambda _: rerender_cb(),
hint="Maximum resolution to render in viewport",
)
self._crop_viewport = ViewerCheckbox(
"Enable ",
False,
cb_hook=lambda han: [self.update_control_panel(), rerender_cb()],
hint="Crop the scene to a specified box",
)
self._background_color = ViewerRGB(
"Background color", (38, 42, 55), cb_hook=lambda _: rerender_cb(), hint="Color of the background"
)
self._crop_handle = self.viser_server.add_transform_controls("Crop", depth_test=False, line_width=4.0)
def update_center(han):
self._crop_handle.position = tuple(p * self.viser_scale_ratio for p in han.value) # type: ignore
self._crop_center = ViewerVec3(
"Crop center",
(0.0, 0.0, 0.0),
step=0.01,
cb_hook=lambda e: [rerender_cb(), update_center(e)],
hint="Center of the crop box",
)
def update_rot(han):
self._crop_handle.wxyz = vtf.SO3.from_rpy_radians(*han.value).wxyz
self._crop_rot = ViewerVec3(
"Crop rotation",
(0.0, 0.0, 0.0),
step=0.01,
cb_hook=lambda e: [rerender_cb(), update_rot(e)],
hint="Rotation of the crop box",
)
self._crop_scale = ViewerVec3(
"Crop scale", (1.0, 1.0, 1.0), step=0.01, cb_hook=lambda _: rerender_cb(), hint="Size of the crop box."
)
@self._crop_handle.on_update
def _update_crop_handle(han):
pos = self._crop_handle.position
self._crop_center.value = tuple(p / self.viser_scale_ratio for p in pos) # type: ignore
rpy = vtf.SO3(self._crop_handle.wxyz).as_rpy_radians()
self._crop_rot.value = (float(rpy.roll), float(rpy.pitch), float(rpy.yaw))
self._time = ViewerSlider("Time", 0.0, 0.0, 1.0, 0.01, cb_hook=lambda _: rerender_cb(), hint="Time to render")
self._time_enabled = time_enabled
self.add_element(self._train_speed)
self.add_element(self._train_util)
with self.viser_server.add_gui_folder("Render Options"):
self.add_element(self._max_res)
self.add_element(self._output_render)
self.add_element(self._colormap)
self.add_element(self._layer_depth)
# colormap options
self.add_element(self._invert, additional_tags=("colormap",))
self.add_element(self._normalize, additional_tags=("colormap",))
self.add_element(self._min, additional_tags=("colormap",))
self.add_element(self._max, additional_tags=("colormap",))
# split options
with self.viser_server.add_gui_folder("Split Screen"):
self.add_element(self._split)
self.add_element(self._split_percentage, additional_tags=("split",))
self.add_element(self._split_output_render, additional_tags=("split",))
self.add_element(self._split_colormap, additional_tags=("split",))
self.add_element(self._split_invert, additional_tags=("split_colormap",))
self.add_element(self._split_normalize, additional_tags=("split_colormap",))
self.add_element(self._split_min, additional_tags=("split_colormap",))
self.add_element(self._split_max, additional_tags=("split_colormap",))
with self.viser_server.add_gui_folder("Crop Viewport"):
self.add_element(self._crop_viewport)
# Crop options
self.add_element(self._background_color, additional_tags=("crop",))
self.add_element(self._crop_center, additional_tags=("crop",))
self.add_element(self._crop_scale, additional_tags=("crop",))
self.add_element(self._crop_rot, additional_tags=("crop",))
self.add_element(self._time, additional_tags=("time",))
self._reset_camera = viser_server.add_gui_button(
label="Reset Up Direction",
icon=viser.Icon.ARROW_BIG_UP_LINES,
color="gray",
hint="Set the up direction of the camera orbit controls to the camera's current up direction.",
)
self._reset_camera.on_click(self._reset_camera_cb)
def _train_speed_cb(self) -> None:
pass
"""Callback for when the train speed is changed"""
if self._train_speed.value == "Fast":
self._train_util.value = 0.95
self._max_res.value = 256
elif self._train_speed.value == "Mid":
self._train_util.value = 0.85
self._max_res.value = 512
elif self._train_speed.value == "Slow":
self._train_util.value = 0.5
self._max_res.value = 1024
def _reset_camera_cb(self, _) -> None:
for client in self.viser_server.get_clients().values():
client.camera.up_direction = vtf.SO3(client.camera.wxyz) @ np.array([0.0, -1.0, 0.0])
def update_output_options(self, new_options: List[str]):
"""
Args:
new_options: a list of new output options
"""
self._output_render.set_options(new_options)
self._split_output_render.set_options(new_options)
self._split_output_render.value = new_options[-1]
def add_element(self, e: ViewerElement, additional_tags: Tuple[str, ...] = tuple()) -> None:
"""Adds an element to the control panel
Args:
e: the element to add
additional_tags: additional tags to add to the element for selection
"""
self._elements_by_tag["all"].append(e)
for t in additional_tags:
self._elements_by_tag[t].append(e)
e.install(self.viser_server)
def update_control_panel(self) -> None:
"""
Sets elements to be hidden or not based on the current state of the control panel
"""
self._colormap.set_disabled(self.output_render == "rgb")
for e in self._elements_by_tag["colormap"]:
e.set_hidden(self.output_render == "rgb")
for e in self._elements_by_tag["split_colormap"]:
e.set_hidden(not self._split.value or self.split_output_render == "rgb")
for e in self._elements_by_tag["crop"]:
e.set_hidden(not self.crop_viewport)
self._time.set_hidden(not self._time_enabled)
self._split_percentage.set_hidden(not self._split.value)
self._split_output_render.set_hidden(not self._split.value)
self._split_colormap.set_hidden(not self._split.value)
self._split_colormap.set_disabled(self.split_output_render == "rgb")
self._crop_handle.visible = self.crop_viewport
def update_colormap_options(self, dimensions: int, dtype: type) -> None:
"""update the colormap options based on the current render
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
"""
self._colormap.set_options(_get_colormap_options(dimensions, dtype))
def update_split_colormap_options(self, dimensions: int, dtype: type) -> None:
"""update the split colormap options based on the current render
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
"""
self._split_colormap.set_options(_get_colormap_options(dimensions, dtype))
@property
def output_render(self) -> str:
"""Returns the current output render"""
return self._output_render.value
@property
def split_output_render(self) -> str:
"""Returns the current output for the split render"""
return self._split_output_render.value
@property
def split(self) -> bool:
"""Returns whether the split is enabled"""
return self._split.value
@property
def split_percentage(self) -> float:
"""Returns the percentage of the screen to split"""
return self._split_percentage.value
@property
def train_util(self) -> float:
"""Returns the current train util setting"""
return self._train_util.value
@property
def max_res(self) -> int:
"""Returns the current max res setting"""
return self._max_res.value
@property
def crop_viewport(self) -> bool:
"""Returns the current crop viewport setting"""
return self._crop_viewport.value
@crop_viewport.setter
def crop_viewport(self, value: bool):
"""Sets the crop viewport setting"""
self._crop_viewport.value = value
@property
def crop_obb(self):
"""Returns the current crop obb setting"""
rxyz = self._crop_rot.value
R = torch.tensor(vtf.SO3.from_rpy_radians(rxyz[0], rxyz[1], rxyz[2]).as_matrix())
obb = OrientedBox(R, torch.tensor(self._crop_center.value), torch.tensor(self._crop_scale.value))
return obb
@property
def background_color(self) -> Tuple[int, int, int]:
"""Returns the current background color"""
return self._background_color.value
@background_color.setter
def background_color(self, value: Tuple[int, int, int]):
"""Sets the background color"""
self._background_color.value = value
@property
def time(self) -> float:
"""Returns the current background color"""
return self._time.value
@time.setter
def time(self, value: float):
"""Sets the background color"""
self._time.value = value
@property
def colormap_options(self) -> ColormapOptions:
"""Returns the current colormap options"""
return ColormapOptions(
colormap=self._colormap.value,
normalize=self._normalize.value,
colormap_min=self._min.value,
colormap_max=self._max.value,
invert=self._invert.value,
)
@property
def split_colormap_options(self) -> ColormapOptions:
"""Returns the current colormap options"""
return ColormapOptions(
colormap=self._split_colormap.value,
normalize=self._split_normalize.value,
colormap_min=self._split_min.value,
colormap_max=self._split_max.value,
invert=self._split_invert.value,
)
@property
def layer_depth(self):
return self._layer_depth.value
def _get_colormap_options(dimensions: int, dtype: type) -> List[Colormaps]:
"""
Given the number of dimensions and data type, returns a list of available colormap options
to use with the visualize() function.
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
Returns:
a list of available colormap options
"""
colormap_options: List[Colormaps] = []
if dimensions == 3:
colormap_options = ["default"]
if dimensions == 1 and dtype in [torch.float64, torch.float32, torch.float16, torch.bfloat16]:
colormap_options = [c for c in list(get_args(Colormaps)) if c not in ("default", "pca")]
if dimensions > 3:
colormap_options = ["pca"]
return colormap_options