From addf477e8cb9ba38538ed8d7edca10f06b847df7 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Fri, 26 May 2023 15:15:36 -0700 Subject: [PATCH] add controls to the beta viewer (#2005) * add controls to the beta viewer * Remove misc file * works now --- nerfstudio/engine/trainer.py | 14 +- nerfstudio/viewer_beta/control_panel.py | 369 +++++++++++++++ .../viewer_beta/render_state_machine.py | 66 ++- nerfstudio/viewer_beta/utils.py | 30 +- .../{viewer_state.py => viewer.py} | 76 ++- nerfstudio/viewer_beta/viewer_elements.py | 440 ++++++++++++++++++ pyproject.toml | 2 +- 7 files changed, 970 insertions(+), 27 deletions(-) create mode 100644 nerfstudio/viewer_beta/control_panel.py rename nerfstudio/viewer_beta/{viewer_state.py => viewer.py} (73%) create mode 100644 nerfstudio/viewer_beta/viewer_elements.py diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index a69c8d68e..713263234 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -33,24 +33,16 @@ from rich.table import Table from torch.cuda.amp.grad_scaler import GradScaler from nerfstudio.configs.experiment_config import ExperimentConfig -from nerfstudio.engine.callbacks import ( - TrainingCallback, - TrainingCallbackAttributes, - TrainingCallbackLocation, -) +from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation from nerfstudio.engine.optimizers import Optimizers from nerfstudio.pipelines.base_pipeline import VanillaPipeline from nerfstudio.utils import profiler, writer -from nerfstudio.utils.decorators import ( - check_eval_enabled, - check_main_thread, - check_viewer_enabled, -) +from nerfstudio.utils.decorators import check_eval_enabled, check_main_thread, check_viewer_enabled from nerfstudio.utils.misc import step_check from nerfstudio.utils.rich_utils import CONSOLE from nerfstudio.utils.writer import EventName, TimeWriter from nerfstudio.viewer.server.viewer_state import ViewerState -from nerfstudio.viewer_beta.viewer_state import ViewerState as ViewerBetaState +from nerfstudio.viewer_beta.viewer import Viewer as ViewerBetaState TRAIN_INTERATION_OUTPUT = Tuple[torch.Tensor, Dict[str, torch.Tensor], Dict[str, torch.Tensor]] TORCH_DEVICE = str diff --git a/nerfstudio/viewer_beta/control_panel.py b/nerfstudio/viewer_beta/control_panel.py new file mode 100644 index 000000000..ea0492eb2 --- /dev/null +++ b/nerfstudio/viewer_beta/control_panel.py @@ -0,0 +1,369 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Control panel for the viewer """ +from collections import defaultdict +from typing import Callable, DefaultDict, List, Tuple, get_args + +import torch +from viser import ViserServer + +from nerfstudio.utils.colormaps import ColormapOptions, Colormaps +from nerfstudio.viewer_beta.viewer_elements import ( # ViewerButtonGroup, + ViewerButtonGroup, + ViewerCheckbox, + ViewerDropdown, + ViewerElement, + ViewerNumber, + ViewerRGB, + ViewerSlider, + ViewerVec3, +) + + +class ControlPanel: + """ + Initializes the control panel with all the elements + Args: + time_enabled: whether or not the time slider should be enabled + rerender_cb: a callback that will be called when the user changes a parameter that requires a rerender + (eg train speed, max res, etc) + crop_update_cb: a callback that will be called when the user changes the crop parameters + update_output_cb: a callback that will be called when the user changes the output render + """ + + def __init__( + self, + viser_server: ViserServer, + time_enabled: bool, + rerender_cb: Callable, + crop_update_cb: Callable, + update_output_cb: Callable, + update_split_output_cb: Callable, + ): + # elements holds a mapping from tag: [elements] + self.viser_server = viser_server + self._elements_by_tag: DefaultDict[str, List[ViewerElement]] = defaultdict(lambda: []) + + self._train_speed = ViewerButtonGroup( + name="Train Speed ", + default_value="Balanced", + options=["Slow", "Balanced", "Fast"], + cb_hook=lambda han: self._train_speed_cb(), + ) + self._output_render = ViewerDropdown( + "Output Render", + "not set", + ["not set"], + cb_hook=lambda han: [self.update_control_panel(), update_output_cb(han), rerender_cb(han)], + hint="The output to render", + ) + self._colormap = ViewerDropdown[Colormaps]( + "Colormap", "default", ["default"], cb_hook=rerender_cb, hint="The colormap to use" + ) + self._invert = ViewerCheckbox("Invert", False, cb_hook=rerender_cb, hint="Invert the colormap") + self._normalize = ViewerCheckbox("Normalize", True, cb_hook=rerender_cb, hint="Normalize the colormap") + self._min = ViewerNumber("Min", 0.0, cb_hook=rerender_cb, hint="Min value of the colormap") + self._max = ViewerNumber("Max", 1.0, cb_hook=rerender_cb, hint="Max value of the colormap") + + self._split = ViewerCheckbox( + "Enable", + False, + cb_hook=lambda han: [self.update_control_panel(), rerender_cb(han)], + hint="Render two outputs", + ) + self._split_percentage = ViewerSlider( + "Split Percentage", 0.5, 0.0, 1.0, 0.01, cb_hook=rerender_cb, hint="Where to split" + ) + self._split_output_render = ViewerDropdown( + "Output Render Split", + "not set", + ["not set"], + cb_hook=lambda han: [self.update_control_panel(), update_split_output_cb(han), rerender_cb(han)], + hint="The second output", + ) + # Hack: spaces are after at the end of the names to make them unique + self._split_colormap = ViewerDropdown[Colormaps]( + "Colormap ", "default", ["default"], cb_hook=rerender_cb, hint="Colormap of the second output" + ) + self._split_invert = ViewerCheckbox( + "Invert ", False, cb_hook=rerender_cb, hint="Invert the colormap of the second output" + ) + self._split_normalize = ViewerCheckbox( + "Normalize ", True, cb_hook=rerender_cb, hint="Normalize the colormap of the second output" + ) + self._split_min = ViewerNumber( + "Min ", 0.0, cb_hook=rerender_cb, hint="Min value of the colormap of the second output" + ) + self._split_max = ViewerNumber( + "Max ", 1.0, cb_hook=rerender_cb, hint="Max value of the colormap of the second output" + ) + + self._train_util = ViewerSlider( + "Train Util", + default_value=0.85, + min_value=0.0, + max_value=1, + step=0.05, + hint="Target training utilization, 0.0 is slow, 1.0 is fast. Doesn't affect final render quality", + ) + self._max_res = ViewerSlider( + "Max Res", 512, 64, 2048, 100, cb_hook=rerender_cb, hint="Maximum resolution to render in viewport" + ) + self._crop_viewport = ViewerCheckbox( + "Enable ", + False, + cb_hook=lambda han: [self.update_control_panel(), crop_update_cb(han), rerender_cb(han)], + hint="Crop the scene to a specified box", + ) + self._background_color = ViewerRGB( + "Background color", (38, 42, 55), cb_hook=crop_update_cb, hint="Color of the background" + ) + self._crop_min = ViewerVec3( + "Crop Min", (-1, -1, -1), 0.05, cb_hook=crop_update_cb, hint="Minimum value of the crop" + ) + self._crop_max = ViewerVec3( + "Crop Max", (1, 1, 1), 0.05, cb_hook=crop_update_cb, hint="Maximum value of the crop" + ) + self._time = ViewerSlider("Time", 0.0, 0.0, 1.0, 0.01, cb_hook=rerender_cb, hint="Time to render") + self._time_enabled = time_enabled + + self.add_element(self._train_speed) + self.add_element(self._train_util) + with self.viser_server.gui_folder("Render Options"): + self.add_element(self._max_res) + self.add_element(self._output_render) + self.add_element(self._colormap) + # colormap options + with self.viser_server.gui_folder(" "): + self.add_element(self._invert, additional_tags=("colormap",)) + self.add_element(self._normalize, additional_tags=("colormap",)) + self.add_element(self._min, additional_tags=("colormap",)) + self.add_element(self._max, additional_tags=("colormap",)) + + # split options + with self.viser_server.gui_folder("Split Screen"): + self.add_element(self._split) + + self.add_element(self._split_percentage, additional_tags=("split",)) + self.add_element(self._split_output_render, additional_tags=("split",)) + self.add_element(self._split_colormap, additional_tags=("split",)) + with self.viser_server.gui_folder(" "): + self.add_element(self._split_invert, additional_tags=("split_colormap",)) + self.add_element(self._split_normalize, additional_tags=("split_colormap",)) + self.add_element(self._split_min, additional_tags=("split_colormap",)) + self.add_element(self._split_max, additional_tags=("split_colormap",)) + + with self.viser_server.gui_folder("Crop Viewport"): + self.add_element(self._crop_viewport) + + # Crop options + self.add_element(self._background_color, additional_tags=("crop",)) + self.add_element(self._crop_min, additional_tags=("crop",)) + self.add_element(self._crop_max, additional_tags=("crop",)) + + self.add_element(self._time, additional_tags=("time",)) + + def _train_speed_cb(self) -> None: + pass + + """Callback for when the train speed is changed""" + if self._train_speed.value == "Fast": + self._train_util.value = 0.95 + self._max_res.value = 256 + elif self._train_speed.value == "Balanced": + self._train_util.value = 0.85 + self._max_res.value = 512 + elif self._train_speed.value == "Slow": + self._train_util.value = 0.5 + self._max_res.value = 1024 + + def update_output_options(self, new_options: List[str]): + """ + Args: + new_options: a list of new output options + """ + self._output_render.set_options(new_options) + self._split_output_render.set_options(new_options) + self._split_output_render.value = new_options[-1] + + def add_element(self, e: ViewerElement, additional_tags: Tuple[str] = tuple()) -> None: + """Adds an element to the control panel + + Args: + e: the element to add + additional_tags: additional tags to add to the element for selection + """ + self._elements_by_tag["all"].append(e) + for t in additional_tags: + self._elements_by_tag[t].append(e) + e.install(self.viser_server) + + def update_control_panel(self) -> None: + """ + Sets elements to be hidden or not based on the current state of the control panel + """ + self._colormap.set_disabled(self.output_render == "rgb") + for e in self._elements_by_tag["colormap"]: + e.set_hidden(self.output_render == "rgb") + for e in self._elements_by_tag["split_colormap"]: + e.set_hidden(not self._split.value or self.split_output_render == "rgb") + for e in self._elements_by_tag["crop"]: + e.set_hidden(not self.crop_viewport) + self._time.set_hidden(not self._time_enabled) + self._split_percentage.set_hidden(not self._split.value) + self._split_output_render.set_hidden(not self._split.value) + self._split_colormap.set_hidden(not self._split.value) + self._split_colormap.set_disabled(self.split_output_render == "rgb") + + def update_colormap_options(self, dimensions: int, dtype: type) -> None: + """update the colormap options based on the current render + + Args: + dimensions: the number of dimensions of the render + dtype: the data type of the render + """ + self._colormap.set_options(_get_colormap_options(dimensions, dtype)) + + def update_split_colormap_options(self, dimensions: int, dtype: type) -> None: + """update the split colormap options based on the current render + + Args: + dimensions: the number of dimensions of the render + dtype: the data type of the render + """ + self._split_colormap.set_options(_get_colormap_options(dimensions, dtype)) + + @property + def output_render(self) -> str: + """Returns the current output render""" + return self._output_render.value + + @property + def split_output_render(self) -> str: + """Returns the current output for the split render""" + return self._split_output_render.value + + @property + def split(self) -> bool: + """Returns whether the split is enabled""" + return self._split.value + + @property + def split_percentage(self) -> float: + """Returns the percentage of the screen to split""" + return self._split_percentage.value + + @property + def train_util(self) -> float: + """Returns the current train util setting""" + return self._train_util.value + + @property + def max_res(self) -> int: + """Returns the current max res setting""" + return self._max_res.value + + @property + def crop_viewport(self) -> bool: + """Returns the current crop viewport setting""" + return self._crop_viewport.value + + @crop_viewport.setter + def crop_viewport(self, value: bool): + """Sets the crop viewport setting""" + self._crop_viewport.value = value + + @property + def crop_min(self) -> Tuple[float, float, float]: + """Returns the current crop min setting""" + return self._crop_min.value + + @crop_min.setter + def crop_min(self, value: Tuple[float, float, float]): + """Sets the crop min setting""" + self._crop_min.value = value + + @property + def crop_max(self) -> Tuple[float, float, float]: + """Returns the current crop max setting""" + return self._crop_max.value + + @crop_max.setter + def crop_max(self, value: Tuple[float, float, float]): + """Sets the crop max setting""" + self._crop_max.value = value + + @property + def background_color(self) -> Tuple[int, int, int]: + """Returns the current background color""" + return self._background_color.value + + @background_color.setter + def background_color(self, value: Tuple[int, int, int]): + """Sets the background color""" + self._background_color.value = value + + @property + def time(self) -> float: + """Returns the current background color""" + return self._time.value + + @time.setter + def time(self, value: float): + """Sets the background color""" + self._time.value = value + + @property + def colormap_options(self) -> ColormapOptions: + """Returns the current colormap options""" + return ColormapOptions( + colormap=self._colormap.value, + normalize=self._normalize.value, + colormap_min=self._min.value, + colormap_max=self._max.value, + invert=self._invert.value, + ) + + @property + def split_colormap_options(self) -> ColormapOptions: + """Returns the current colormap options""" + return ColormapOptions( + colormap=self._split_colormap.value, + normalize=self._split_normalize.value, + colormap_min=self._split_min.value, + colormap_max=self._split_max.value, + invert=self._split_invert.value, + ) + + +def _get_colormap_options(dimensions: int, dtype: type) -> List[Colormaps]: + """ + Given the number of dimensions and data type, returns a list of available colormap options + to use with the visualize() function. + + Args: + dimensions: the number of dimensions of the render + dtype: the data type of the render + Returns: + a list of available colormap options + """ + colormap_options: List[Colormaps] = [] + if dimensions == 3: + colormap_options = ["default"] + if dimensions == 1 and dtype in [torch.float64, torch.float32, torch.float16, torch.bfloat16]: + colormap_options = [c for c in list(get_args(Colormaps)) if c != "default"] + if dimensions > 3: + colormap_options = ["pca"] + return colormap_options diff --git a/nerfstudio/viewer_beta/render_state_machine.py b/nerfstudio/viewer_beta/render_state_machine.py index f2323b1d8..d18200761 100644 --- a/nerfstudio/viewer_beta/render_state_machine.py +++ b/nerfstudio/viewer_beta/render_state_machine.py @@ -22,13 +22,15 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, get_args import torch -from nerfstudio.utils import writer +from nerfstudio.model_components.renderers import background_color_override_context +from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils +from nerfstudio.viewer_beta import utils from nerfstudio.viewer_beta.utils import CameraState, get_camera if TYPE_CHECKING: - from nerfstudio.viewer_beta.viewer_state import ViewerState + from nerfstudio.viewer_beta.viewer import Viewer RenderStates = Literal["low_move", "low_static", "high"] RenderActions = Literal["rerender", "move", "static", "step"] @@ -52,7 +54,7 @@ class RenderStateMachine(threading.Thread): viewer: the viewer state """ - def __init__(self, viewer: ViewerState): + def __init__(self, viewer: Viewer): threading.Thread.__init__(self) self.transitions: Dict[RenderStates, Dict[RenderActions, RenderStates]] = { s: {} for s in get_args(RenderStates) @@ -111,6 +113,14 @@ class RenderStateMachine(threading.Thread): camera_state: the current camera state """ + # initialize the camera ray bundle + utils.update_render_aabb( + crop_viewport=self.viewer.control_panel.crop_viewport, + crop_min=self.viewer.control_panel.crop_min, + crop_max=self.viewer.control_panel.crop_max, + model=self.viewer.get_model(), + ) + image_height, image_width = self._calculate_image_res(camera_state.aspect) camera = get_camera(camera_state, image_height, image_width) @@ -123,8 +133,20 @@ class RenderStateMachine(threading.Thread): with TimeWriter(None, None, write=False) as vis_t: self.viewer.get_model().eval() step = self.viewer.step - with torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if self.viewer.control_panel.crop_viewport: + color = self.viewer.control_panel.background_color + if color is None: + background_color = torch.tensor([0.0, 0.0, 0.0], device=self.viewer.pipeline.model.device) + else: + background_color = torch.tensor( + [color[0] / 255.0, color[1] / 255.0, color[2] / 255.0], + device=self.viewer.get_model().device, + ) + with background_color_override_context(background_color), torch.no_grad(): + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + else: + with torch.no_grad(): + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) self.viewer.get_model().train() num_rays = len(camera_ray_bundle) render_time = vis_t.duration @@ -173,7 +195,37 @@ class RenderStateMachine(threading.Thread): Args: outputs: the dictionary of outputs to choose from, from the model """ - selected_output = (outputs["rgb"] * 255).type(torch.uint8) + output_keys = set(outputs.keys()) + if self.output_keys != output_keys: + self.output_keys = output_keys + self.viewer.control_panel.update_output_options(list(outputs.keys())) + + output_render = self.viewer.control_panel.output_render + self.viewer.update_colormap_options( + dimensions=outputs[output_render].shape[-1], dtype=outputs[output_render].dtype + ) + selected_output = colormaps.apply_colormap( + image=outputs[self.viewer.control_panel.output_render], + colormap_options=self.viewer.control_panel.colormap_options, + ) + + if self.viewer.control_panel.split: + split_output_render = self.viewer.control_panel.split_output_render + self.viewer.update_split_colormap_options( + dimensions=outputs[split_output_render].shape[-1], dtype=outputs[split_output_render].dtype + ) + split_output = colormaps.apply_colormap( + image=outputs[self.viewer.control_panel.split_output_render], + colormap_options=self.viewer.control_panel.split_colormap_options, + ) + split_index = min( + int(self.viewer.control_panel.split_percentage * selected_output.shape[1]), + selected_output.shape[1] - 1, + ) + selected_output = torch.cat([selected_output[:, :split_index], split_output[:, split_index:]], dim=1) + selected_output[:, split_index] = torch.tensor([0.133, 0.157, 0.192], device=selected_output.device) + + selected_output = (selected_output * 255).type(torch.uint8) self.viewer.viser_server.set_background_image( selected_output.cpu().numpy(), @@ -190,7 +242,7 @@ class RenderStateMachine(threading.Thread): image_height: the maximum image height that can be rendered in the time budget image_width: the maximum image width that can be rendered in the time budget """ - max_res = self.viewer.max_res + max_res = self.viewer.control_panel.max_res if self.state == "high": # high res is always static image_height = max_res diff --git a/nerfstudio/viewer_beta/utils.py b/nerfstudio/viewer_beta/utils.py index 77f4e198f..0dd435b88 100644 --- a/nerfstudio/viewer_beta/utils.py +++ b/nerfstudio/viewer_beta/utils.py @@ -13,13 +13,15 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional, Tuple, Union import numpy as np import torch from jaxtyping import Float from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.models.base_model import Model @dataclass @@ -65,3 +67,29 @@ def get_camera( times=torch.tensor([0.0], dtype=torch.float32), ) return camera + + +def update_render_aabb( + crop_viewport: bool, crop_min: Tuple[float, float, float], crop_max: Tuple[float, float, float], model: Model +): + """ + update the render aabb box for the viewer: + + Args: + crop_viewport: whether to crop the viewport + crop_min: min of the crop box + crop_max: max of the crop box + model: the model to render + """ + + if crop_viewport: + crop_min_tensor = torch.tensor(crop_min, dtype=torch.float32) + crop_max_tensor = torch.tensor(crop_max, dtype=torch.float32) + + if isinstance(model.render_aabb, SceneBox): + model.render_aabb.aabb[0] = crop_min_tensor + model.render_aabb.aabb[1] = crop_max_tensor + else: + model.render_aabb = SceneBox(aabb=torch.stack([crop_min_tensor, crop_max_tensor], dim=0)) + else: + model.render_aabb = None diff --git a/nerfstudio/viewer_beta/viewer_state.py b/nerfstudio/viewer_beta/viewer.py similarity index 73% rename from nerfstudio/viewer_beta/viewer_state.py rename to nerfstudio/viewer_beta/viewer.py index 115ff6f9a..f4e8d3429 100644 --- a/nerfstudio/viewer_beta/viewer_state.py +++ b/nerfstudio/viewer_beta/viewer.py @@ -32,6 +32,7 @@ from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName from nerfstudio.viewer.server import viewer_utils +from nerfstudio.viewer_beta.control_panel import ControlPanel from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStateMachine from nerfstudio.viewer_beta.utils import CameraState @@ -42,7 +43,7 @@ VISER_NERFSTUDIO_SCALE_RATIO: int = 10 @decorate_all([check_main_thread]) -class ViewerState: +class Viewer: """Class to hold state for viewer variables Args: @@ -54,9 +55,12 @@ class ViewerState: Attributes: viewer_url: url to open viewer + viser_server: the viser server """ viewer_url: str + viser_server: viser.ViserServer + camera_state: Optional[CameraState] = None def __init__( self, @@ -74,6 +78,7 @@ class ViewerState: self.pipeline = pipeline self.log_filename = log_filename self.datapath = datapath.parent if datapath.is_file() else datapath + self.include_time = self.pipeline.datamanager.includes_time if self.config.websocket_port is None: websocket_port = viewer_utils.get_free_port(default_port=self.config.websocket_port_default) @@ -94,12 +99,19 @@ class ViewerState: self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port) - self.render_statemachine = RenderStateMachine(self) - self.render_statemachine.start() - self.viser_server.on_client_connect(self.handle_new_client) - self.camera_state: Optional[CameraState] = None + self.control_panel = ControlPanel( + self.viser_server, + self.include_time, + self._interrupt_render, + self._crop_params_update, + self._output_type_change, + self._output_split_type_change, + ) + + self.render_statemachine = RenderStateMachine(self) + self.render_statemachine.start() def handle_new_client(self, client: viser.ClientHandle) -> None: @client.camera.on_update @@ -112,6 +124,35 @@ class ViewerState: self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w) self.render_statemachine.action(RenderAction("move", self.camera_state)) + def _interrupt_render(self, _) -> None: + """Interrupt current render.""" + if self.camera_message is not None: + self.render_statemachine.action(RenderAction("rerender", self.camera_message)) + + def _crop_params_update(self, _) -> None: + """Update crop parameters""" + print("Crop params not set up") + # crop_min = torch.tensor(self.control_panel.crop_min, dtype=torch.float32) + # crop_max = torch.tensor(self.control_panel.crop_max, dtype=torch.float32) + # scene_box = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0)) + # self.viser_server.update_scene_box(scene_box) + # crop_scale = crop_max - crop_min + # crop_center = (crop_max + crop_min) / 2.0 + # self.viser_server.send_crop_params( + # crop_enabled=self.control_panel.crop_viewport, + # crop_bg_color=self.control_panel.background_color, + # crop_scale=tuple(crop_scale.tolist()), + # crop_center=tuple(crop_center.tolist()), + # ) + # if self.camera_message is not None: + # self.render_statemachine.action(RenderAction("rerender", self.camera_message)) + + def _output_type_change(self, _): + self.output_type_changed = True + + def _output_split_type_change(self, _): + self.output_split_type_changed = True + def _pick_drawn_image_idxs(self, total_num: int) -> list[int]: """Determine indicies of images to display in viewer. @@ -161,7 +202,6 @@ class ViewerState: self.train_state = train_state self.train_util = 0.9 - self.max_res = 512 def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None: """updates the scene based on the graph weights @@ -183,7 +223,7 @@ class ViewerState: train_s = GLOBAL_BUFFER["events"][EventName.TRAIN_RAYS_PER_SEC.value]["avg"] vis_s = GLOBAL_BUFFER["events"][EventName.VIS_RAYS_PER_SEC.value]["avg"] train_util = self.train_util - vis_n = self.max_res**2 + vis_n = self.control_panel.max_res**2 train_n = num_rays_per_batch train_time = train_n / train_s vis_time = vis_n / vis_s @@ -195,6 +235,28 @@ class ViewerState: self.last_step = step self.render_statemachine.action(RenderAction("step", self.camera_state)) + def update_colormap_options(self, dimensions: int, dtype: type) -> None: + """update the colormap options based on the current render + + Args: + dimensions: the number of dimensions of the render + dtype: the data type of the render + """ + if self.output_type_changed: + self.control_panel.update_colormap_options(dimensions, dtype) + self.output_type_changed = False + + def update_split_colormap_options(self, dimensions: int, dtype: type) -> None: + """update the colormap options based on the current render + + Args: + dimensions: the number of dimensions of the render + dtype: the data type of the render + """ + if self.output_split_type_changed: + self.control_panel.update_split_colormap_options(dimensions, dtype) + self.output_split_type_changed = False + def get_model(self) -> Model: """Returns the model.""" return self.pipeline.model diff --git a/nerfstudio/viewer_beta/viewer_elements.py b/nerfstudio/viewer_beta/viewer_elements.py new file mode 100644 index 000000000..5fbfc8598 --- /dev/null +++ b/nerfstudio/viewer_beta/viewer_elements.py @@ -0,0 +1,440 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" Viewer GUI elements for the nerfstudio viewer """ + + +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, Callable, Generic, List, Optional, Tuple, Union + +from typing_extensions import LiteralString, TypeVar +from viser import ( + GuiButtonGroupHandle, + GuiButtonHandle, + GuiDropdownHandle, + GuiHandle, + ViserServer, +) + +TValue = TypeVar("TValue") +TString = TypeVar("TString", default=str, bound=str) + + +class ViewerElement(Generic[TValue]): + """Base class for all viewer elements + + Args: + name: The name of the element + disabled: If the element is disabled + """ + + def __init__( + self, + name: str, + disabled: bool = False, + cb_hook: Callable = lambda element: None, + ) -> None: + self.name = name + self.gui_handle: Optional[Union[GuiHandle[TValue], GuiButtonHandle, GuiButtonGroupHandle]] = None + self.disabled = disabled + self.cb_hook = cb_hook + + @abstractmethod + def _create_gui_handle(self, viser_server: ViserServer) -> None: + """ + Returns the GuiHandle object which actually controls the parameter in the gui. + + Args: + viser_server: The server to install the gui element into. + """ + ... + + def remove(self) -> None: + """Removes the gui element from the viewer""" + if self.gui_handle is not None: + self.gui_handle.remove() + self.gui_handle = None + + def set_hidden(self, hidden: bool) -> None: + """Sets the hidden state of the gui element""" + assert self.gui_handle is not None + self.gui_handle.visible = not hidden + + def set_disabled(self, disabled: bool) -> None: + """Sets the disabled state of the gui element""" + assert self.gui_handle is not None + self.gui_handle.disabled = disabled + + @abstractmethod + def install(self, viser_server: ViserServer) -> None: + """Installs the gui element into the given viser_server""" + ... + + +class ViewerButton(ViewerElement[bool]): + """A button in the viewer + + Args: + name: The name of the button + cb_hook: The function to call when the button is pressed + disabled: If the button is disabled + """ + + gui_handle: GuiButtonHandle + + def __init__(self, name: str, cb_hook: Callable[[ViewerButton], Any], disabled: bool = False): + super().__init__(name, disabled=disabled, cb_hook=cb_hook) + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + self.gui_handle = viser_server.add_gui_button(name=self.name, disabled=self.disabled) + + def install(self, viser_server: ViserServer) -> None: + self._create_gui_handle(viser_server) + + assert self.gui_handle is not None + self.gui_handle.on_click(lambda _: self.cb_hook(self)) + + +class ViewerParameter(ViewerElement[TValue], Generic[TValue]): + """A viewer element with state + + Args: + name: The name of the element + default_value: The default value of the element + disabled: If the element is disabled + cb_hook: Callback to call on update + """ + + gui_handle: GuiHandle + + def __init__( + self, + name: str, + default_value: TValue, + disabled: bool = False, + cb_hook: Callable = lambda element: None, + ) -> None: + super().__init__(name, disabled=disabled, cb_hook=cb_hook) + self.default_value = default_value + + def install(self, viser_server: ViserServer) -> None: + """ + Based on the type provided by default_value, installs a gui element inside the given viser_server + + Args: + viser_server: The server to install the gui element into. + """ + self._create_gui_handle(viser_server) + + assert self.gui_handle is not None + self.gui_handle.on_update(lambda _: self.cb_hook(self)) + + @abstractmethod + def _create_gui_handle(self, viser_server: ViserServer) -> None: + ... + + @property + def value(self) -> TValue: + """Returns the current value of the viewer element""" + if self.gui_handle is None: + return self.default_value + return self.gui_handle.value + + @value.setter + def value(self, value: TValue) -> None: + if self.gui_handle is not None: + self.gui_handle.value = value + else: + self.default_value = value + + +IntOrFloat = TypeVar("IntOrFloat", int, float) + + +class ViewerSlider(ViewerParameter[IntOrFloat], Generic[IntOrFloat]): + """A slider in the viewer + + Args: + name: The name of the slider + default_value: The default value of the slider + min_value: The minimum value of the slider + max_value: The maximum value of the slider + step: The step size of the slider + disabled: If the slider is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + def __init__( + self, + name: str, + default_value: IntOrFloat, + min_value: IntOrFloat, + max_value: IntOrFloat, + step: IntOrFloat = 0.1, + disabled: bool = False, + cb_hook: Callable[[ViewerSlider], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert isinstance(default_value, (float, int)) + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.min = min_value + self.max = max_value + self.step = step + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + assert self.gui_handle is None, "gui_handle should be initialized once" + self.gui_handle = viser_server.add_gui_slider( + self.name, self.min, self.max, self.step, self.default_value, disabled=self.disabled, hint=self.hint + ) + + +class ViewerText(ViewerParameter[str]): + """A text field in the viewer + + Args: + name: The name of the text field + default_value: The default value of the text field + disabled: If the text field is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + def __init__( + self, + name: str, + default_value: str, + disabled: bool = False, + cb_hook: Callable[[ViewerText], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert isinstance(default_value, str) + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + assert self.gui_handle is None, "gui_handle should be initialized once" + self.gui_handle = viser_server.add_gui_text( + self.name, self.default_value, disabled=self.disabled, hint=self.hint + ) + + +class ViewerNumber(ViewerParameter[IntOrFloat], Generic[IntOrFloat]): + """A number field in the viewer + + Args: + name: The name of the number field + default_value: The default value of the number field + disabled: If the number field is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + default_value: IntOrFloat + + def __init__( + self, + name: str, + default_value: IntOrFloat, + disabled: bool = False, + cb_hook: Callable[[ViewerNumber], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert isinstance(default_value, (float, int)) + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + assert self.gui_handle is None, "gui_handle should be initialized once" + self.gui_handle = viser_server.add_gui_number( + self.name, self.default_value, disabled=self.disabled, hint=self.hint + ) + + +class ViewerCheckbox(ViewerParameter[bool]): + """A checkbox in the viewer + + Args: + name: The name of the checkbox + default_value: The default value of the checkbox + disabled: If the checkbox is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + def __init__( + self, + name: str, + default_value: bool, + disabled: bool = False, + cb_hook: Callable[[ViewerCheckbox], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert isinstance(default_value, bool) + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + assert self.gui_handle is None, "gui_handle should be initialized once" + self.gui_handle = viser_server.add_gui_checkbox( + self.name, self.default_value, disabled=self.disabled, hint=self.hint + ) + + +TLiteralString = TypeVar("TLiteralString", bound=LiteralString) + + +class ViewerDropdown(ViewerParameter[TString], Generic[TString]): + """A dropdown in the viewer + + Args: + name: The name of the dropdown + default_value: The default value of the dropdown + options: The options of the dropdown + disabled: If the dropdown is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + gui_handle: Optional[GuiDropdownHandle[TString]] + + def __init__( + self, + name: str, + default_value: TString, + options: List[TString], + disabled: bool = False, + cb_hook: Callable[[ViewerDropdown], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert default_value in options + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.options = options + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + assert self.gui_handle is None, "gui_handle should be initialized once" + self.gui_handle = viser_server.add_gui_dropdown( + self.name, self.options, self.default_value, disabled=self.disabled, hint=self.hint # type: ignore + ) + + def set_options(self, new_options: List[TString]) -> None: + """ + Sets the options of the dropdown, + + Args: + new_options: The new options. If the current option isn't in the new options, the first option is selected. + """ + self.options = new_options + if self.gui_handle is not None: + self.gui_handle.options = new_options + + +class ViewerButtonGroup(ViewerParameter[TString], Generic[TString]): + """A button group in the viewer. Unlike other fields, cannot be disabled. + + Args: + name: The name of the button group + options: The options of the button group + cb_hook: Callback to call on update + """ + + gui_handle: GuiButtonGroupHandle + + def __init__( + self, + name: str, + default_value: TString, + options: List[TString], + cb_hook: Callable[[ViewerDropdown], Any] = lambda element: None, + ): + super().__init__(name, disabled=False, default_value=default_value, cb_hook=cb_hook) + self.options = options + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + assert self.gui_handle is None, "gui_handle should be initialized once" + self.gui_handle = viser_server.add_gui_button_group(self.name, self.options) + + def install(self, viser_server: ViserServer) -> None: + self._create_gui_handle(viser_server) + + assert self.gui_handle is not None + self.gui_handle.on_click(lambda _: self.cb_hook(self)) + + +class ViewerRGB(ViewerParameter[Tuple[int, int, int]]): + """ + An RGB color picker for the viewer + + Args: + name: The name of the color picker + default_value: The default value of the color picker + disabled: If the color picker is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + def __init__( + self, + name, + default_value: Tuple[int, int, int], + disabled=False, + cb_hook: Callable[[ViewerRGB], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert len(default_value) == 3 + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + self.gui_handle = viser_server.add_gui_rgb( + self.name, self.default_value, disabled=self.disabled, hint=self.hint + ) + + +class ViewerVec3(ViewerParameter[Tuple[float, float, float]]): + """ + 3 number boxes in a row to input a vector + + Args: + name: The name of the vector + default_value: The default value of the vector + step: The step of the vector + disabled: If the vector is disabled + cb_hook: Callback to call on update + hint: The hint text + """ + + def __init__( + self, + name, + default_value: Tuple[float, float, float], + step=0.1, + disabled=False, + cb_hook: Callable[[ViewerVec3], Any] = lambda element: None, + hint: Optional[str] = None, + ): + assert len(default_value) == 3 + super().__init__(name, default_value, disabled=disabled, cb_hook=cb_hook) + self.step = step + self.hint = hint + + def _create_gui_handle(self, viser_server: ViserServer) -> None: + self.gui_handle = viser_server.add_gui_vector3( + self.name, self.default_value, self.step, disabled=self.disabled, hint=self.hint + ) diff --git a/pyproject.toml b/pyproject.toml index c66c34b04..9765a988f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ "torchvision>=0.14.1", "torchmetrics[image]>=0.9.3", "typing_extensions>=4.4.0", - "viser>=0.0.10", + "viser>=0.0.12", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas",