Files
nerfstudio/nerfstudio/viewer_beta/viewer.py
T

409 lines
17 KiB
Python

# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Manage the state of the viewer """
from __future__ import annotations
import threading
import time
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Literal, Optional
import numpy as np
import torch
import torchvision
import viser
import viser.theme
import viser.transforms as vtf
from nerfstudio.cameras.camera_optimizers import CameraOptimizer
from nerfstudio.configs import base_config as cfg
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.models.base_model import Model
from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils.decorators import check_main_thread, decorate_all
from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName
from nerfstudio.viewer.server import viewer_utils
from nerfstudio.viewer_beta.control_panel import ControlPanel
from nerfstudio.viewer_beta.export_panel import populate_export_tab
from nerfstudio.viewer_beta.render_panel import populate_render_tab
from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStateMachine
from nerfstudio.viewer_beta.utils import CameraState, parse_object
from nerfstudio.viewer_beta.viewer_elements import ViewerControl, ViewerElement
if TYPE_CHECKING:
from nerfstudio.engine.trainer import Trainer
VISER_NERFSTUDIO_SCALE_RATIO: float = 10.0
@decorate_all([check_main_thread])
class Viewer:
"""Class to hold state for viewer variables
Args:
config: viewer setup configuration
log_filename: filename to log viewer output to
datapath: path to data
pipeline: pipeline object to use
trainer: trainer object to use
share: print a shareable URL
Attributes:
viewer_url: url to open viewer
viser_server: the viser server
"""
viewer_url: str
viser_server: viser.ViserServer
camera_state: Optional[CameraState] = None
def __init__(
self,
config: cfg.ViewerConfig,
log_filename: Path,
datapath: Path,
pipeline: Pipeline,
trainer: Optional[Trainer] = None,
train_lock: Optional[threading.Lock] = None,
share: bool = False,
):
self.config = config
self.trainer = trainer
self.last_step = 0
self.train_lock = train_lock
self.pipeline = pipeline
self.log_filename = log_filename
self.datapath = datapath.parent if datapath.is_file() else datapath
self.include_time = self.pipeline.datamanager.includes_time
if self.config.websocket_port is None:
websocket_port = viewer_utils.get_free_port(default_port=self.config.websocket_port_default)
else:
websocket_port = self.config.websocket_port
self.log_filename.parent.mkdir(exist_ok=True)
self.viewer_url = viewer_utils.get_viewer_url(websocket_port)
# viewer specific variables
self.output_type_changed = True
self.output_split_type_changed = True
self.step = 0
self.train_btn_state: Literal["training", "paused", "completed"] = "training"
self._prev_train_state: Literal["training", "paused", "completed"] = "training"
self.client: Optional[viser.ClientHandle] = None
self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port, share=share)
buttons = (
viser.theme.TitlebarButton(
text="Getting Started",
icon=None,
href="https://nerf.studio",
),
viser.theme.TitlebarButton(
text="Github",
icon="GitHub",
href="https://github.com/nerfstudio-project/nerfstudio",
),
viser.theme.TitlebarButton(
text="Documentation",
icon="Description",
href="https://docs.nerf.studio",
),
)
image = viser.theme.TitlebarImage(
image_url_light="https://docs.nerf.studio/_static/imgs/logo.png",
image_url_dark="https://docs.nerf.studio/_static/imgs/logo-dark.png",
image_alt="NerfStudio Logo",
href="https://docs.nerf.studio/",
)
titlebar_theme = viser.theme.TitlebarConfig(buttons=buttons, image=image)
self.viser_server.configure_theme(
titlebar_content=titlebar_theme,
control_layout="collapsible",
dark_mode=True,
brand_color=(255, 211, 105),
)
self.render_statemachine = RenderStateMachine(self, VISER_NERFSTUDIO_SCALE_RATIO)
self.viser_server.on_client_connect(self.handle_new_client)
tabs = self.viser_server.add_gui_tab_group()
control_tab = tabs.add_tab("Control", viser.Icon.SETTINGS)
with control_tab:
self.control_panel = ControlPanel(
self.viser_server,
self.include_time,
VISER_NERFSTUDIO_SCALE_RATIO,
self._interrupt_render,
self._crop_params_update,
self._output_type_change,
self._output_split_type_change,
self._toggle_training_state,
self.set_camera_visibility,
)
config_path = self.log_filename.parents[0] / "config.yml"
with tabs.add_tab("Render", viser.Icon.CAMERA):
populate_render_tab(self.viser_server, config_path, self.datapath, self.control_panel)
with tabs.add_tab("Export", viser.Icon.PACKAGE_EXPORT):
populate_export_tab(self.viser_server, self.control_panel, config_path)
# Keep track of the pointers to generated GUI folders, because each generated folder holds a unique ID.
viewer_gui_folders = dict()
def nested_folder_install(folder_labels: List[str], prev_labels: List[str], element: ViewerElement):
if len(folder_labels) == 0:
element.install(self.viser_server)
# also rewire the hook to rerender
prev_cb = element.cb_hook
element.cb_hook = lambda element: [prev_cb(element), self._interrupt_render(element)]
else:
# recursively create folders
# If the folder name is "Custom Elements/a/b", then:
# in the beginning: folder_path will be
# "/".join([] + ["Custom Elements"]) --> "Custom Elements"
# later, folder_path will be
# "/".join(["Custom Elements"] + ["a"]) --> "Custom Elements/a"
# "/".join(["Custom Elements", "a"] + ["b"]) --> "Custom Elements/a/b"
# --> the element will be installed in the folder "Custom Elements/a/b"
#
# Note that the gui_folder is created only when the folder is not in viewer_gui_folders,
# and we use the folder_path as the key to check if the folder is already created.
# Otherwise, use the existing folder as context manager.
folder_path = "/".join(prev_labels + [folder_labels[0]])
if folder_path not in viewer_gui_folders:
viewer_gui_folders[folder_path] = self.viser_server.add_gui_folder(folder_labels[0])
with viewer_gui_folders[folder_path]:
nested_folder_install(folder_labels[1:], prev_labels + [folder_labels[0]], element)
with control_tab:
self.viewer_elements = []
self.viewer_elements.extend(parse_object(pipeline, ViewerElement, "Custom Elements"))
for param_path, element in self.viewer_elements:
folder_labels = param_path.split("/")[:-1]
nested_folder_install(folder_labels, [], element)
# scrape the trainer/pipeline for any ViewerControl objects to initialize them
self.viewer_controls: List[ViewerControl] = [
e for (_, e) in parse_object(self.trainer, ViewerControl, "Custom Elements")
]
for c in self.viewer_controls:
c._setup(self)
self.render_statemachine.start()
def handle_new_client(self, client: viser.ClientHandle) -> None:
self.client = client
self.last_move_time = 0
@client.camera.on_update
def _(cam: viser.CameraHandle) -> None:
assert self.client is not None
with client.atomic():
self.last_move_time = time.time()
R = vtf.SO3(wxyz=self.client.camera.wxyz)
R = R @ vtf.SO3.from_x_radians(np.pi)
R = torch.tensor(R.as_matrix())
pos = torch.tensor(self.client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO
c2w = torch.concatenate([R, pos[:, None]], dim=1)
self.camera_state = CameraState(fov=self.client.camera.fov, aspect=self.client.camera.aspect, c2w=c2w)
self.render_statemachine.action(RenderAction("move", self.camera_state))
def set_camera_visibility(self, visible: bool) -> None:
"""Toggle the visibility of the training cameras."""
with self.viser_server.atomic():
for idx in self.camera_handles:
self.camera_handles[idx].visible = visible
def update_camera_poses(self):
# Update the train camera locations based on optimization
assert self.camera_handles is not None
if hasattr(self.pipeline.datamanager, "train_camera_optimizer"):
camera_optimizer = self.pipeline.datamanager.train_camera_optimizer
elif hasattr(self.pipeline.model, "camera_optimizer"):
camera_optimizer = self.pipeline.model.camera_optimizer
else:
return
idxs = list(self.camera_handles.keys())
with torch.no_grad():
assert isinstance(camera_optimizer, CameraOptimizer)
c2ws_delta = camera_optimizer(torch.tensor(idxs, device=camera_optimizer.device)).cpu().numpy()
for idx in idxs:
# both are numpy arrays
c2w_orig = self.original_c2w[idx]
c2w_delta = c2ws_delta[idx, ...]
c2w = c2w_orig @ np.concatenate((c2w_delta, np.array([[0, 0, 0, 1]])), axis=0)
R = vtf.SO3.from_matrix(c2w[:3, :3]) # type: ignore
R = R @ vtf.SO3.from_x_radians(np.pi)
self.camera_handles[idx].position = c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO
self.camera_handles[idx].wxyz = R.wxyz
def _interrupt_render(self, _) -> None:
"""Interrupt current render."""
if self.camera_state is not None:
self.render_statemachine.action(RenderAction("rerender", self.camera_state))
def _toggle_training_state(self, _) -> None:
"""Toggle the trainer's training state."""
if self.trainer is not None:
if self.trainer.training_state == "training":
self.trainer.training_state = "paused"
elif self.trainer.training_state == "paused":
self.trainer.training_state = "training"
def _crop_params_update(self, _) -> None:
"""Update crop parameters"""
if self.camera_state is not None:
self.render_statemachine.action(RenderAction("move", self.camera_state))
def _output_type_change(self, _):
self.output_type_changed = True
def _output_split_type_change(self, _):
self.output_split_type_changed = True
def _pick_drawn_image_idxs(self, total_num: int) -> list[int]:
"""Determine indicies of images to display in viewer.
Args:
total_num: total number of training images.
Returns:
List of indices from [0, total_num-1].
"""
if self.config.max_num_display_images < 0:
num_display_images = total_num
else:
num_display_images = min(self.config.max_num_display_images, total_num)
# draw indices, roughly evenly spaced
return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist()
def init_scene(
self,
train_dataset: InputDataset,
train_state: Literal["training", "paused", "completed"],
eval_dataset: Optional[InputDataset] = None,
) -> None:
"""Draw some images and the scene aabb in the viewer.
Args:
dataset: dataset to render in the scene
train_state: Current status of training
"""
# draw the training cameras and images
self.camera_handles: Dict[int, viser.CameraFrustumHandle] = {}
self.original_c2w: Dict[int, np.ndarray] = {}
image_indices = self._pick_drawn_image_idxs(len(train_dataset))
for idx in image_indices:
image = train_dataset[idx]["image"]
camera = train_dataset.cameras[idx]
image_uint8 = (image * 255).detach().type(torch.uint8)
image_uint8 = image_uint8.permute(2, 0, 1)
image_uint8 = torchvision.transforms.functional.resize(image_uint8, 100) # type: ignore
image_uint8 = image_uint8.permute(1, 2, 0)
image_uint8 = image_uint8.cpu().numpy()
c2w = camera.camera_to_worlds.cpu().numpy()
R = vtf.SO3.from_matrix(c2w[:3, :3])
R = R @ vtf.SO3.from_x_radians(np.pi)
camera_handle = self.viser_server.add_camera_frustum(
name=f"/cameras/camera_{idx:05d}",
fov=float(2 * np.arctan(camera.cx / camera.fx[0])),
scale=0.1,
aspect=float(camera.cx[0] / camera.cy[0]),
image=image_uint8,
wxyz=R.wxyz,
position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO,
)
@camera_handle.on_click
def _(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None:
assert self.client is not None
with self.client.atomic():
self.client.camera.position = event.target.position
self.client.camera.wxyz = event.target.wxyz
self.camera_handles[idx] = camera_handle
self.original_c2w[idx] = c2w
self.train_state = train_state
self.train_util = 0.9
def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None:
"""updates the scene based on the graph weights
Args:
step: iteration step of training
num_rays_per_batch: number of rays per batch, used during training
"""
self.step = step
if self.camera_state is None:
return
# this stops training while moving to make the response smoother
while time.time() - self.last_move_time < 0.1:
time.sleep(0.05)
# self.render_statemachine.action(RenderAction("static", self.camera_state))
if self.trainer is not None and self.trainer.training_state == "training" and self.train_util != 1:
if (
EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
):
train_s = GLOBAL_BUFFER["events"][EventName.TRAIN_RAYS_PER_SEC.value]["avg"]
vis_s = GLOBAL_BUFFER["events"][EventName.VIS_RAYS_PER_SEC.value]["avg"]
train_util = self.train_util
vis_n = self.control_panel.max_res**2
train_n = num_rays_per_batch
train_time = train_n / train_s
vis_time = vis_n / vis_s
render_freq = train_util * vis_time / (train_time - train_util * train_time)
else:
render_freq = 30
if step > self.last_step + render_freq:
self.last_step = step
self.render_statemachine.action(RenderAction("step", self.camera_state))
self.update_camera_poses()
self.control_panel.update_step(step)
def update_colormap_options(self, dimensions: int, dtype: type) -> None:
"""update the colormap options based on the current render
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
"""
if self.output_type_changed:
self.control_panel.update_colormap_options(dimensions, dtype)
self.output_type_changed = False
def update_split_colormap_options(self, dimensions: int, dtype: type) -> None:
"""update the colormap options based on the current render
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
"""
if self.output_split_type_changed:
self.control_panel.update_split_colormap_options(dimensions, dtype)
self.output_split_type_changed = False
def get_model(self) -> Model:
"""Returns the model."""
return self.pipeline.model
def training_complete(self) -> None:
"""Called when training is complete."""
self.training_state = "completed"