Files
nerfstudio/nerfstudio/viewer_beta/viewer.py
T
2023-06-17 21:15:33 -07:00

267 lines
10 KiB
Python

# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Manage the state of the viewer """
from __future__ import annotations
import threading
from pathlib import Path
from typing import TYPE_CHECKING, Literal, Optional
import numpy as np
import torch
import torchvision
import viser
import viser.transforms as vtf
from nerfstudio.configs import base_config as cfg
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.models.base_model import Model
from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils.decorators import check_main_thread, decorate_all
from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName
from nerfstudio.viewer.server import viewer_utils
from nerfstudio.viewer_beta.control_panel import ControlPanel
from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStateMachine
from nerfstudio.viewer_beta.utils import CameraState
if TYPE_CHECKING:
from nerfstudio.engine.trainer import Trainer
VISER_NERFSTUDIO_SCALE_RATIO: int = 10
@decorate_all([check_main_thread])
class Viewer:
"""Class to hold state for viewer variables
Args:
config: viewer setup configuration
log_filename: filename to log viewer output to
datapath: path to data
pipeline: pipeline object to use
trainer: trainer object to use
Attributes:
viewer_url: url to open viewer
viser_server: the viser server
"""
viewer_url: str
viser_server: viser.ViserServer
camera_state: Optional[CameraState] = None
def __init__(
self,
config: cfg.ViewerConfig,
log_filename: Path,
datapath: Path,
pipeline: Pipeline,
trainer: Optional[Trainer] = None,
train_lock: Optional[threading.Lock] = None,
):
self.config = config
self.trainer = trainer
self.last_step = 0
self.train_lock = train_lock
self.pipeline = pipeline
self.log_filename = log_filename
self.datapath = datapath.parent if datapath.is_file() else datapath
self.include_time = self.pipeline.datamanager.includes_time
if self.config.websocket_port is None:
websocket_port = viewer_utils.get_free_port(default_port=self.config.websocket_port_default)
else:
websocket_port = self.config.websocket_port
self.log_filename.parent.mkdir(exist_ok=True)
self.viewer_url = viewer_utils.get_viewer_url(websocket_port)
# viewer specific variables
self.output_type_changed = True
self.output_split_type_changed = True
self.step = 0
self.train_btn_state: Literal["training", "paused", "completed"] = "training"
self._prev_train_state: Literal["training", "paused", "completed"] = "training"
self.camera_message = None
self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port)
self.viser_server.on_client_connect(self.handle_new_client)
self.control_panel = ControlPanel(
self.viser_server,
self.include_time,
self._interrupt_render,
self._crop_params_update,
self._output_type_change,
self._output_split_type_change,
)
self.render_statemachine = RenderStateMachine(self)
self.render_statemachine.start()
def handle_new_client(self, client: viser.ClientHandle) -> None:
@client.camera.on_update
def _(_: viser.CameraHandle) -> None:
R = vtf.SO3(wxyz=client.camera.wxyz)
R = R @ vtf.SO3.from_x_radians(np.pi)
R = torch.tensor(R.as_matrix())
pos = torch.tensor(client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO
c2w = torch.concatenate([R, pos[:, None]], dim=1)
self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w)
self.render_statemachine.action(RenderAction("move", self.camera_state))
def _interrupt_render(self, _) -> None:
"""Interrupt current render."""
if self.camera_state is not None:
self.render_statemachine.action(RenderAction("rerender", self.camera_state))
def _crop_params_update(self, _) -> None:
"""Update crop parameters"""
print("Crop params not set up")
# crop_min = torch.tensor(self.control_panel.crop_min, dtype=torch.float32)
# crop_max = torch.tensor(self.control_panel.crop_max, dtype=torch.float32)
# scene_box = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0))
# self.viser_server.update_scene_box(scene_box)
# crop_scale = crop_max - crop_min
# crop_center = (crop_max + crop_min) / 2.0
# self.viser_server.send_crop_params(
# crop_enabled=self.control_panel.crop_viewport,
# crop_bg_color=self.control_panel.background_color,
# crop_scale=tuple(crop_scale.tolist()),
# crop_center=tuple(crop_center.tolist()),
# )
# if self.camera_message is not None:
# self.render_statemachine.action(RenderAction("rerender", self.camera_message))
def _output_type_change(self, _):
self.output_type_changed = True
def _output_split_type_change(self, _):
self.output_split_type_changed = True
def _pick_drawn_image_idxs(self, total_num: int) -> list[int]:
"""Determine indicies of images to display in viewer.
Args:
total_num: total number of training images.
Returns:
List of indices from [0, total_num-1].
"""
if self.config.max_num_display_images < 0:
num_display_images = total_num
else:
num_display_images = min(self.config.max_num_display_images, total_num)
# draw indices, roughly evenly spaced
return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist()
def init_scene(self, dataset: InputDataset, train_state: Literal["training", "paused", "completed"]) -> None:
"""Draw some images and the scene aabb in the viewer.
Args:
dataset: dataset to render in the scene
train_state: Current status of training
"""
# draw the training cameras and images
image_indices = self._pick_drawn_image_idxs(len(dataset))
for idx in image_indices:
image = dataset[idx]["image"]
camera = dataset.cameras[idx]
image_uint8 = (image * 255).detach().type(torch.uint8)
image_uint8 = image_uint8.permute(2, 0, 1)
image_uint8 = torchvision.transforms.functional.resize(image_uint8, 100) # type: ignore
image_uint8 = image_uint8.permute(1, 2, 0)
image_uint8 = image_uint8.cpu().numpy()
c2w = camera.camera_to_worlds.cpu().numpy()
R = vtf.SO3.from_matrix(c2w[:3, :3])
R = R @ vtf.SO3.from_x_radians(np.pi)
self.viser_server.add_camera_frustum(
name=f"Camera {idx}",
fov=float(camera.fx[0]),
scale=0.2,
aspect=float(camera.cx[0] / camera.cy[0]),
image=image_uint8,
wxyz=R.wxyz,
position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO,
)
self.train_state = train_state
self.train_util = 0.9
def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None:
"""updates the scene based on the graph weights
Args:
step: iteration step of training
num_rays_per_batch: number of rays per batch, used during training
"""
self.step = step
if self.camera_state is None:
return
if self.trainer is not None and self.trainer.training_state == "training" and self.train_util != 1:
if (
EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
):
train_s = GLOBAL_BUFFER["events"][EventName.TRAIN_RAYS_PER_SEC.value]["avg"]
vis_s = GLOBAL_BUFFER["events"][EventName.VIS_RAYS_PER_SEC.value]["avg"]
train_util = self.train_util
vis_n = self.control_panel.max_res**2
train_n = num_rays_per_batch
train_time = train_n / train_s
vis_time = vis_n / vis_s
render_freq = train_util * vis_time / (train_time - train_util * train_time)
else:
render_freq = 30
if step > self.last_step + render_freq:
self.last_step = step
self.render_statemachine.action(RenderAction("step", self.camera_state))
def update_colormap_options(self, dimensions: int, dtype: type) -> None:
"""update the colormap options based on the current render
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
"""
if self.output_type_changed:
self.control_panel.update_colormap_options(dimensions, dtype)
self.output_type_changed = False
def update_split_colormap_options(self, dimensions: int, dtype: type) -> None:
"""update the colormap options based on the current render
Args:
dimensions: the number of dimensions of the render
dtype: the data type of the render
"""
if self.output_split_type_changed:
self.control_panel.update_split_colormap_options(dimensions, dtype)
self.output_split_type_changed = False
def get_model(self) -> Model:
"""Returns the model."""
return self.pipeline.model
def training_complete(self) -> None:
"""Called when training is complete."""
self.training_state = "completed"