mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
267 lines
10 KiB
Python
267 lines
10 KiB
Python
# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
""" Manage the state of the viewer """
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Literal, Optional
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torchvision
|
|
import viser
|
|
import viser.transforms as vtf
|
|
|
|
from nerfstudio.configs import base_config as cfg
|
|
from nerfstudio.data.datasets.base_dataset import InputDataset
|
|
from nerfstudio.models.base_model import Model
|
|
from nerfstudio.pipelines.base_pipeline import Pipeline
|
|
from nerfstudio.utils.decorators import check_main_thread, decorate_all
|
|
from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName
|
|
from nerfstudio.viewer.server import viewer_utils
|
|
from nerfstudio.viewer_beta.control_panel import ControlPanel
|
|
from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStateMachine
|
|
from nerfstudio.viewer_beta.utils import CameraState
|
|
|
|
if TYPE_CHECKING:
|
|
from nerfstudio.engine.trainer import Trainer
|
|
|
|
VISER_NERFSTUDIO_SCALE_RATIO: int = 10
|
|
|
|
|
|
@decorate_all([check_main_thread])
|
|
class Viewer:
|
|
"""Class to hold state for viewer variables
|
|
|
|
Args:
|
|
config: viewer setup configuration
|
|
log_filename: filename to log viewer output to
|
|
datapath: path to data
|
|
pipeline: pipeline object to use
|
|
trainer: trainer object to use
|
|
|
|
Attributes:
|
|
viewer_url: url to open viewer
|
|
viser_server: the viser server
|
|
"""
|
|
|
|
viewer_url: str
|
|
viser_server: viser.ViserServer
|
|
camera_state: Optional[CameraState] = None
|
|
|
|
def __init__(
|
|
self,
|
|
config: cfg.ViewerConfig,
|
|
log_filename: Path,
|
|
datapath: Path,
|
|
pipeline: Pipeline,
|
|
trainer: Optional[Trainer] = None,
|
|
train_lock: Optional[threading.Lock] = None,
|
|
):
|
|
self.config = config
|
|
self.trainer = trainer
|
|
self.last_step = 0
|
|
self.train_lock = train_lock
|
|
self.pipeline = pipeline
|
|
self.log_filename = log_filename
|
|
self.datapath = datapath.parent if datapath.is_file() else datapath
|
|
self.include_time = self.pipeline.datamanager.includes_time
|
|
|
|
if self.config.websocket_port is None:
|
|
websocket_port = viewer_utils.get_free_port(default_port=self.config.websocket_port_default)
|
|
else:
|
|
websocket_port = self.config.websocket_port
|
|
self.log_filename.parent.mkdir(exist_ok=True)
|
|
|
|
self.viewer_url = viewer_utils.get_viewer_url(websocket_port)
|
|
|
|
# viewer specific variables
|
|
self.output_type_changed = True
|
|
self.output_split_type_changed = True
|
|
self.step = 0
|
|
self.train_btn_state: Literal["training", "paused", "completed"] = "training"
|
|
self._prev_train_state: Literal["training", "paused", "completed"] = "training"
|
|
|
|
self.camera_message = None
|
|
|
|
self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port)
|
|
|
|
self.viser_server.on_client_connect(self.handle_new_client)
|
|
|
|
self.control_panel = ControlPanel(
|
|
self.viser_server,
|
|
self.include_time,
|
|
self._interrupt_render,
|
|
self._crop_params_update,
|
|
self._output_type_change,
|
|
self._output_split_type_change,
|
|
)
|
|
|
|
self.render_statemachine = RenderStateMachine(self)
|
|
self.render_statemachine.start()
|
|
|
|
def handle_new_client(self, client: viser.ClientHandle) -> None:
|
|
@client.camera.on_update
|
|
def _(_: viser.CameraHandle) -> None:
|
|
R = vtf.SO3(wxyz=client.camera.wxyz)
|
|
R = R @ vtf.SO3.from_x_radians(np.pi)
|
|
R = torch.tensor(R.as_matrix())
|
|
pos = torch.tensor(client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO
|
|
c2w = torch.concatenate([R, pos[:, None]], dim=1)
|
|
self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w)
|
|
self.render_statemachine.action(RenderAction("move", self.camera_state))
|
|
|
|
def _interrupt_render(self, _) -> None:
|
|
"""Interrupt current render."""
|
|
if self.camera_state is not None:
|
|
self.render_statemachine.action(RenderAction("rerender", self.camera_state))
|
|
|
|
def _crop_params_update(self, _) -> None:
|
|
"""Update crop parameters"""
|
|
print("Crop params not set up")
|
|
# crop_min = torch.tensor(self.control_panel.crop_min, dtype=torch.float32)
|
|
# crop_max = torch.tensor(self.control_panel.crop_max, dtype=torch.float32)
|
|
# scene_box = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0))
|
|
# self.viser_server.update_scene_box(scene_box)
|
|
# crop_scale = crop_max - crop_min
|
|
# crop_center = (crop_max + crop_min) / 2.0
|
|
# self.viser_server.send_crop_params(
|
|
# crop_enabled=self.control_panel.crop_viewport,
|
|
# crop_bg_color=self.control_panel.background_color,
|
|
# crop_scale=tuple(crop_scale.tolist()),
|
|
# crop_center=tuple(crop_center.tolist()),
|
|
# )
|
|
# if self.camera_message is not None:
|
|
# self.render_statemachine.action(RenderAction("rerender", self.camera_message))
|
|
|
|
def _output_type_change(self, _):
|
|
self.output_type_changed = True
|
|
|
|
def _output_split_type_change(self, _):
|
|
self.output_split_type_changed = True
|
|
|
|
def _pick_drawn_image_idxs(self, total_num: int) -> list[int]:
|
|
"""Determine indicies of images to display in viewer.
|
|
|
|
Args:
|
|
total_num: total number of training images.
|
|
|
|
Returns:
|
|
List of indices from [0, total_num-1].
|
|
"""
|
|
if self.config.max_num_display_images < 0:
|
|
num_display_images = total_num
|
|
else:
|
|
num_display_images = min(self.config.max_num_display_images, total_num)
|
|
# draw indices, roughly evenly spaced
|
|
return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist()
|
|
|
|
def init_scene(self, dataset: InputDataset, train_state: Literal["training", "paused", "completed"]) -> None:
|
|
"""Draw some images and the scene aabb in the viewer.
|
|
|
|
Args:
|
|
dataset: dataset to render in the scene
|
|
train_state: Current status of training
|
|
"""
|
|
|
|
# draw the training cameras and images
|
|
image_indices = self._pick_drawn_image_idxs(len(dataset))
|
|
for idx in image_indices:
|
|
image = dataset[idx]["image"]
|
|
camera = dataset.cameras[idx]
|
|
image_uint8 = (image * 255).detach().type(torch.uint8)
|
|
image_uint8 = image_uint8.permute(2, 0, 1)
|
|
image_uint8 = torchvision.transforms.functional.resize(image_uint8, 100) # type: ignore
|
|
image_uint8 = image_uint8.permute(1, 2, 0)
|
|
image_uint8 = image_uint8.cpu().numpy()
|
|
c2w = camera.camera_to_worlds.cpu().numpy()
|
|
R = vtf.SO3.from_matrix(c2w[:3, :3])
|
|
R = R @ vtf.SO3.from_x_radians(np.pi)
|
|
self.viser_server.add_camera_frustum(
|
|
name=f"Camera {idx}",
|
|
fov=float(camera.fx[0]),
|
|
scale=0.2,
|
|
aspect=float(camera.cx[0] / camera.cy[0]),
|
|
image=image_uint8,
|
|
wxyz=R.wxyz,
|
|
position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO,
|
|
)
|
|
|
|
self.train_state = train_state
|
|
self.train_util = 0.9
|
|
|
|
def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None:
|
|
"""updates the scene based on the graph weights
|
|
|
|
Args:
|
|
step: iteration step of training
|
|
num_rays_per_batch: number of rays per batch, used during training
|
|
"""
|
|
self.step = step
|
|
|
|
if self.camera_state is None:
|
|
return
|
|
|
|
if self.trainer is not None and self.trainer.training_state == "training" and self.train_util != 1:
|
|
if (
|
|
EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
|
|
and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
|
|
):
|
|
train_s = GLOBAL_BUFFER["events"][EventName.TRAIN_RAYS_PER_SEC.value]["avg"]
|
|
vis_s = GLOBAL_BUFFER["events"][EventName.VIS_RAYS_PER_SEC.value]["avg"]
|
|
train_util = self.train_util
|
|
vis_n = self.control_panel.max_res**2
|
|
train_n = num_rays_per_batch
|
|
train_time = train_n / train_s
|
|
vis_time = vis_n / vis_s
|
|
|
|
render_freq = train_util * vis_time / (train_time - train_util * train_time)
|
|
else:
|
|
render_freq = 30
|
|
if step > self.last_step + render_freq:
|
|
self.last_step = step
|
|
self.render_statemachine.action(RenderAction("step", self.camera_state))
|
|
|
|
def update_colormap_options(self, dimensions: int, dtype: type) -> None:
|
|
"""update the colormap options based on the current render
|
|
|
|
Args:
|
|
dimensions: the number of dimensions of the render
|
|
dtype: the data type of the render
|
|
"""
|
|
if self.output_type_changed:
|
|
self.control_panel.update_colormap_options(dimensions, dtype)
|
|
self.output_type_changed = False
|
|
|
|
def update_split_colormap_options(self, dimensions: int, dtype: type) -> None:
|
|
"""update the colormap options based on the current render
|
|
|
|
Args:
|
|
dimensions: the number of dimensions of the render
|
|
dtype: the data type of the render
|
|
"""
|
|
if self.output_split_type_changed:
|
|
self.control_panel.update_split_colormap_options(dimensions, dtype)
|
|
self.output_split_type_changed = False
|
|
|
|
def get_model(self) -> Model:
|
|
"""Returns the model."""
|
|
return self.pipeline.model
|
|
|
|
def training_complete(self) -> None:
|
|
"""Called when training is complete."""
|
|
self.training_state = "completed"
|