diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py index 7ffe28add..2f57d50e4 100644 --- a/nerfstudio/configs/experiment_config.py +++ b/nerfstudio/configs/experiment_config.py @@ -68,7 +68,7 @@ class ExperimentConfig(InstantiateConfig): } ) """Dictionary of optimizer groups and their schedulers""" - vis: Literal["viewer", "wandb", "tensorboard", "viewer+wandb", "viewer+tensorboard"] = "wandb" + vis: Literal["viewer", "wandb", "tensorboard", "viewer+wandb", "viewer+tensorboard", "viewer_beta"] = "wandb" """Which visualizer to use.""" data: Optional[Path] = None """Alias for --pipeline.datamanager.data""" @@ -79,6 +79,10 @@ class ExperimentConfig(InstantiateConfig): """Checks if a viewer is enabled.""" return ("viewer" == self.vis) | ("viewer+wandb" == self.vis) | ("viewer+tensorboard" == self.vis) + def is_viewer_beta_enabled(self) -> bool: + """Checks if a viewer beta is enabled.""" + return "viewer_beta" == self.vis + def is_wandb_enabled(self) -> bool: """Checks if wandb is enabled.""" return ("wandb" == self.vis) | ("viewer+wandb" == self.vis) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index aa704d044..737feb779 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -50,6 +50,7 @@ from nerfstudio.utils.misc import step_check from nerfstudio.utils.rich_utils import CONSOLE from nerfstudio.utils.writer import EventName, TimeWriter from nerfstudio.viewer.server.viewer_state import ViewerState +from nerfstudio.viewer_beta.viewer_state import ViewerState as ViewerBetaState TRAIN_INTERATION_OUTPUT = Tuple[torch.Tensor, Dict[str, torch.Tensor], Dict[str, torch.Tensor]] TORCH_DEVICE = str @@ -167,6 +168,16 @@ class Trainer: train_lock=self.train_lock, ) banner_messages = [f"Viewer at: {self.viewer_state.viewer_url}"] + if self.config.is_viewer_beta_enabled() and self.local_rank == 0: + self.viewer_state = ViewerBetaState( + self.config.viewer, + log_filename=viewer_log_path, + datapath=self.base_dir, + pipeline=self.pipeline, + trainer=self, + train_lock=self.train_lock, + ) + banner_messages = [f"Viewer Beta at: {self.viewer_state.viewer_url}"] self._check_viewer_warnings() self._load_checkpoint() @@ -304,7 +315,7 @@ class Trainer: def _check_viewer_warnings(self) -> None: """Helper to print out any warnings regarding the way the viewer/loggers are enabled""" if ( - self.config.is_viewer_enabled() + (self.config.is_viewer_enabled() or self.config.is_viewer_beta_enabled()) and not self.config.is_tensorboard_enabled() and not self.config.is_wandb_enabled() ): diff --git a/nerfstudio/utils/decorators.py b/nerfstudio/utils/decorators.py index d36717de9..056d7a5dd 100644 --- a/nerfstudio/utils/decorators.py +++ b/nerfstudio/utils/decorators.py @@ -50,11 +50,11 @@ def check_profiler_enabled(func: Callable) -> Callable: def check_viewer_enabled(func: Callable) -> Callable: - """Decorator: check if viewer is enabled and only run on main process""" + """Decorator: check if the viewer or beta viewer is enabled and only run on main process""" def wrapper(self, *args, **kwargs): ret = None - if self.config.is_viewer_enabled() and comms.is_main_process(): + if (self.config.is_viewer_beta_enabled() or self.config.is_viewer_beta_enabled()) and comms.is_main_process(): ret = func(self, *args, **kwargs) return ret diff --git a/nerfstudio/viewer_beta/__init__.py b/nerfstudio/viewer_beta/__init__.py new file mode 100644 index 000000000..21acafa24 --- /dev/null +++ b/nerfstudio/viewer_beta/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/viewer_beta/render_state_machine.py b/nerfstudio/viewer_beta/render_state_machine.py new file mode 100644 index 000000000..f2323b1d8 --- /dev/null +++ b/nerfstudio/viewer_beta/render_state_machine.py @@ -0,0 +1,218 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This file contains the render state machine, which is responsible for deciding when to render the image """ +from __future__ import annotations + +import contextlib +import threading +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, get_args + +import torch + +from nerfstudio.utils import writer +from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter +from nerfstudio.viewer.server import viewer_utils +from nerfstudio.viewer_beta.utils import CameraState, get_camera + +if TYPE_CHECKING: + from nerfstudio.viewer_beta.viewer_state import ViewerState + +RenderStates = Literal["low_move", "low_static", "high"] +RenderActions = Literal["rerender", "move", "static", "step"] + + +@dataclass +class RenderAction: + """Message to the render state machine""" + + action: RenderActions + """The action to take """ + camera_state: CameraState + """The current camera state """ + + +class RenderStateMachine(threading.Thread): + """The render state machine is responsible for deciding how to render the image. + It decides the resolution and whether to interrupt the current render. + + Args: + viewer: the viewer state + """ + + def __init__(self, viewer: ViewerState): + threading.Thread.__init__(self) + self.transitions: Dict[RenderStates, Dict[RenderActions, RenderStates]] = { + s: {} for s in get_args(RenderStates) + } + # by default, everything is a self-transition + for a in get_args(RenderActions): + for s in get_args(RenderStates): + self.transitions[s][a] = s + # then define the actions between states + self.transitions["low_move"]["static"] = "low_static" + self.transitions["low_static"]["static"] = "high" + self.transitions["low_static"]["step"] = "high" + self.transitions["low_static"]["move"] = "low_move" + self.transitions["high"]["move"] = "low_move" + self.transitions["high"]["rerender"] = "low_static" + self.next_action: Optional[RenderAction] = None + self.state: RenderStates = "low_static" + self.render_trigger = threading.Event() + self.target_fps = 24 + self.viewer = viewer + self.interrupt_render_flag = False + self.daemon = True + self.output_keys = {} + + def action(self, action: RenderAction): + """Takes an action and updates the state machine + + Args: + action: the action to take + """ + if self.next_action is None: + self.next_action = action + elif action.action == "step" and ( + self.state == "low_move" or self.next_action.action in ("move", "static", "rerender") + ): + # ignore steps if: + # 1. we are in low_moving state + # 2. the current next_action is move, static, or rerender + return + elif self.next_action == "rerender": + # never overwrite rerenders + pass + else: + # monimal use case, just set the next action + self.next_action = action + + # handle interrupt logic + if self.state == "high" and self.next_action.action in ("move", "rerender"): + self.interrupt_render_flag = True + self.render_trigger.set() + + def _render_img(self, camera_state: CameraState): + """Takes the current camera, generates rays, and renders the iamge + + Args: + camera_state: the current camera state + """ + + image_height, image_width = self._calculate_image_res(camera_state.aspect) + + camera = get_camera(camera_state, image_height, image_width) + camera = camera.to(self.viewer.get_model().device) + assert camera is not None, "render called before viewer connected" + + with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): + camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) + + with TimeWriter(None, None, write=False) as vis_t: + self.viewer.get_model().eval() + step = self.viewer.step + with torch.no_grad(): + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + self.viewer.get_model().train() + num_rays = len(camera_ray_bundle) + render_time = vis_t.duration + if writer.is_initialized(): + writer.put_time( + name=EventName.VIS_RAYS_PER_SEC, duration=num_rays / render_time, step=step, avg_over_steps=True + ) + return outputs + + def run(self): + """Main loop for the render thread""" + while True: + self.render_trigger.wait() + self.render_trigger.clear() + action = self.next_action + assert action is not None, "Action should never be None at this point" + self.next_action = None + if self.state == "high" and action.action == "static": + # if we are in high res and we get a static action, we don't need to do anything + continue + self.state = self.transitions[self.state][action.action] + try: + with viewer_utils.SetTrace(self.check_interrupt): + outputs = self._render_img(action.camera_state) + except viewer_utils.IOChangeException: + # if we got interrupted, don't send the output to the viewer + continue + self._send_output_to_viewer(outputs) + # if we rendered a static low res, we need to self-trigger a static high-res + if self.state == "low_static": + self.action(RenderAction("static", action.camera_state)) + + def check_interrupt(self, frame, event, arg): + """Raises interrupt when flag has been set and not already on lowest resolution. + Used in conjunction with SetTrace. + """ + if event == "line": + if self.interrupt_render_flag: + self.interrupt_render_flag = False + raise viewer_utils.IOChangeException + return self.check_interrupt + + def _send_output_to_viewer(self, outputs: Dict[str, Any]): + """Chooses the correct output and sends it to the viewer + + Args: + outputs: the dictionary of outputs to choose from, from the model + """ + selected_output = (outputs["rgb"] * 255).type(torch.uint8) + + self.viewer.viser_server.set_background_image( + selected_output.cpu().numpy(), + format=self.viewer.config.image_format, + jpeg_quality=self.viewer.config.jpeg_quality, + ) + + def _calculate_image_res(self, aspect_ratio: float) -> Tuple[int, int]: + """Calculate the maximum image height that can be rendered in the time budget + + Args: + apect_ratio: the aspect ratio of the current view + Returns: + image_height: the maximum image height that can be rendered in the time budget + image_width: the maximum image width that can be rendered in the time budget + """ + max_res = self.viewer.max_res + if self.state == "high": + # high res is always static + image_height = max_res + image_width = int(image_height * aspect_ratio) + if image_width > max_res: + image_width = max_res + image_height = int(image_width / aspect_ratio) + elif self.state in ("low_move", "low_static"): + if writer.is_initialized() and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]: + vis_rays_per_sec = GLOBAL_BUFFER["events"][EventName.VIS_RAYS_PER_SEC.value]["avg"] + else: + vis_rays_per_sec = 100000 + target_fps = self.target_fps + num_vis_rays = vis_rays_per_sec / target_fps + image_height = (num_vis_rays / aspect_ratio) ** 0.5 + image_height = int(round(image_height, -1)) + image_height = max(min(max_res, image_height), 30) + image_width = int(image_height * aspect_ratio) + if image_width > max_res: + image_width = max_res + image_height = int(image_width / aspect_ratio) + else: + raise ValueError(f"Invalid state: {self.state}") + + return image_height, image_width diff --git a/nerfstudio/viewer_beta/utils.py b/nerfstudio/viewer_beta/utils.py new file mode 100644 index 000000000..77f4e198f --- /dev/null +++ b/nerfstudio/viewer_beta/utils.py @@ -0,0 +1,67 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Union + +import numpy as np +import torch +from jaxtyping import Float + +from nerfstudio.cameras.cameras import Cameras, CameraType + + +@dataclass +class CameraState: + """A dataclass for storing the camera state.""" + + fov: float + """ The field of view of the camera. """ + aspect: float + """ The aspect ratio of the image. """ + c2w: Float[torch.Tensor, "3 4"] + """ The camera matrix. """ + + +def get_camera( + camera_state: CameraState, image_height: int, image_width: Optional[Union[int, float]] = None +) -> Cameras: + """Returns the camera intrinsics matrix and the camera to world homogeneous matrix. + + Args: + camera_state: the camera state + image_size: the size of the image (height, width) + """ + # intrinsics + fov = camera_state.fov + aspect = camera_state.aspect + if image_width is None: + image_width = aspect * image_height + pp_w = image_width / 2.0 + pp_h = image_height / 2.0 + focal_length = pp_h / np.tan(fov / 2.0) + intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float() + + camera_type = CameraType.PERSPECTIVE + + camera = Cameras( + fx=intrinsics_matrix[0, 0], + fy=intrinsics_matrix[1, 1], + cx=pp_w, + cy=pp_h, + camera_type=camera_type, + camera_to_worlds=camera_state.c2w, + times=torch.tensor([0.0], dtype=torch.float32), + ) + return camera diff --git a/nerfstudio/viewer_beta/viewer_state.py b/nerfstudio/viewer_beta/viewer_state.py new file mode 100644 index 000000000..115ff6f9a --- /dev/null +++ b/nerfstudio/viewer_beta/viewer_state.py @@ -0,0 +1,204 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Manage the state of the viewer """ +from __future__ import annotations + +import threading +from pathlib import Path +from typing import TYPE_CHECKING, Literal, Optional + +import numpy as np +import torch +import torchvision +import viser +import viser.transforms as vtf + +from nerfstudio.configs import base_config as cfg +from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.models.base_model import Model +from nerfstudio.pipelines.base_pipeline import Pipeline +from nerfstudio.utils.decorators import check_main_thread, decorate_all +from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName +from nerfstudio.viewer.server import viewer_utils +from nerfstudio.viewer_beta.render_state_machine import RenderAction, RenderStateMachine +from nerfstudio.viewer_beta.utils import CameraState + +if TYPE_CHECKING: + from nerfstudio.engine.trainer import Trainer + +VISER_NERFSTUDIO_SCALE_RATIO: int = 10 + + +@decorate_all([check_main_thread]) +class ViewerState: + """Class to hold state for viewer variables + + Args: + config: viewer setup configuration + log_filename: filename to log viewer output to + datapath: path to data + pipeline: pipeline object to use + trainer: trainer object to use + + Attributes: + viewer_url: url to open viewer + """ + + viewer_url: str + + def __init__( + self, + config: cfg.ViewerConfig, + log_filename: Path, + datapath: Path, + pipeline: Pipeline, + trainer: Optional[Trainer] = None, + train_lock: Optional[threading.Lock] = None, + ): + self.config = config + self.trainer = trainer + self.last_step = 0 + self.train_lock = train_lock + self.pipeline = pipeline + self.log_filename = log_filename + self.datapath = datapath.parent if datapath.is_file() else datapath + + if self.config.websocket_port is None: + websocket_port = viewer_utils.get_free_port(default_port=self.config.websocket_port_default) + else: + websocket_port = self.config.websocket_port + self.log_filename.parent.mkdir(exist_ok=True) + + self.viewer_url = viewer_utils.get_viewer_url(websocket_port) + + # viewer specific variables + self.output_type_changed = True + self.output_split_type_changed = True + self.step = 0 + self.train_btn_state: Literal["training", "paused", "completed"] = "training" + self._prev_train_state: Literal["training", "paused", "completed"] = "training" + + self.camera_message = None + + self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port) + + self.render_statemachine = RenderStateMachine(self) + self.render_statemachine.start() + + self.viser_server.on_client_connect(self.handle_new_client) + + self.camera_state: Optional[CameraState] = None + + def handle_new_client(self, client: viser.ClientHandle) -> None: + @client.camera.on_update + def _(_: viser.CameraHandle) -> None: + R = vtf.SO3(wxyz=client.camera.wxyz) + R = R @ vtf.SO3.from_x_radians(np.pi) + R = torch.tensor(R.as_matrix()) + pos = torch.tensor(client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO + c2w = torch.concatenate([R, pos[:, None]], dim=1) + self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w) + self.render_statemachine.action(RenderAction("move", self.camera_state)) + + def _pick_drawn_image_idxs(self, total_num: int) -> list[int]: + """Determine indicies of images to display in viewer. + + Args: + total_num: total number of training images. + + Returns: + List of indices from [0, total_num-1]. + """ + if self.config.max_num_display_images < 0: + num_display_images = total_num + else: + num_display_images = min(self.config.max_num_display_images, total_num) + # draw indices, roughly evenly spaced + return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist() + + def init_scene(self, dataset: InputDataset, train_state: Literal["training", "paused", "completed"]) -> None: + """Draw some images and the scene aabb in the viewer. + + Args: + dataset: dataset to render in the scene + train_state: Current status of training + """ + + # draw the training cameras and images + image_indices = self._pick_drawn_image_idxs(len(dataset)) + for idx in image_indices: + image = dataset[idx]["image"] + camera = dataset.cameras[idx] + image_uint8 = (image * 255).detach().type(torch.uint8) + image_uint8 = image_uint8.permute(2, 0, 1) + image_uint8 = torchvision.transforms.functional.resize(image_uint8, 100) # type: ignore + image_uint8 = image_uint8.permute(1, 2, 0) + image_uint8 = image_uint8.cpu().numpy() + c2w = camera.camera_to_worlds.cpu().numpy() + R = vtf.SO3.from_matrix(c2w[:3, :3]) + R = R @ vtf.SO3.from_x_radians(np.pi) + self.viser_server.add_camera_frustum( + name=f"Camera {idx}", + fov=float(camera.fx[0]), + scale=0.2, + aspect=float(camera.cx[0] / camera.cy[0]), + image=image_uint8, + wxyz=R.wxyz, + position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO, + ) + + self.train_state = train_state + self.train_util = 0.9 + self.max_res = 512 + + def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None: + """updates the scene based on the graph weights + + Args: + step: iteration step of training + num_rays_per_batch: number of rays per batch, used during training + """ + self.step = step + + if self.camera_state is None: + return + + if self.trainer is not None and self.trainer.training_state == "training" and self.train_util != 1: + if ( + EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"] + and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"] + ): + train_s = GLOBAL_BUFFER["events"][EventName.TRAIN_RAYS_PER_SEC.value]["avg"] + vis_s = GLOBAL_BUFFER["events"][EventName.VIS_RAYS_PER_SEC.value]["avg"] + train_util = self.train_util + vis_n = self.max_res**2 + train_n = num_rays_per_batch + train_time = train_n / train_s + vis_time = vis_n / vis_s + + render_freq = train_util * vis_time / (train_time - train_util * train_time) + else: + render_freq = 30 + if step > self.last_step + render_freq: + self.last_step = step + self.render_statemachine.action(RenderAction("step", self.camera_state)) + + def get_model(self) -> Model: + """Returns the model.""" + return self.pipeline.model + + def training_complete(self) -> None: + """Called when training is complete.""" + self.training_state = "completed" diff --git a/pyproject.toml b/pyproject.toml index 49147b268..c66c34b04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ "torchvision>=0.14.1", "torchmetrics[image]>=0.9.3", "typing_extensions>=4.4.0", - "viser>=0.0.5", + "viser>=0.0.10", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas",