From c1c7a3b2831defd0f3ffd4d6cb410f2454d124c5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 17 Apr 2023 17:57:04 -0700 Subject: [PATCH 01/42] Fix ns-install-cli (#1754) * Make ns-dev-sync-viser-message-defs a tyro CLI to fix ns-install-cli error * Fix pylint --- scripts/completions/install.py | 1 + scripts/viewer/sync_viser_message_defs.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/completions/install.py b/scripts/completions/install.py index 067bea2bd..a7acc862c 100644 --- a/scripts/completions/install.py +++ b/scripts/completions/install.py @@ -31,6 +31,7 @@ HEADER_LINE = "# Source nerfstudio autocompletions." def _get_all_entry_points() -> List[str]: + # TODO: we should filter out entrypoints that are not tyro CLIs. entry_points = importlib_metadata.distribution("nerfstudio").entry_points return [x.name for x in entry_points] diff --git a/scripts/viewer/sync_viser_message_defs.py b/scripts/viewer/sync_viser_message_defs.py index ef0b380d7..452621ddc 100644 --- a/scripts/viewer/sync_viser_message_defs.py +++ b/scripts/viewer/sync_viser_message_defs.py @@ -1,14 +1,14 @@ """Generate viser message definitions for TypeScript, by parsing Python dataclasses.""" - import pathlib import subprocess +import tyro from viser.infra import generate_typescript_interfaces from nerfstudio.viewer.viser import NerfstudioMessage -def entrypoint() -> None: +def main() -> None: """Generate viser message definitions for TypeScript, by parsing Python dataclasses.""" # Generate typescript source. @@ -32,5 +32,11 @@ def entrypoint() -> None: subprocess.run(args=["prettier", "-w", str(target_path)], check=False) +def entrypoint() -> None: + """Entrypoint for use with pyproject scripts.""" + # All entrypoints must currently be tyro CLIs. + tyro.cli(main) + + if __name__ == "__main__": entrypoint() From 797e9012fb8bdb167cdf7f31ead595d864ada2a9 Mon Sep 17 00:00:00 2001 From: Dylan Hu Date: Tue, 18 Apr 2023 02:33:54 -0400 Subject: [PATCH 02/42] Fix divide by zero when train util set to 1 (#1757) * fix divide by zero when train util set to 1 * Run black --------- Co-authored-by: Brent Yi --- nerfstudio/viewer/server/viewer_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/viewer/server/viewer_state.py b/nerfstudio/viewer/server/viewer_state.py index beee00ba7..54660b666 100644 --- a/nerfstudio/viewer/server/viewer_state.py +++ b/nerfstudio/viewer/server/viewer_state.py @@ -303,7 +303,7 @@ class ViewerState: if self.camera_message is None: return - if self.trainer is not None and self.trainer.is_training: + if self.trainer is not None and self.trainer.is_training and self.control_panel.train_util != 1: if ( EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"] and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"] From e48468cfaee7d38ef05b4fb59df179a18a52c1db Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Mon, 17 Apr 2023 23:34:19 -0700 Subject: [PATCH 03/42] Update disconnected state UI (#1758) --- nerfstudio/viewer/app/src/index.scss | 29 +++++++++++++++++-- .../src/modules/RenderWindow/RenderWindow.jsx | 10 +++++++ .../SidePanel/StatusPanel/StatusPanel.jsx | 12 -------- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/nerfstudio/viewer/app/src/index.scss b/nerfstudio/viewer/app/src/index.scss index a3af6c244..83e09b372 100644 --- a/nerfstudio/viewer/app/src/index.scss +++ b/nerfstudio/viewer/app/src/index.scss @@ -17,6 +17,7 @@ code { .App { position: relative; height: 100vh; + background-color: #1d222a; } $base-color1: #292d39; @@ -358,8 +359,32 @@ $sidebar-width: 100% !default; border: $custom-border; } -.RenderWindow { - z-index: 1; +#not-connected-overlay { + position: absolute; + background-color: #1d222a; + height: 100%; + width: 100%; + text-align: center; + margin: 0; + padding: 0; + z-index: 10; + opacity: 0.8; +} + +#not-connected-overlay-text { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + background-color: #393939; + border-radius: 10px; + padding-left: 15px; + padding-right: 15px; + padding-top: 5px; + padding-bottom: 5px; + color: #ff6969; + font-size: 18px; + z-index: 10; } .ViewerWindow-render-crop-container { diff --git a/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx b/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx index 598a13298..e6c7e6d5f 100644 --- a/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx @@ -1,13 +1,23 @@ import * as React from 'react'; +import { useSelector } from 'react-redux'; export default function RenderWindow() { + const isWebsocketConnected = useSelector( + (state) => state.websocketState.isConnected, + ); + return (
+
); diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx index a330f0311..d3f49ffab 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx @@ -66,11 +66,6 @@ export default function StatusPanel(props: StatusPanelProps) { const is_training_text = isTraining ? 'Pause Training' : 'Resume Training'; const training_icon = isTraining ? : ; - const websocket_connected_text = isWebsocketConnected - ? 'Renderer Connected' - : 'Renderer Disconnected'; - const websocket_connected_color = isWebsocketConnected ? 'success' : 'error'; - return (
@@ -128,13 +123,6 @@ export default function StatusPanel(props: StatusPanelProps) { Resolution: {eval_res}
- ); } From ce733a83081ae810ebb72fee692494e31fe180d9 Mon Sep 17 00:00:00 2001 From: Yuantao Chen <44546388+Tao-11-chen@users.noreply.github.com> Date: Tue, 18 Apr 2023 14:35:34 +0800 Subject: [PATCH 04/42] Delete useless lists of tensors put to CUDA which may cause OOM. (#1748) eval_large_imgs --- nerfstudio/models/base_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index a73ea67a1..e03893195 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -177,12 +177,12 @@ class Model(nn.Module): ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx) outputs = self.forward(ray_bundle=ray_bundle) for output_name, output in outputs.items(): # type: ignore + if not torch.is_tensor(output): + # TODO: handle lists of tensors as well + continue outputs_lists[output_name].append(output) outputs = {} for output_name, outputs_list in outputs_lists.items(): - if not torch.is_tensor(outputs_list[0]): - # TODO: handle lists of tensors as well - continue outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore return outputs From 3d2e58ed8456e9cc72fd0844a9e05e30ab6295ab Mon Sep 17 00:00:00 2001 From: Alexander Kristoffersen Date: Tue, 18 Apr 2023 00:03:34 -0700 Subject: [PATCH 05/42] Colab Fix: change localhost to 0.0.0.0 (#1759) Update viewer_state.py change localhost -> 0.0.0.0 --- nerfstudio/viewer/server/viewer_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/viewer/server/viewer_state.py b/nerfstudio/viewer/server/viewer_state.py index 54660b666..b6e96623f 100644 --- a/nerfstudio/viewer/server/viewer_state.py +++ b/nerfstudio/viewer/server/viewer_state.py @@ -122,7 +122,7 @@ class ViewerState: self.camera_message = None - self.viser_server = ViserServer(host="localhost", port=websocket_port) + self.viser_server = ViserServer(host="0.0.0.0", port=websocket_port) self.viser_server.register_handler(IsTrainingMessage, self._handle_is_training) self.viser_server.register_handler(SaveCheckpointMessage, self._handle_save_checkpoint) From 37322ec4532be24de14955793dc1632ba0a13404 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 18 Apr 2023 09:57:51 -0700 Subject: [PATCH 06/42] Add typing extensions to pyproject (#1763) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0dd44fd23..eb872f3c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "torchmetrics[image]>=0.9.3", "torchtyping>=0.1.4", "torchvision>=0.13.0", + "typing_extensions>=4.4.0", "viser>=0.0.5", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", From ea14152ae65ca379584f255965f33bab51f05a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Tue, 18 Apr 2023 20:29:09 +0200 Subject: [PATCH 07/42] Add PyTorch profiler to support tracing of forward/backward calls (#1761) * Add PyTorch profiler support * Add support for wrapped code, fix linting errors * Update docs --- .../debugging_tools/profiling.md | 27 +++- nerfstudio/configs/base_config.py | 8 +- nerfstudio/engine/trainer.py | 2 +- nerfstudio/utils/decorators.py | 2 +- nerfstudio/utils/profiler.py | 121 ++++++++++++++++-- 5 files changed, 142 insertions(+), 18 deletions(-) diff --git a/docs/developer_guides/debugging_tools/profiling.md b/docs/developer_guides/debugging_tools/profiling.md index ed8796cec..aabada5f4 100644 --- a/docs/developer_guides/debugging_tools/profiling.md +++ b/docs/developer_guides/debugging_tools/profiling.md @@ -23,7 +23,24 @@ class RayGenerator(nn.Module): ... ``` -At termination of training or end of the training run, the profiler will print out the average execution time for all of the functions that have the profiler tag. +Alternatively, you can also time parts of the code: +```python +... + +def forward(self, ray_indices: TensorType["num_rays", 3]) -> RayBundle: + # implementation here + with profiler.time_function("code1"): + # some code here + ... + + with profiler.time_function("code2"): + # some code here + ... + ... +``` + + +At termination of training or end of the training run, the profiler will print out the average execution time for all of the functions or code blocks that have the profiler tag. :::{admonition} Tip :class: info @@ -32,6 +49,14 @@ Use this profiler if there are *specific/individual functions* you want to measu ::: +#### Profiling with PyTorch profiler + +If you want to profile the training or evaluation code and track the memory and CUDA kernel launches, consider using [PyTorch profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html). +It will run the profiler for some selected step numbers, once with `CUDA_LAUNCH_BLOCKING=1`, once with `CUDA_LAUNCH_BLOCKING=0`. +The PyTorch profiler can be enabled with `--logging.profiler=pytorch` flag. +The outputs of the profiler are trace files stored in `{PATH_TO_MODEL_OUTPUT}/profiler_traces`, and can be loaded in Google Chrome by typing `chrome://tracing`. + + #### Profiling with PySpy If you want to profile the entire codebase, consider using [PySpy](https://github.com/benfred/py-spy). diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index fd5608b4e..b2665a649 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -119,9 +119,11 @@ class LoggingConfig(PrintableConfig): e.g. if 20, averages will be computed over past 20 occurrences.""" local_writer: LocalWriterConfig = LocalWriterConfig(enable=True) """if provided, will print stats locally. if None, will disable printing""" - enable_profiler: bool = True - """whether to enable profiling code; prints speed of functions at the end of a program. - profiler logs run times of functions and prints at end of training""" + profiler: Literal["none", "basic", "pytorch"] = "basic" + """how to profile the code; + "basic" - prints speed of all decorated functions at the end of a program. + "pytorch" - same as basic, but it also traces few training steps. + """ # Viewer related configs diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 7f07a3b5d..1dddb21d5 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -184,7 +184,7 @@ class Trainer: self.config.logging, max_iter=self.config.max_num_iterations, banner_messages=banner_messages ) writer.put_config(name="config", config_dict=dataclasses.asdict(self.config), step=0) - profiler.setup_profiler(self.config.logging) + profiler.setup_profiler(self.config.logging, writer_log_path) def setup_optimizers(self) -> Optimizers: """Helper to set up the optimizers diff --git a/nerfstudio/utils/decorators.py b/nerfstudio/utils/decorators.py index 6f92ac679..356369f82 100644 --- a/nerfstudio/utils/decorators.py +++ b/nerfstudio/utils/decorators.py @@ -42,7 +42,7 @@ def check_profiler_enabled(func: Callable) -> Callable: def wrapper(self, *args, **kwargs): ret = None - if self.config.enable_profiler: + if self.config.profiler != "none": ret = func(self, *args, **kwargs) return ret diff --git a/nerfstudio/utils/profiler.py b/nerfstudio/utils/profiler.py index c1fc96dcb..246569444 100644 --- a/nerfstudio/utils/profiler.py +++ b/nerfstudio/utils/profiler.py @@ -17,10 +17,17 @@ Profiler base class and functionality """ from __future__ import annotations +import functools +import os import time -from typing import Callable +from collections import deque +from contextlib import contextmanager +from pathlib import Path +from typing import Callable, Dict, List, Optional, Tuple, Union from rich.console import Console +from torch.autograd.profiler import ContextDecorator +from torch.profiler import ProfilerActivity, profile, record_function from nerfstudio.configs import base_config as cfg from nerfstudio.utils import comms @@ -33,32 +40,122 @@ from nerfstudio.utils.decorators import ( CONSOLE = Console(width=120) PROFILER = [] +PYTORCH_PROFILER = None -def time_function(func: Callable) -> Callable: - """Decorator: time a function call""" +class time_function(ContextDecorator): # pylint: disable=invalid-name + """Decorator/Context manager: time a function call or a block of code""" - def wrapper(*args, **kwargs): - start = time.time() - ret = func(*args, **kwargs) + def __init__(self, name: str): + self.name = name + self.start = None + self._profiler_contexts = deque() + self._function_call_args: Optional[Tuple[Tuple, Dict]] = None + + def __new__(cls, func: Union[str, Callable]): + instance = super().__new__(cls) + if isinstance(func, str): + instance.__init__(func) + return instance + if callable(func): + instance.__init__(func.__qualname__) + return instance(func) + raise ValueError(f"Argument func of type {type(func)} is not a string or a callable.") + + def __enter__(self): + self.start = time.time() + if PYTORCH_PROFILER is not None: + args, kwargs = tuple(), {} + if self._function_call_args is not None: + args, kwargs = self._function_call_args + ctx = PYTORCH_PROFILER.record_function(self.name, *args, **kwargs) + ctx.__enter__() # pylint: disable=no-member + self._profiler_contexts.append(ctx) + if self._function_call_args is None: + ctx = record_function(self.name) + ctx.__enter__() + self._profiler_contexts.append(ctx) + + def __exit__(self, *args, **kwargs): + while self._profiler_contexts: + context = self._profiler_contexts.pop() + context.__exit__(*args, **kwargs) if PROFILER: - class_str = func.__qualname__ - PROFILER[0].update_time(class_str, start, time.time()) - return ret + PROFILER[0].update_time(self.name, self.start, time.time()) - return wrapper + def __call__(self, func: Callable): + @functools.wraps(func) + def inner(*args, **kwargs): + self._function_call_args = (args, kwargs) + with self: + out = func(*args, **kwargs) + self._function_call_args = None + return out + + return inner def flush_profiler(config: cfg.LoggingConfig): """Method that checks if profiler is enabled before flushing""" - if config.enable_profiler and PROFILER: + if config.profiler != "none" and PROFILER: PROFILER[0].print_profile() -def setup_profiler(config: cfg.LoggingConfig): +def setup_profiler(config: cfg.LoggingConfig, log_dir: Path): """Initialization of profilers""" + global PYTORCH_PROFILER # pylint: disable=global-statement if comms.is_main_process(): PROFILER.append(Profiler(config)) + if config.profiler == "pytorch": + PYTORCH_PROFILER = PytorchProfiler(log_dir) + + +class PytorchProfiler: # pylint: disable=too-few-public-methods + """ + Wrapper for Pytorch Profiler + """ + + def __init__(self, output_path: Path, trace_steps: Optional[List[int]] = None): + self.output_path = output_path / "profiler_traces" + if trace_steps is None: + # Some arbitrary steps which likely do not overlap with steps usually chosen to run callbacks + trace_steps = [12, 17] + self.trace_steps = trace_steps + + @contextmanager + def record_function(self, function: str, *args, **_kwargs): + """ + Context manager that records a function call and saves the trace to a json file. + Traced functions are: train_iteration, eval_iteration + """ + if function.endswith("train_iteration") or function.endswith("eval_iteration"): + step = args[1] + assert isinstance(step, int) + assert len(args) == 2 + stage = function.split(".")[-1].split("_")[0] + if step in self.trace_steps: + launch_kernel_blocking = self.trace_steps.index(step) % 2 == 0 + backup_lb_var = "" + if launch_kernel_blocking: + backup_lb_var = os.environ.get("CUDA_LAUNCH_BLOCKING", "") + os.environ["CUDA_LAUNCH_BLOCKING"] = "1" + with profile( + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + record_shapes=True, + with_stack=True, + profile_memory=True, + ) as prof: + yield None + if launch_kernel_blocking: + os.environ["CUDA_LAUNCH_BLOCKING"] = backup_lb_var + self.output_path.mkdir(parents=True, exist_ok=True) + prof.export_chrome_trace( + str(self.output_path / f"trace_{stage}_{step}{'_blocking' if launch_kernel_blocking else ''}.json") + ) + return + # Functions are recorded automatically + yield None + return @decorate_all([check_profiler_enabled, check_main_thread]) From b7fd273638563de4e913705362fc14304efd0ff2 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 18 Apr 2023 15:06:03 -0700 Subject: [PATCH 08/42] Add time controls to viewer (#1764) --- .../data/datamanagers/base_datamanager.py | 3 + .../data/dataparsers/base_dataparser.py | 5 + .../data/dataparsers/dnerf_dataparser.py | 1 + .../data/dataparsers/dycheck_dataparser.py | 1 + nerfstudio/viewer/app/package.json | 2 +- .../SidePanel/CameraPanel/CameraPanel.jsx | 197 +++++++++++++++++- .../src/modules/WebSocket/ViserMessages.tsx | 11 +- .../src/modules/WebSocket/ViserWebSocket.tsx | 18 ++ nerfstudio/viewer/app/src/reducer.js | 3 + nerfstudio/viewer/server/control_panel.py | 19 +- .../viewer/server/render_state_machine.py | 2 +- nerfstudio/viewer/server/viewer_state.py | 15 +- nerfstudio/viewer/viser/message_api.py | 4 + nerfstudio/viewer/viser/messages.py | 13 ++ pyproject.toml | 1 + 15 files changed, 280 insertions(+), 15 deletions(-) diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 7e2ebedeb..d696519af 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -179,6 +179,7 @@ class DataManager(nn.Module): eval_count (int): the step number of our eval iteration, needs to be incremented manually train_dataset (Dataset): the dataset for the train dataset eval_dataset (Dataset): the dataset for the eval dataset + includes_time (bool): whether the dataset includes time information Additional attributes specific to each subclass are defined in the setup_train and setup_eval functions. @@ -189,6 +190,7 @@ class DataManager(nn.Module): eval_dataset: Optional[Dataset] = None train_sampler: Optional[DistributedSampler] = None eval_sampler: Optional[DistributedSampler] = None + includes_time: bool = False def __init__(self): """Constructor for the DataManager class. @@ -407,6 +409,7 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method else: self.config.data = self.config.dataparser.data self.dataparser = self.dataparser_config.setup() + self.includes_time = self.dataparser.includes_time self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") self.train_dataset = self.create_train_dataset() diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 12367e51f..f92b34172 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -127,9 +127,14 @@ class DataParser: Args: config: datasetparser config containing all information needed to instantiate dataset + + Attributes: + config: datasetparser config containing all information needed to instantiate dataset + includes_time: Does the dataset include time information in the camera poses. """ config: DataParserConfig + includes_time: bool = False def __init__(self, config: DataParserConfig): super().__init__() diff --git a/nerfstudio/data/dataparsers/dnerf_dataparser.py b/nerfstudio/data/dataparsers/dnerf_dataparser.py index c26621e92..d7411b4c6 100644 --- a/nerfstudio/data/dataparsers/dnerf_dataparser.py +++ b/nerfstudio/data/dataparsers/dnerf_dataparser.py @@ -53,6 +53,7 @@ class DNeRF(DataParser): """DNeRF Dataset""" config: DNeRFDataParserConfig + includes_time: bool = True def __init__(self, config: DNeRFDataParserConfig): super().__init__(config=config) diff --git a/nerfstudio/data/dataparsers/dycheck_dataparser.py b/nerfstudio/data/dataparsers/dycheck_dataparser.py index 92cca35ff..7ddf051e1 100644 --- a/nerfstudio/data/dataparsers/dycheck_dataparser.py +++ b/nerfstudio/data/dataparsers/dycheck_dataparser.py @@ -202,6 +202,7 @@ class Dycheck(DataParser): """Dycheck (https://arxiv.org/abs/2210.13445) Dataset `iphone` subset""" config: DycheckDataParserConfig + includes_time: bool = True def __init__(self, config: DycheckDataParserConfig): super().__init__(config=config) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 349b25249..cc5eea7ca 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-04-10-0", + "version": "23-04-18-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 8a727be27..720501734 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -48,6 +48,7 @@ import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; import { ViserWebSocketContext, sendWebsocketMessage, + makeThrottledMessageSender, } from '../../WebSocket/ViserWebSocket'; import RenderModal from '../../RenderModal'; import LoadPathModal from '../../LoadPathModal'; @@ -65,6 +66,100 @@ function set_camera_position(camera, matrix) { mat.decompose(camera.position, camera.quaternion, camera.scale); } +function RenderTimeSelector(props) { + const throttled_time_message_sender = props.throttled_time_message_sender; + const disabled = props.disabled; + const isGlobal = props.isGlobal; + const camera = props.camera; + const globalRenderTime = props.globalRenderTime; + const setGlobalRenderTime = props.setGlobalRenderTime; + const applyAll = props.applyAll; + const setAllCameraRenderTime = props.setAllCameraRenderTime; + + const getRenderTimeLabel = () => { + if (!isGlobal) { + return camera.renderTime; + } + camera.renderTime = globalRenderTime; + return globalRenderTime; + }; + + const [UIRenderTime, setUIRenderTime] = React.useState( + isGlobal ? globalRenderTime : getRenderTimeLabel(), + ); + + const [valid, setValid] = React.useState(true); + + useEffect(() => { + setUIRenderTime(getRenderTimeLabel()); + throttled_time_message_sender({ + type: 'TimeConditionMessage', + time: camera.renderTime, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [camera, globalRenderTime]); + + const setRndrTime = (val) => { + if (!isGlobal) { + camera.renderTime = val; + } else { + camera.renderTime = val; + setGlobalRenderTime(val); + } + + if (applyAll) { + setAllCameraRenderTime(val); + } + throttled_time_message_sender({ + type: 'TimeConditionMessage', + time: camera.renderTime, + }); + }; + + const handleValidation = (e) => { + const valueFloat = parseFloat(e.target.value); + let valueStr = String(valueFloat); + if (e.target.value >= 0 && e.target.value <= 1) { + setValid(true); + if (valueFloat === 1.0) { + valueStr = '1.0'; + } + if (valueFloat === 0.0) { + valueStr = '0.0'; + } + setUIRenderTime(parseFloat(valueStr)); + setRndrTime(parseFloat(valueStr)); + } else { + setValid(false); + } + }; + + return ( + setUIRenderTime(e.target.value)} + onBlur={(e) => handleValidation(e)} + disabled={disabled} + sx={{ + input: { + WebkitTextFillColor: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, + color: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, + }, + }} + value={UIRenderTime} + error={!valid} + helperText={!valid ? 'RenderTime should be between 0.0 and 1.0' : ''} + variant="standard" + /> + ); +} + function FovSelector(props) { const fovLabel = props.fovLabel; const setFovLabel = props.setFovLabel; @@ -200,6 +295,7 @@ function FovSelector(props) { } function CameraList(props) { + const throttled_time_message_sender = props.throttled_time_message_sender; const sceneTree = props.sceneTree; const cameras = props.cameras; const camera_main = props.camera_main; @@ -211,7 +307,6 @@ function CameraList(props) { const cameraProperties = props.cameraProperties; const setCameraProperties = props.setCameraProperties; const isAnimated = props.isAnimated; - const dispatch = props.dispatch; // eslint-disable-next-line no-unused-vars const slider_value = props.slider_value; const set_slider_value = props.set_slider_value; @@ -411,13 +506,19 @@ function CameraList(props) { fovLabel={fovLabel} setFovLabel={setFovLabel} camera={camera} - dispatch={dispatch} disabled={!isAnimated('FOV')} isGlobal={false} - changeMain={false} /> )} - {!isAnimated('FOV') && ( + {isAnimated('RenderTime') && ( + + )} + {!isAnimated('FOV') && !isAnimated('RenderTime') && (

Animated camera properties will show up here!

@@ -455,7 +556,10 @@ export default function CameraPanel(props) { ); const viser_websocket = useContext(ViserWebSocketContext); + const throttled_time_message_sender = + makeThrottledMessageSender(viser_websocket); const DEFAULT_FOV = 50; + const DEFAULT_RENDER_TIME = 0.0; // react state const [cameras, setCameras] = React.useState([]); @@ -471,6 +575,8 @@ export default function CameraPanel(props) { const [load_path_modal_open, setLoadPathModalOpen] = React.useState(false); const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); + const [globalRenderTime, setGlobalRenderTime] = + React.useState(DEFAULT_RENDER_TIME); // leva store const cameraPropsStore = useCreateStore(); @@ -506,6 +612,10 @@ export default function CameraPanel(props) { const crop_center = useSelector((state) => state.renderingState.crop_center); const crop_scale = useSelector((state) => state.renderingState.crop_scale); + const display_render_time = useSelector( + (state) => state.renderingState.use_time_conditioning, + ); + const setRenderHeight = (value) => { dispatch({ type: 'write', @@ -561,7 +671,7 @@ export default function CameraPanel(props) { const camera_main_copy = camera_main.clone(); camera_main_copy.aspect = 1.0; camera_main_copy.fov = globalFov; - camera_main_copy.renderTime = 0.0; + camera_main_copy.renderTime = globalRenderTime; const new_camera_properties = new Map(); camera_main_copy.properties = new_camera_properties; new_camera_properties.set('FOV', globalFov); @@ -800,14 +910,18 @@ export default function CameraPanel(props) { let lookat = null; let up = null; let fov = null; + let render_time = null; position = curve_object.curve_positions.getPoint(point); lookat = curve_object.curve_lookats.getPoint(point); up = curve_object.curve_ups.getPoint(point); fov = curve_object.curve_fovs.getPoint(point).z; + render_time = curve_object.curve_render_times.getPoint(point).z; + render_time = Math.max(Math.min(render_time, 1.0), 0.0); // clamp time values to [0, 1] const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); setFieldOfView(fov); setGlobalFov(fov); + setGlobalRenderTime(render_time); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [slider_value, render_height, render_width]); @@ -847,11 +961,21 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); - camera_path.push({ - camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix - fov, - aspect: camera_render.aspect, - }); + if (display_render_time) { + const renderTime = curve_object.curve_render_times.getPoint(pt).z; + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] + }); + } else { + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + }); + } } const keyframes = []; @@ -1026,6 +1150,12 @@ export default function CameraPanel(props) { } }; + const setAllCameraRenderTime = (val) => { + for (let i = 0; i < cameras.length; i += 1) { + cameras[i].renderTime = val; + } + }; + return (
@@ -1090,6 +1220,52 @@ export default function CameraPanel(props) { />
+ {display_render_time && ( +
+ + + + +
+ )} {camera_type !== 'equirectangular' && (
@@ -1345,6 +1521,7 @@ export default function CameraPanel(props) {
None: """Callback for when the train speed is changed""" if self.train_speed == "Fast": @@ -135,6 +141,7 @@ class ControlPanel: e.set_hidden(self.output_render == "rgb") for e in self._elements_by_tag["crop"]: e.set_hidden(not self.crop_viewport) + self._time.set_hidden(not self._time_enabled) def update_colormap_options(self, dimensions: int, dtype: type) -> None: """update the colormap options based on the current render @@ -235,3 +242,13 @@ class ControlPanel: def background_color(self, value: Tuple[int, int, int]): """Sets the background color""" self._background_color.value = value + + @property + def time(self) -> float: + """Returns the current background color""" + return self._time.value + + @time.setter + def time(self, value: float): + """Sets the background color""" + self._time.value = value diff --git a/nerfstudio/viewer/server/render_state_machine.py b/nerfstudio/viewer/server/render_state_machine.py index 9a955e42b..e94fc7a2a 100644 --- a/nerfstudio/viewer/server/render_state_machine.py +++ b/nerfstudio/viewer/server/render_state_machine.py @@ -155,7 +155,7 @@ class RenderStateMachine(threading.Thread): cy=intrinsics_matrix[1, 2], camera_type=camera_type, camera_to_worlds=camera_to_world[None, ...], - times=torch.tensor([0.0]), + times=torch.tensor([self.viewer.control_panel.time], dtype=torch.float32), ) camera = camera.to(self.viewer.get_model().device) diff --git a/nerfstudio/viewer/server/viewer_state.py b/nerfstudio/viewer/server/viewer_state.py index b6e96623f..22e0bd021 100644 --- a/nerfstudio/viewer/server/viewer_state.py +++ b/nerfstudio/viewer/server/viewer_state.py @@ -51,6 +51,7 @@ from nerfstudio.viewer.viser.messages import ( IsTrainingMessage, NerfstudioMessage, SaveCheckpointMessage, + TimeConditionMessage, ) if TYPE_CHECKING: @@ -110,6 +111,8 @@ class ViewerState: CONSOLE.print(Panel(table, title="[bold][yellow]Viewer[/bold]", expand=False)) + self.include_time = self.pipeline.datamanager.includes_time + # viewer specific variables self.prev_moving = False self.output_type_changed = True @@ -130,8 +133,13 @@ class ViewerState: self.viser_server.register_handler(CameraPathOptionsRequest, self._handle_camera_path_option_request) self.viser_server.register_handler(CameraPathPayloadMessage, self._handle_camera_path_payload) self.viser_server.register_handler(CropParamsMessage, self._handle_crop_params_message) + if self.include_time: + self.viser_server.use_time_conditioning() + self.viser_server.register_handler(TimeConditionMessage, self._handle_time_condition_message) - self.control_panel = ControlPanel(self._interrupt_render, self._crop_params_update, self._output_type_change) + self.control_panel = ControlPanel( + self.include_time, self._interrupt_render, self._crop_params_update, self._output_type_change + ) self.control_panel.install(self.viser_server) def nested_folder_install(folder_labels: List[str], element: ViewerElement): @@ -233,6 +241,11 @@ class ViewerState: self.control_panel.crop_min = tuple(crop_min.tolist()) self.control_panel.crop_max = tuple(crop_max.tolist()) + def _handle_time_condition_message(self, message: NerfstudioMessage) -> None: + """Handle time conditioning message from viewer.""" + assert isinstance(message, TimeConditionMessage) + self.control_panel.time = message.time + @property def is_training(self) -> bool: """Get is_training flag from viewer.""" diff --git a/nerfstudio/viewer/viser/message_api.py b/nerfstudio/viewer/viser/message_api.py index c0471af69..aa4314627 100644 --- a/nerfstudio/viewer/viser/message_api.py +++ b/nerfstudio/viewer/viser/message_api.py @@ -532,6 +532,10 @@ class MessageApi(abc.ABC): ) return GuiHandle(handle_state) + def use_time_conditioning(self) -> None: + """Use time conditioning.""" + self._queue(messages.UseTimeConditioningMessage()) + def _handle_gui_updates( self: MessageApi, client_id: ClientId, diff --git a/nerfstudio/viewer/viser/messages.py b/nerfstudio/viewer/viser/messages.py index dda627d36..d34efb046 100644 --- a/nerfstudio/viewer/viser/messages.py +++ b/nerfstudio/viewer/viser/messages.py @@ -228,3 +228,16 @@ class StatusMessage(NerfstudioMessage): @dataclasses.dataclass class SaveCheckpointMessage(NerfstudioMessage): """Save checkpoint message.""" + + +@dataclasses.dataclass +class UseTimeConditioningMessage(NerfstudioMessage): + """Use time conditioning message.""" + + +@dataclasses.dataclass +class TimeConditionMessage(NerfstudioMessage): + """Time conditioning message.""" + + time: float + """ Time conditioning value """ diff --git a/pyproject.toml b/pyproject.toml index eb872f3c8..feec29ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,7 @@ disable = [ "too-many-branches", "too-many-instance-attributes", "too-many-locals", + "too-many-statements", "unnecessary-ellipsis", ] From cbb0511619d9663b1ad6322609a804808d4f35d0 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 18 Apr 2023 16:55:38 -0700 Subject: [PATCH 09/42] Add completed training state (#1768) --- nerfstudio/engine/trainer.py | 20 ++++-- .../SidePanel/StatusPanel/StatusPanel.jsx | 39 ++++++++--- .../src/modules/WebSocket/ViserMessages.tsx | 8 +-- .../src/modules/WebSocket/ViserWebSocket.tsx | 6 +- nerfstudio/viewer/app/src/reducer.js | 2 +- nerfstudio/viewer/app/src/themes/theme.ts | 1 + nerfstudio/viewer/server/viewer_state.py | 69 ++++++++++--------- nerfstudio/viewer/viser/message_api.py | 6 +- nerfstudio/viewer/viser/messages.py | 4 +- scripts/viewer/run_viewer.py | 4 +- 10 files changed, 100 insertions(+), 59 deletions(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 1dddb21d5..210decdb2 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -104,7 +104,7 @@ class Trainer: pipeline: The pipeline object. optimizers: The optimizers object. callbacks: The callbacks object. - is_training: Whether the model is training. + training_state: Current model training state. """ pipeline: VanillaPipeline @@ -118,7 +118,7 @@ class Trainer: self.world_size = world_size self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}" self.mixed_precision: bool = self.config.mixed_precision - self.is_training: bool = True + self.training_state: Literal["training", "paused", "completed"] = "training" if self.device == "cpu": self.mixed_precision = False CONSOLE.print("Mixed precision is disabled for CPU training.") @@ -216,7 +216,7 @@ class Trainer: num_iterations = self.config.max_num_iterations step = 0 for step in range(self._start_step, self._start_step + num_iterations): - while not self.is_training: + while self.training_state == "paused": time.sleep(0.01) with self.train_lock: with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: @@ -280,6 +280,8 @@ class Trainer: CONSOLE.rule() CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center") if not self.config.viewer.quit_on_train_completion: + self.training_state = "completed" + self._train_complete_viewer() CONSOLE.print("Use ctrl+c to quit", justify="center") while True: time.sleep(0.01) @@ -304,7 +306,7 @@ class Trainer: assert self.viewer_state and self.pipeline.datamanager.train_dataset self.viewer_state.init_scene( dataset=self.pipeline.datamanager.train_dataset, - start_train=True, + train_state="training", ) @check_viewer_enabled @@ -323,6 +325,16 @@ class Trainer: time.sleep(0.03) # sleep to allow buffer to reset CONSOLE.log("Viewer failed. Continuing training.") + @check_viewer_enabled + def _train_complete_viewer(self) -> None: + """Let the viewer know that the training is complete""" + assert self.viewer_state is not None + try: + self.viewer_state.training_complete() + except RuntimeError: + time.sleep(0.03) # sleep to allow buffer to reset + CONSOLE.log("Viewer failed. Continuing training.") + @check_viewer_enabled def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None: """Performs update on rays/sec calculation for training diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx index d3f49ffab..8c70863c8 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx @@ -4,6 +4,7 @@ import Button from '@mui/material/Button'; import { useDispatch, useSelector } from 'react-redux'; import PauseIcon from '@mui/icons-material/Pause'; import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import CelebrationOutlinedIcon from '@mui/icons-material/CelebrationOutlined'; import ViewInArIcon from '@mui/icons-material/ViewInAr'; import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined'; import { @@ -18,7 +19,9 @@ interface StatusPanelProps { export default function StatusPanel(props: StatusPanelProps) { const dispatch = useDispatch(); const viser_websocket = React.useContext(ViserWebSocketContext); - const isTraining = useSelector((state) => state.renderingState.isTraining); + const training_state = useSelector( + (state) => state.renderingState.training_state, + ); const sceneTree = props.sceneTree; const isWebsocketConnected = useSelector( @@ -53,18 +56,38 @@ export default function StatusPanel(props: StatusPanelProps) { }, [camera_choice, is_scene_visible]); const handlePlayChange = () => { + let new_state = null; + if (training_state === 'training') { + new_state = 'paused'; + } else if (training_state === 'paused') { + new_state = 'training'; + } else { + return; + } dispatch({ type: 'write', - path: 'renderingState/isTraining', - data: !isTraining, + path: 'renderingState/training_state', + data: new_state, }); sendWebsocketMessage(viser_websocket, { - type: 'IsTrainingMessage', - is_training: !isTraining, + type: 'TrainingStateMessage', + training_state: new_state, }); }; - const is_training_text = isTraining ? 'Pause Training' : 'Resume Training'; - const training_icon = isTraining ? : ; + let is_training_text = ''; + let training_icon = null; + let color = 'secondary'; + if (training_state === 'training') { + is_training_text = 'Pause Training'; + training_icon = ; + } else if (training_state === 'paused') { + is_training_text = 'Resume Training'; + training_icon = ; + } else { + is_training_text = 'Training Complete'; + color = 'success'; + training_icon = ; + } return (
@@ -72,7 +95,7 @@ export default function StatusPanel(props: StatusPanelProps) {