From cbb0511619d9663b1ad6322609a804808d4f35d0 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 18 Apr 2023 16:55:38 -0700 Subject: [PATCH] Add completed training state (#1768) --- nerfstudio/engine/trainer.py | 20 ++++-- .../SidePanel/StatusPanel/StatusPanel.jsx | 39 ++++++++--- .../src/modules/WebSocket/ViserMessages.tsx | 8 +-- .../src/modules/WebSocket/ViserWebSocket.tsx | 6 +- nerfstudio/viewer/app/src/reducer.js | 2 +- nerfstudio/viewer/app/src/themes/theme.ts | 1 + nerfstudio/viewer/server/viewer_state.py | 69 ++++++++++--------- nerfstudio/viewer/viser/message_api.py | 6 +- nerfstudio/viewer/viser/messages.py | 4 +- scripts/viewer/run_viewer.py | 4 +- 10 files changed, 100 insertions(+), 59 deletions(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 1dddb21d5..210decdb2 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -104,7 +104,7 @@ class Trainer: pipeline: The pipeline object. optimizers: The optimizers object. callbacks: The callbacks object. - is_training: Whether the model is training. + training_state: Current model training state. """ pipeline: VanillaPipeline @@ -118,7 +118,7 @@ class Trainer: self.world_size = world_size self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}" self.mixed_precision: bool = self.config.mixed_precision - self.is_training: bool = True + self.training_state: Literal["training", "paused", "completed"] = "training" if self.device == "cpu": self.mixed_precision = False CONSOLE.print("Mixed precision is disabled for CPU training.") @@ -216,7 +216,7 @@ class Trainer: num_iterations = self.config.max_num_iterations step = 0 for step in range(self._start_step, self._start_step + num_iterations): - while not self.is_training: + while self.training_state == "paused": time.sleep(0.01) with self.train_lock: with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: @@ -280,6 +280,8 @@ class Trainer: CONSOLE.rule() CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center") if not self.config.viewer.quit_on_train_completion: + self.training_state = "completed" + self._train_complete_viewer() CONSOLE.print("Use ctrl+c to quit", justify="center") while True: time.sleep(0.01) @@ -304,7 +306,7 @@ class Trainer: assert self.viewer_state and self.pipeline.datamanager.train_dataset self.viewer_state.init_scene( dataset=self.pipeline.datamanager.train_dataset, - start_train=True, + train_state="training", ) @check_viewer_enabled @@ -323,6 +325,16 @@ class Trainer: time.sleep(0.03) # sleep to allow buffer to reset CONSOLE.log("Viewer failed. Continuing training.") + @check_viewer_enabled + def _train_complete_viewer(self) -> None: + """Let the viewer know that the training is complete""" + assert self.viewer_state is not None + try: + self.viewer_state.training_complete() + except RuntimeError: + time.sleep(0.03) # sleep to allow buffer to reset + CONSOLE.log("Viewer failed. Continuing training.") + @check_viewer_enabled def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None: """Performs update on rays/sec calculation for training diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx index d3f49ffab..8c70863c8 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx @@ -4,6 +4,7 @@ import Button from '@mui/material/Button'; import { useDispatch, useSelector } from 'react-redux'; import PauseIcon from '@mui/icons-material/Pause'; import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import CelebrationOutlinedIcon from '@mui/icons-material/CelebrationOutlined'; import ViewInArIcon from '@mui/icons-material/ViewInAr'; import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined'; import { @@ -18,7 +19,9 @@ interface StatusPanelProps { export default function StatusPanel(props: StatusPanelProps) { const dispatch = useDispatch(); const viser_websocket = React.useContext(ViserWebSocketContext); - const isTraining = useSelector((state) => state.renderingState.isTraining); + const training_state = useSelector( + (state) => state.renderingState.training_state, + ); const sceneTree = props.sceneTree; const isWebsocketConnected = useSelector( @@ -53,18 +56,38 @@ export default function StatusPanel(props: StatusPanelProps) { }, [camera_choice, is_scene_visible]); const handlePlayChange = () => { + let new_state = null; + if (training_state === 'training') { + new_state = 'paused'; + } else if (training_state === 'paused') { + new_state = 'training'; + } else { + return; + } dispatch({ type: 'write', - path: 'renderingState/isTraining', - data: !isTraining, + path: 'renderingState/training_state', + data: new_state, }); sendWebsocketMessage(viser_websocket, { - type: 'IsTrainingMessage', - is_training: !isTraining, + type: 'TrainingStateMessage', + training_state: new_state, }); }; - const is_training_text = isTraining ? 'Pause Training' : 'Resume Training'; - const training_icon = isTraining ? : ; + let is_training_text = ''; + let training_icon = null; + let color = 'secondary'; + if (training_state === 'training') { + is_training_text = 'Pause Training'; + training_icon = ; + } else if (training_state === 'paused') { + is_training_text = 'Resume Training'; + training_icon = ; + } else { + is_training_text = 'Training Complete'; + color = 'success'; + training_icon = ; + } return (
@@ -72,7 +95,7 @@ export default function StatusPanel(props: StatusPanelProps) {