diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py
index 1dddb21d5..210decdb2 100644
--- a/nerfstudio/engine/trainer.py
+++ b/nerfstudio/engine/trainer.py
@@ -104,7 +104,7 @@ class Trainer:
pipeline: The pipeline object.
optimizers: The optimizers object.
callbacks: The callbacks object.
- is_training: Whether the model is training.
+ training_state: Current model training state.
"""
pipeline: VanillaPipeline
@@ -118,7 +118,7 @@ class Trainer:
self.world_size = world_size
self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}"
self.mixed_precision: bool = self.config.mixed_precision
- self.is_training: bool = True
+ self.training_state: Literal["training", "paused", "completed"] = "training"
if self.device == "cpu":
self.mixed_precision = False
CONSOLE.print("Mixed precision is disabled for CPU training.")
@@ -216,7 +216,7 @@ class Trainer:
num_iterations = self.config.max_num_iterations
step = 0
for step in range(self._start_step, self._start_step + num_iterations):
- while not self.is_training:
+ while self.training_state == "paused":
time.sleep(0.01)
with self.train_lock:
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
@@ -280,6 +280,8 @@ class Trainer:
CONSOLE.rule()
CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center")
if not self.config.viewer.quit_on_train_completion:
+ self.training_state = "completed"
+ self._train_complete_viewer()
CONSOLE.print("Use ctrl+c to quit", justify="center")
while True:
time.sleep(0.01)
@@ -304,7 +306,7 @@ class Trainer:
assert self.viewer_state and self.pipeline.datamanager.train_dataset
self.viewer_state.init_scene(
dataset=self.pipeline.datamanager.train_dataset,
- start_train=True,
+ train_state="training",
)
@check_viewer_enabled
@@ -323,6 +325,16 @@ class Trainer:
time.sleep(0.03) # sleep to allow buffer to reset
CONSOLE.log("Viewer failed. Continuing training.")
+ @check_viewer_enabled
+ def _train_complete_viewer(self) -> None:
+ """Let the viewer know that the training is complete"""
+ assert self.viewer_state is not None
+ try:
+ self.viewer_state.training_complete()
+ except RuntimeError:
+ time.sleep(0.03) # sleep to allow buffer to reset
+ CONSOLE.log("Viewer failed. Continuing training.")
+
@check_viewer_enabled
def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None:
"""Performs update on rays/sec calculation for training
diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx
index d3f49ffab..8c70863c8 100644
--- a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx
+++ b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx
@@ -4,6 +4,7 @@ import Button from '@mui/material/Button';
import { useDispatch, useSelector } from 'react-redux';
import PauseIcon from '@mui/icons-material/Pause';
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import CelebrationOutlinedIcon from '@mui/icons-material/CelebrationOutlined';
import ViewInArIcon from '@mui/icons-material/ViewInAr';
import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined';
import {
@@ -18,7 +19,9 @@ interface StatusPanelProps {
export default function StatusPanel(props: StatusPanelProps) {
const dispatch = useDispatch();
const viser_websocket = React.useContext(ViserWebSocketContext);
- const isTraining = useSelector((state) => state.renderingState.isTraining);
+ const training_state = useSelector(
+ (state) => state.renderingState.training_state,
+ );
const sceneTree = props.sceneTree;
const isWebsocketConnected = useSelector(
@@ -53,18 +56,38 @@ export default function StatusPanel(props: StatusPanelProps) {
}, [camera_choice, is_scene_visible]);
const handlePlayChange = () => {
+ let new_state = null;
+ if (training_state === 'training') {
+ new_state = 'paused';
+ } else if (training_state === 'paused') {
+ new_state = 'training';
+ } else {
+ return;
+ }
dispatch({
type: 'write',
- path: 'renderingState/isTraining',
- data: !isTraining,
+ path: 'renderingState/training_state',
+ data: new_state,
});
sendWebsocketMessage(viser_websocket, {
- type: 'IsTrainingMessage',
- is_training: !isTraining,
+ type: 'TrainingStateMessage',
+ training_state: new_state,
});
};
- const is_training_text = isTraining ? 'Pause Training' : 'Resume Training';
- const training_icon = isTraining ? : ;
+ let is_training_text = '';
+ let training_icon = null;
+ let color = 'secondary';
+ if (training_state === 'training') {
+ is_training_text = 'Pause Training';
+ training_icon = ;
+ } else if (training_state === 'paused') {
+ is_training_text = 'Resume Training';
+ training_icon = ;
+ } else {
+ is_training_text = 'Training Complete';
+ color = 'success';
+ training_icon = ;
+ }
return (
@@ -72,7 +95,7 @@ export default function StatusPanel(props: StatusPanelProps) {