diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 9b4d6d8a1..0634f62e5 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -28,6 +28,7 @@ from threading import Lock from typing import DefaultDict, Dict, List, Literal, Optional, Tuple, Type, cast import torch +import viser from rich import box, style from rich.panel import Panel from rich.table import Table @@ -137,6 +138,9 @@ class Trainer: self.viewer_state = None + # used to keep track of the current step + self.step = 0 + def setup(self, test_mode: Literal["test", "val", "inference"] = "val") -> None: """Setup the Trainer by calling other setup functions. @@ -233,8 +237,15 @@ class Trainer: with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME): num_iterations = self.config.max_num_iterations step = 0 + self.stop_training = False for step in range(self._start_step, self._start_step + num_iterations): + self.step = step + if self.stop_training: + break while self.training_state == "paused": + if self.stop_training: + self._after_train() + return time.sleep(0.01) with self.train_lock: with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: @@ -291,12 +302,26 @@ class Trainer: writer.write_out_storage() - # save checkpoint at the end of training - self.save_checkpoint(step) + # save checkpoint at the end of training, and write out any remaining events + self._after_train() + def shutdown(self) -> None: + """Stop the trainer and stop all associated threads/processes (such as the viewer).""" + self.stop_training = True # tell the training loop to stop + if self.viewer_state is not None: + # stop the viewer + # this condition excludes the case where `viser_server` is either `None` or an + # instance of `viewer_legacy`'s `ViserServer` instead of the upstream one. + if isinstance(self.viewer_state.viser_server, viser.ViserServer): + self.viewer_state.viser_server.stop() + + def _after_train(self) -> None: + """Function to run after training is complete""" + self.training_state = "completed" # used to update the webui state + # save checkpoint at the end of training + self.save_checkpoint(self.step) # write out any remaining events (e.g., total train time) writer.write_out_storage() - table = Table( title=None, show_header=False, @@ -309,7 +334,7 @@ class Trainer: # after train end callbacks for callback in self.callbacks: - callback.run_callback_at_location(step=step, location=TrainingCallbackLocation.AFTER_TRAIN) + callback.run_callback_at_location(step=self.step, location=TrainingCallbackLocation.AFTER_TRAIN) if not self.config.viewer.quit_on_train_completion: self._train_complete_viewer()