Add completed training state (#1768)

This commit is contained in:
Matthew Tancik authored and GitHub committed 2023-04-18 16:55:38 -07:00
1 parent b7fd273638
commit cbb0511619
10 files changed
+100 -59

No files matched your search

+16 -4
View File
@@ -104,7 +104,7 @@ class Trainer:
pipeline: The pipeline object.
optimizers: The optimizers object.
callbacks: The callbacks object.
is_training: Whether the model is training.
training_state: Current model training state.
"""
pipeline: VanillaPipeline
@@ -118,7 +118,7 @@ class Trainer:
self.world_size = world_size
self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}"
self.mixed_precision: bool = self.config.mixed_precision
self.is_training: bool = True
self.training_state: Literal["training", "paused", "completed"] = "training"
if self.device == "cpu":
self.mixed_precision = False
CONSOLE.print("Mixed precision is disabled for CPU training.")
@@ -216,7 +216,7 @@ class Trainer:
num_iterations = self.config.max_num_iterations
step = 0
for step in range(self._start_step, self._start_step + num_iterations):
while not self.is_training:
while self.training_state == "paused":
time.sleep(0.01)
with self.train_lock:
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
@@ -280,6 +280,8 @@ class Trainer:
CONSOLE.rule()
CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center")
if not self.config.viewer.quit_on_train_completion:
self.training_state = "completed"
self._train_complete_viewer()
CONSOLE.print("Use ctrl+c to quit", justify="center")
while True:
time.sleep(0.01)
@@ -304,7 +306,7 @@ class Trainer:
assert self.viewer_state and self.pipeline.datamanager.train_dataset
self.viewer_state.init_scene(
dataset=self.pipeline.datamanager.train_dataset,
start_train=True,
train_state="training",
)
@check_viewer_enabled
@@ -323,6 +325,16 @@ class Trainer:
time.sleep(0.03) # sleep to allow buffer to reset
CONSOLE.log("Viewer failed. Continuing training.")
@check_viewer_enabled
def _train_complete_viewer(self) -> None:
"""Let the viewer know that the training is complete"""
assert self.viewer_state is not None
try:
self.viewer_state.training_complete()
except RuntimeError:
time.sleep(0.03) # sleep to allow buffer to reset
CONSOLE.log("Viewer failed. Continuing training.")
@check_viewer_enabled
def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None:
"""Performs update on rays/sec calculation for training
@@ -4,6 +4,7 @@ import Button from '@mui/material/Button';
import { useDispatch, useSelector } from 'react-redux';
import PauseIcon from '@mui/icons-material/Pause';
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
import CelebrationOutlinedIcon from '@mui/icons-material/CelebrationOutlined';
import ViewInArIcon from '@mui/icons-material/ViewInAr';
import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined';
import {
@@ -18,7 +19,9 @@ interface StatusPanelProps {
export default function StatusPanel(props: StatusPanelProps) {
const dispatch = useDispatch();
const viser_websocket = React.useContext(ViserWebSocketContext);
const isTraining = useSelector((state) => state.renderingState.isTraining);
const training_state = useSelector(
(state) => state.renderingState.training_state,
);
const sceneTree = props.sceneTree;
const isWebsocketConnected = useSelector(
@@ -53,18 +56,38 @@ export default function StatusPanel(props: StatusPanelProps) {
}, [camera_choice, is_scene_visible]);
const handlePlayChange = () => {
let new_state = null;
if (training_state === 'training') {
new_state = 'paused';
} else if (training_state === 'paused') {
new_state = 'training';
} else {
return;
}
dispatch({
type: 'write',
path: 'renderingState/isTraining',
data: !isTraining,
path: 'renderingState/training_state',
data: new_state,
});
sendWebsocketMessage(viser_websocket, {
type: 'IsTrainingMessage',
is_training: !isTraining,
type: 'TrainingStateMessage',
training_state: new_state,
});
};
const is_training_text = isTraining ? 'Pause Training' : 'Resume Training';
const training_icon = isTraining ? <PauseIcon /> : <PlayArrowIcon />;
let is_training_text = '';
let training_icon = null;
let color = 'secondary';
if (training_state === 'training') {
is_training_text = 'Pause Training';
training_icon = <PauseIcon />;
} else if (training_state === 'paused') {
is_training_text = 'Resume Training';
training_icon = <PlayArrowIcon />;
} else {
is_training_text = 'Training Complete';
color = 'success';
training_icon = <CelebrationOutlinedIcon />;
}
return (
<div className="StatusPanel">
@@ -72,7 +95,7 @@ export default function StatusPanel(props: StatusPanelProps) {
<Button
className="StatusPanel-play-button"
variant="contained"
color="secondary"
color={color}
onClick={handlePlayChange}
disabled={!isWebsocketConnected}
startIcon={training_icon}
@@ -78,9 +78,9 @@ interface DatasetImageMessage {
idx: string;
json: any;
}
interface IsTrainingMessage {
type: 'IsTrainingMessage';
is_training: boolean;
interface TrainingStateMessage {
type: 'TrainingStateMessage';
training_state: 'training' | 'paused' | 'completed';
}
interface CameraPathPayloadMessage {
type: 'CameraPathPayloadMessage';
@@ -129,7 +129,7 @@ export type Message =
| CameraMessage
| SceneBoxMessage
| DatasetImageMessage
| IsTrainingMessage
| TrainingStateMessage
| CameraPathPayloadMessage
| CameraPathOptionsRequest
| CameraPathsMessage
@@ -184,11 +184,11 @@ function handleMessage(
break;
}
// Set training value.
case 'IsTrainingMessage': {
case 'TrainingStateMessage': {
dispatch({
type: 'write',
path: 'renderingState/isTraining',
data: message.is_training,
path: 'renderingState/training_state',
data: message.training_state,
});
break;
}
+1 -1
View File
@@ -40,7 +40,7 @@ const initialState = {
field_of_view: 50,
camera_type: 'perspective',
isTraining: true,
training_state: 'training',
step: 0,
eval_res: '?',
@@ -4,6 +4,7 @@ export const appTheme = createTheme({
palette: {
primary: { main: '#EEEEEE' },
secondary: { main: '#FFD369' },
success: { main: '#5daf5d' },
text: {
primary: '#EEEEEE',
secondary: '#FFD369',
+37 -32
View File
@@ -25,6 +25,7 @@ from rich import box, style
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from typing_extensions import Literal
from nerfstudio.configs import base_config as cfg
from nerfstudio.data.datasets.base_dataset import InputDataset
@@ -48,10 +49,10 @@ from nerfstudio.viewer.viser.messages import (
CameraPathOptionsRequest,
CameraPathPayloadMessage,
CropParamsMessage,
IsTrainingMessage,
NerfstudioMessage,
SaveCheckpointMessage,
TimeConditionMessage,
TrainingStateMessage,
)
if TYPE_CHECKING:
@@ -114,20 +115,16 @@ class ViewerState:
self.include_time = self.pipeline.datamanager.includes_time
# viewer specific variables
self.prev_moving = False
self.output_type_changed = True
self.check_done_render = True
self.step = 0
self.camera_moving = False
self.prev_camera_timestamp = 0
self.static_fps = 1.0
self.train_btn_state = True
self.train_btn_state: Literal["training", "paused", "completed"] = "training"
self._prev_train_state: Literal["training", "paused", "completed"] = "training"
self.camera_message = None
self.viser_server = ViserServer(host="0.0.0.0", port=websocket_port)
self.viser_server.register_handler(IsTrainingMessage, self._handle_is_training)
self.viser_server.register_handler(TrainingStateMessage, self._handle_training_state_message)
self.viser_server.register_handler(SaveCheckpointMessage, self._handle_save_checkpoint)
self.viser_server.register_handler(CameraMessage, self._handle_camera_update)
self.viser_server.register_handler(CameraPathOptionsRequest, self._handle_camera_path_option_request)
@@ -184,15 +181,15 @@ class ViewerState:
crop_center=tuple(crop_center.tolist()),
)
def _handle_is_training(self, message: NerfstudioMessage) -> None:
"""Handle is_training message from viewer."""
assert isinstance(message, IsTrainingMessage)
self.is_training = message.is_training
self.train_btn_state = message.is_training
self.viser_server.set_is_training(message.is_training)
def _handle_training_state_message(self, message: NerfstudioMessage) -> None:
"""Handle training state message from viewer."""
assert isinstance(message, TrainingStateMessage)
self.train_btn_state = message.training_state
self.training_state = message.training_state
self.viser_server.set_training_state(message.training_state)
def _handle_save_checkpoint(self, message: NerfstudioMessage) -> None:
"""Handle is_training message from viewer."""
"""Handle save checkpoint message from viewer."""
assert isinstance(message, SaveCheckpointMessage)
if self.trainer is not None:
self.trainer.save_checkpoint(self.step)
@@ -201,13 +198,13 @@ class ViewerState:
"""Handle camera update message from viewer."""
assert isinstance(message, CameraMessage)
self.camera_message = message
self.camera_moving = message.is_moving
if message.is_moving:
self.render_statemachine.action(RenderAction("move", self.camera_message))
self.is_training = False
if self.training_state == "training":
self.training_state = "paused"
else:
self.render_statemachine.action(RenderAction("static", self.camera_message))
self.is_training = self.train_btn_state
self.training_state = self.train_btn_state
def _handle_camera_path_option_request(self, message: NerfstudioMessage) -> None:
"""Handle camera path option request message from viewer."""
@@ -247,17 +244,17 @@ class ViewerState:
self.control_panel.time = message.time
@property
def is_training(self) -> bool:
"""Get is_training flag from viewer."""
def training_state(self) -> Literal["training", "paused", "completed"]:
"""Get training state flag."""
if self.trainer is not None:
return self.trainer.is_training
return False
return self.trainer.training_state
return self.train_btn_state
@is_training.setter
def is_training(self, is_training: bool) -> None:
"""Set is_training flag in viewer."""
@training_state.setter
def training_state(self, training_state: Literal["training", "paused", "completed"]) -> None:
"""Set training state flag."""
if self.trainer is not None:
self.trainer.is_training = is_training
self.trainer.training_state = training_state
def _pick_drawn_image_idxs(self, total_num: int) -> list[int]:
"""Determine indicies of images to display in viewer.
@@ -275,13 +272,12 @@ class ViewerState:
# draw indices, roughly evenly spaced
return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist()
def init_scene(self, dataset: InputDataset, start_train=True) -> None:
def init_scene(self, dataset: InputDataset, train_state=Literal["training", "paused", "completed"]) -> None:
"""Draw some images and the scene aabb in the viewer.
Args:
dataset: dataset to render in the scene
start_train: whether to start train when viewer init;
if False, only displays dataset until resume train is toggled
train_state: Current status of training
"""
self.viser_server.send_file_path_info(
config_base_dir=self.log_filename.parents[0],
@@ -301,8 +297,8 @@ class ViewerState:
self.viser_server.update_scene_box(dataset.scene_box)
# set the initial state whether to train or not
self.train_btn_state = start_train
self.viser_server.set_is_training(start_train)
self.train_btn_state = train_state
self.viser_server.set_training_state(train_state)
def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None:
"""updates the scene based on the graph weights
@@ -316,7 +312,11 @@ class ViewerState:
if self.camera_message is None:
return
if self.trainer is not None and self.trainer.is_training and self.control_panel.train_util != 1:
if (
self.trainer is not None
and self.trainer.training_state == "training"
and self.control_panel.train_util != 1
):
if (
EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
@@ -350,3 +350,8 @@ class ViewerState:
def get_model(self) -> Model:
"""Returns the model."""
return self.pipeline.model
def training_complete(self) -> None:
"""Called when training is complete."""
self.training_state = "completed"
self.viser_server.set_training_state("completed")
+3 -3
View File
@@ -439,13 +439,13 @@ class MessageApi(abc.ABC):
"""
self._queue(messages.DatasetImageMessage(idx=idx, json=json))
def set_is_training(self, is_training: bool) -> None:
def set_training_state(self, training_state: Literal["training", "paused", "completed"]) -> None:
"""Set the training mode.
Args:
is_training: The training mode.
training_state: The training mode.
"""
self._queue(messages.IsTrainingMessage(is_training=is_training))
self._queue(messages.TrainingStateMessage(training_state=training_state))
def send_camera_paths(self, camera_paths: Dict[str, Any]) -> None:
"""Send camera paths to the scene.
+2 -2
View File
@@ -171,10 +171,10 @@ class DatasetImageMessage(NerfstudioMessage):
@dataclasses.dataclass
class IsTrainingMessage(NerfstudioMessage):
class TrainingStateMessage(NerfstudioMessage):
"""Wheather the scene is in training mode or not."""
is_training: bool
training_state: Literal["training", "paused", "completed"]
"""True if the model is currently trianing, False otherwise"""
+2 -2
View File
@@ -87,9 +87,9 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
assert viewer_state and pipeline.datamanager.train_dataset
viewer_state.init_scene(
dataset=pipeline.datamanager.train_dataset,
start_train=False,
train_state="completed",
)
viewer_state.viser_server.set_is_training(False)
viewer_state.viser_server.set_training_state("completed")
viewer_state.update_scene(step=step)
while True:
time.sleep(0.01)