mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 13:53:44 +08:00
Add completed training state (#1768)
This commit is contained in:
1 parent
b7fd273638
commit
cbb0511619
10 files changed
+100
-59
No files matched your search
@@ -104,7 +104,7 @@ class Trainer:
|
||||
pipeline: The pipeline object.
|
||||
optimizers: The optimizers object.
|
||||
callbacks: The callbacks object.
|
||||
is_training: Whether the model is training.
|
||||
training_state: Current model training state.
|
||||
"""
|
||||
|
||||
pipeline: VanillaPipeline
|
||||
@@ -118,7 +118,7 @@ class Trainer:
|
||||
self.world_size = world_size
|
||||
self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}"
|
||||
self.mixed_precision: bool = self.config.mixed_precision
|
||||
self.is_training: bool = True
|
||||
self.training_state: Literal["training", "paused", "completed"] = "training"
|
||||
if self.device == "cpu":
|
||||
self.mixed_precision = False
|
||||
CONSOLE.print("Mixed precision is disabled for CPU training.")
|
||||
@@ -216,7 +216,7 @@ class Trainer:
|
||||
num_iterations = self.config.max_num_iterations
|
||||
step = 0
|
||||
for step in range(self._start_step, self._start_step + num_iterations):
|
||||
while not self.is_training:
|
||||
while self.training_state == "paused":
|
||||
time.sleep(0.01)
|
||||
with self.train_lock:
|
||||
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
|
||||
@@ -280,6 +280,8 @@ class Trainer:
|
||||
CONSOLE.rule()
|
||||
CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center")
|
||||
if not self.config.viewer.quit_on_train_completion:
|
||||
self.training_state = "completed"
|
||||
self._train_complete_viewer()
|
||||
CONSOLE.print("Use ctrl+c to quit", justify="center")
|
||||
while True:
|
||||
time.sleep(0.01)
|
||||
@@ -304,7 +306,7 @@ class Trainer:
|
||||
assert self.viewer_state and self.pipeline.datamanager.train_dataset
|
||||
self.viewer_state.init_scene(
|
||||
dataset=self.pipeline.datamanager.train_dataset,
|
||||
start_train=True,
|
||||
train_state="training",
|
||||
)
|
||||
|
||||
@check_viewer_enabled
|
||||
@@ -323,6 +325,16 @@ class Trainer:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
CONSOLE.log("Viewer failed. Continuing training.")
|
||||
|
||||
@check_viewer_enabled
|
||||
def _train_complete_viewer(self) -> None:
|
||||
"""Let the viewer know that the training is complete"""
|
||||
assert self.viewer_state is not None
|
||||
try:
|
||||
self.viewer_state.training_complete()
|
||||
except RuntimeError:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
CONSOLE.log("Viewer failed. Continuing training.")
|
||||
|
||||
@check_viewer_enabled
|
||||
def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None:
|
||||
"""Performs update on rays/sec calculation for training
|
||||
|
||||
@@ -4,6 +4,7 @@ import Button from '@mui/material/Button';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import PauseIcon from '@mui/icons-material/Pause';
|
||||
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
|
||||
import CelebrationOutlinedIcon from '@mui/icons-material/CelebrationOutlined';
|
||||
import ViewInArIcon from '@mui/icons-material/ViewInAr';
|
||||
import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined';
|
||||
import {
|
||||
@@ -18,7 +19,9 @@ interface StatusPanelProps {
|
||||
export default function StatusPanel(props: StatusPanelProps) {
|
||||
const dispatch = useDispatch();
|
||||
const viser_websocket = React.useContext(ViserWebSocketContext);
|
||||
const isTraining = useSelector((state) => state.renderingState.isTraining);
|
||||
const training_state = useSelector(
|
||||
(state) => state.renderingState.training_state,
|
||||
);
|
||||
const sceneTree = props.sceneTree;
|
||||
|
||||
const isWebsocketConnected = useSelector(
|
||||
@@ -53,18 +56,38 @@ export default function StatusPanel(props: StatusPanelProps) {
|
||||
}, [camera_choice, is_scene_visible]);
|
||||
|
||||
const handlePlayChange = () => {
|
||||
let new_state = null;
|
||||
if (training_state === 'training') {
|
||||
new_state = 'paused';
|
||||
} else if (training_state === 'paused') {
|
||||
new_state = 'training';
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
dispatch({
|
||||
type: 'write',
|
||||
path: 'renderingState/isTraining',
|
||||
data: !isTraining,
|
||||
path: 'renderingState/training_state',
|
||||
data: new_state,
|
||||
});
|
||||
sendWebsocketMessage(viser_websocket, {
|
||||
type: 'IsTrainingMessage',
|
||||
is_training: !isTraining,
|
||||
type: 'TrainingStateMessage',
|
||||
training_state: new_state,
|
||||
});
|
||||
};
|
||||
const is_training_text = isTraining ? 'Pause Training' : 'Resume Training';
|
||||
const training_icon = isTraining ? <PauseIcon /> : <PlayArrowIcon />;
|
||||
let is_training_text = '';
|
||||
let training_icon = null;
|
||||
let color = 'secondary';
|
||||
if (training_state === 'training') {
|
||||
is_training_text = 'Pause Training';
|
||||
training_icon = <PauseIcon />;
|
||||
} else if (training_state === 'paused') {
|
||||
is_training_text = 'Resume Training';
|
||||
training_icon = <PlayArrowIcon />;
|
||||
} else {
|
||||
is_training_text = 'Training Complete';
|
||||
color = 'success';
|
||||
training_icon = <CelebrationOutlinedIcon />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="StatusPanel">
|
||||
@@ -72,7 +95,7 @@ export default function StatusPanel(props: StatusPanelProps) {
|
||||
<Button
|
||||
className="StatusPanel-play-button"
|
||||
variant="contained"
|
||||
color="secondary"
|
||||
color={color}
|
||||
onClick={handlePlayChange}
|
||||
disabled={!isWebsocketConnected}
|
||||
startIcon={training_icon}
|
||||
|
||||
@@ -78,9 +78,9 @@ interface DatasetImageMessage {
|
||||
idx: string;
|
||||
json: any;
|
||||
}
|
||||
interface IsTrainingMessage {
|
||||
type: 'IsTrainingMessage';
|
||||
is_training: boolean;
|
||||
interface TrainingStateMessage {
|
||||
type: 'TrainingStateMessage';
|
||||
training_state: 'training' | 'paused' | 'completed';
|
||||
}
|
||||
interface CameraPathPayloadMessage {
|
||||
type: 'CameraPathPayloadMessage';
|
||||
@@ -129,7 +129,7 @@ export type Message =
|
||||
| CameraMessage
|
||||
| SceneBoxMessage
|
||||
| DatasetImageMessage
|
||||
| IsTrainingMessage
|
||||
| TrainingStateMessage
|
||||
| CameraPathPayloadMessage
|
||||
| CameraPathOptionsRequest
|
||||
| CameraPathsMessage
|
||||
|
||||
@@ -184,11 +184,11 @@ function handleMessage(
|
||||
break;
|
||||
}
|
||||
// Set training value.
|
||||
case 'IsTrainingMessage': {
|
||||
case 'TrainingStateMessage': {
|
||||
dispatch({
|
||||
type: 'write',
|
||||
path: 'renderingState/isTraining',
|
||||
data: message.is_training,
|
||||
path: 'renderingState/training_state',
|
||||
data: message.training_state,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ const initialState = {
|
||||
field_of_view: 50,
|
||||
camera_type: 'perspective',
|
||||
|
||||
isTraining: true,
|
||||
training_state: 'training',
|
||||
|
||||
step: 0,
|
||||
eval_res: '?',
|
||||
|
||||
@@ -4,6 +4,7 @@ export const appTheme = createTheme({
|
||||
palette: {
|
||||
primary: { main: '#EEEEEE' },
|
||||
secondary: { main: '#FFD369' },
|
||||
success: { main: '#5daf5d' },
|
||||
text: {
|
||||
primary: '#EEEEEE',
|
||||
secondary: '#FFD369',
|
||||
|
||||
@@ -25,6 +25,7 @@ from rich import box, style
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.configs import base_config as cfg
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
@@ -48,10 +49,10 @@ from nerfstudio.viewer.viser.messages import (
|
||||
CameraPathOptionsRequest,
|
||||
CameraPathPayloadMessage,
|
||||
CropParamsMessage,
|
||||
IsTrainingMessage,
|
||||
NerfstudioMessage,
|
||||
SaveCheckpointMessage,
|
||||
TimeConditionMessage,
|
||||
TrainingStateMessage,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -114,20 +115,16 @@ class ViewerState:
|
||||
self.include_time = self.pipeline.datamanager.includes_time
|
||||
|
||||
# viewer specific variables
|
||||
self.prev_moving = False
|
||||
self.output_type_changed = True
|
||||
self.check_done_render = True
|
||||
self.step = 0
|
||||
self.camera_moving = False
|
||||
self.prev_camera_timestamp = 0
|
||||
self.static_fps = 1.0
|
||||
self.train_btn_state = True
|
||||
self.train_btn_state: Literal["training", "paused", "completed"] = "training"
|
||||
self._prev_train_state: Literal["training", "paused", "completed"] = "training"
|
||||
|
||||
self.camera_message = None
|
||||
|
||||
self.viser_server = ViserServer(host="0.0.0.0", port=websocket_port)
|
||||
|
||||
self.viser_server.register_handler(IsTrainingMessage, self._handle_is_training)
|
||||
self.viser_server.register_handler(TrainingStateMessage, self._handle_training_state_message)
|
||||
self.viser_server.register_handler(SaveCheckpointMessage, self._handle_save_checkpoint)
|
||||
self.viser_server.register_handler(CameraMessage, self._handle_camera_update)
|
||||
self.viser_server.register_handler(CameraPathOptionsRequest, self._handle_camera_path_option_request)
|
||||
@@ -184,15 +181,15 @@ class ViewerState:
|
||||
crop_center=tuple(crop_center.tolist()),
|
||||
)
|
||||
|
||||
def _handle_is_training(self, message: NerfstudioMessage) -> None:
|
||||
"""Handle is_training message from viewer."""
|
||||
assert isinstance(message, IsTrainingMessage)
|
||||
self.is_training = message.is_training
|
||||
self.train_btn_state = message.is_training
|
||||
self.viser_server.set_is_training(message.is_training)
|
||||
def _handle_training_state_message(self, message: NerfstudioMessage) -> None:
|
||||
"""Handle training state message from viewer."""
|
||||
assert isinstance(message, TrainingStateMessage)
|
||||
self.train_btn_state = message.training_state
|
||||
self.training_state = message.training_state
|
||||
self.viser_server.set_training_state(message.training_state)
|
||||
|
||||
def _handle_save_checkpoint(self, message: NerfstudioMessage) -> None:
|
||||
"""Handle is_training message from viewer."""
|
||||
"""Handle save checkpoint message from viewer."""
|
||||
assert isinstance(message, SaveCheckpointMessage)
|
||||
if self.trainer is not None:
|
||||
self.trainer.save_checkpoint(self.step)
|
||||
@@ -201,13 +198,13 @@ class ViewerState:
|
||||
"""Handle camera update message from viewer."""
|
||||
assert isinstance(message, CameraMessage)
|
||||
self.camera_message = message
|
||||
self.camera_moving = message.is_moving
|
||||
if message.is_moving:
|
||||
self.render_statemachine.action(RenderAction("move", self.camera_message))
|
||||
self.is_training = False
|
||||
if self.training_state == "training":
|
||||
self.training_state = "paused"
|
||||
else:
|
||||
self.render_statemachine.action(RenderAction("static", self.camera_message))
|
||||
self.is_training = self.train_btn_state
|
||||
self.training_state = self.train_btn_state
|
||||
|
||||
def _handle_camera_path_option_request(self, message: NerfstudioMessage) -> None:
|
||||
"""Handle camera path option request message from viewer."""
|
||||
@@ -247,17 +244,17 @@ class ViewerState:
|
||||
self.control_panel.time = message.time
|
||||
|
||||
@property
|
||||
def is_training(self) -> bool:
|
||||
"""Get is_training flag from viewer."""
|
||||
def training_state(self) -> Literal["training", "paused", "completed"]:
|
||||
"""Get training state flag."""
|
||||
if self.trainer is not None:
|
||||
return self.trainer.is_training
|
||||
return False
|
||||
return self.trainer.training_state
|
||||
return self.train_btn_state
|
||||
|
||||
@is_training.setter
|
||||
def is_training(self, is_training: bool) -> None:
|
||||
"""Set is_training flag in viewer."""
|
||||
@training_state.setter
|
||||
def training_state(self, training_state: Literal["training", "paused", "completed"]) -> None:
|
||||
"""Set training state flag."""
|
||||
if self.trainer is not None:
|
||||
self.trainer.is_training = is_training
|
||||
self.trainer.training_state = training_state
|
||||
|
||||
def _pick_drawn_image_idxs(self, total_num: int) -> list[int]:
|
||||
"""Determine indicies of images to display in viewer.
|
||||
@@ -275,13 +272,12 @@ class ViewerState:
|
||||
# draw indices, roughly evenly spaced
|
||||
return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist()
|
||||
|
||||
def init_scene(self, dataset: InputDataset, start_train=True) -> None:
|
||||
def init_scene(self, dataset: InputDataset, train_state=Literal["training", "paused", "completed"]) -> None:
|
||||
"""Draw some images and the scene aabb in the viewer.
|
||||
|
||||
Args:
|
||||
dataset: dataset to render in the scene
|
||||
start_train: whether to start train when viewer init;
|
||||
if False, only displays dataset until resume train is toggled
|
||||
train_state: Current status of training
|
||||
"""
|
||||
self.viser_server.send_file_path_info(
|
||||
config_base_dir=self.log_filename.parents[0],
|
||||
@@ -301,8 +297,8 @@ class ViewerState:
|
||||
self.viser_server.update_scene_box(dataset.scene_box)
|
||||
|
||||
# set the initial state whether to train or not
|
||||
self.train_btn_state = start_train
|
||||
self.viser_server.set_is_training(start_train)
|
||||
self.train_btn_state = train_state
|
||||
self.viser_server.set_training_state(train_state)
|
||||
|
||||
def update_scene(self, step: int, num_rays_per_batch: Optional[int] = None) -> None:
|
||||
"""updates the scene based on the graph weights
|
||||
@@ -316,7 +312,11 @@ class ViewerState:
|
||||
if self.camera_message is None:
|
||||
return
|
||||
|
||||
if self.trainer is not None and self.trainer.is_training and self.control_panel.train_util != 1:
|
||||
if (
|
||||
self.trainer is not None
|
||||
and self.trainer.training_state == "training"
|
||||
and self.control_panel.train_util != 1
|
||||
):
|
||||
if (
|
||||
EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
|
||||
and EventName.VIS_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"]
|
||||
@@ -350,3 +350,8 @@ class ViewerState:
|
||||
def get_model(self) -> Model:
|
||||
"""Returns the model."""
|
||||
return self.pipeline.model
|
||||
|
||||
def training_complete(self) -> None:
|
||||
"""Called when training is complete."""
|
||||
self.training_state = "completed"
|
||||
self.viser_server.set_training_state("completed")
|
||||
@@ -439,13 +439,13 @@ class MessageApi(abc.ABC):
|
||||
"""
|
||||
self._queue(messages.DatasetImageMessage(idx=idx, json=json))
|
||||
|
||||
def set_is_training(self, is_training: bool) -> None:
|
||||
def set_training_state(self, training_state: Literal["training", "paused", "completed"]) -> None:
|
||||
"""Set the training mode.
|
||||
|
||||
Args:
|
||||
is_training: The training mode.
|
||||
training_state: The training mode.
|
||||
"""
|
||||
self._queue(messages.IsTrainingMessage(is_training=is_training))
|
||||
self._queue(messages.TrainingStateMessage(training_state=training_state))
|
||||
|
||||
def send_camera_paths(self, camera_paths: Dict[str, Any]) -> None:
|
||||
"""Send camera paths to the scene.
|
||||
|
||||
@@ -171,10 +171,10 @@ class DatasetImageMessage(NerfstudioMessage):
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class IsTrainingMessage(NerfstudioMessage):
|
||||
class TrainingStateMessage(NerfstudioMessage):
|
||||
"""Wheather the scene is in training mode or not."""
|
||||
|
||||
is_training: bool
|
||||
training_state: Literal["training", "paused", "completed"]
|
||||
"""True if the model is currently trianing, False otherwise"""
|
||||
|
||||
|
||||
|
||||
@@ -87,9 +87,9 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
|
||||
assert viewer_state and pipeline.datamanager.train_dataset
|
||||
viewer_state.init_scene(
|
||||
dataset=pipeline.datamanager.train_dataset,
|
||||
start_train=False,
|
||||
train_state="completed",
|
||||
)
|
||||
viewer_state.viser_server.set_is_training(False)
|
||||
viewer_state.viser_server.set_training_state("completed")
|
||||
viewer_state.update_scene(step=step)
|
||||
while True:
|
||||
time.sleep(0.01)
|
||||
|
||||
Reference in new issue
Block a user