mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
[wip] logging separated s.t. only one is enabled (#458)
* updated viewer/writer configs * allow viewer + tb/wandb but with errors; moved localwriter to its own thing * nits and all * update test
This commit is contained in:
1 parent
a79ef3fb11
commit
4e7ef375a2
11 files changed
+200
-166
No files matched your search
@@ -26,6 +26,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import torch
|
||||
from rich import console
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
|
||||
from nerfactory.configs import base as cfg
|
||||
@@ -37,12 +38,13 @@ from nerfactory.utils.callbacks import (
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
from nerfactory.utils.decorators import check_main_thread
|
||||
from nerfactory.utils.decorators import check_main_thread, check_viewer_enabled
|
||||
from nerfactory.utils.misc import step_check
|
||||
from nerfactory.utils.writer import EventName, TimeWriter
|
||||
from nerfactory.viewer.server import viewer_utils
|
||||
|
||||
logging.getLogger("PIL").setLevel(logging.WARNING)
|
||||
CONSOLE = console.Console()
|
||||
|
||||
|
||||
def train_loop(local_rank: int, world_size: int, config: cfg.Config) -> Any:
|
||||
@@ -81,21 +83,19 @@ class Trainer:
|
||||
self.pipeline: Pipeline
|
||||
self.optimizers: Optimizers
|
||||
self.start_step = 0
|
||||
# visualizer variable
|
||||
banner_messages = None
|
||||
self.visualizer_state = viewer_utils.VisualizerState(config.viewer, config_base_dir=self.config.base_dir)
|
||||
if config.viewer.enable:
|
||||
banner_messages = [f"Viewer at: {self.visualizer_state.viewer_url}"]
|
||||
self.grad_scaler = GradScaler(enabled=self.mixed_precision)
|
||||
# training callbacks
|
||||
self.callbacks: List[TrainingCallback]
|
||||
# logging variables
|
||||
writer.setup_event_writers(
|
||||
# optimizers
|
||||
self.grad_scaler = GradScaler(enabled=self.mixed_precision)
|
||||
# logging/viewer variables
|
||||
self.viewer_state, banner_messages = viewer_utils.setup_viewer(config.viewer)
|
||||
writer.setup_event_writer(config.logging)
|
||||
writer.setup_local_writer(
|
||||
config.logging, max_iter=config.trainer.max_num_iterations, banner_messages=banner_messages
|
||||
)
|
||||
profiler.setup_profiler(config.logging)
|
||||
|
||||
writer.put_config(name="config", config_dict=dataclasses.asdict(config), step=0)
|
||||
self._check_viewer_warnings()
|
||||
|
||||
def setup(self, test_mode=False):
|
||||
"""Setup the Trainer by calling other setup functions.
|
||||
@@ -119,13 +119,11 @@ class Trainer:
|
||||
"""Train the model."""
|
||||
assert self.pipeline.datamanager.train_input_dataset is not None, "Missing DatsetInputs"
|
||||
|
||||
self.visualizer_state.init_scene(
|
||||
dataset=self.pipeline.datamanager.train_input_dataset, start_train=self.config.viewer.start_train
|
||||
)
|
||||
self._init_viewer_scene()
|
||||
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
|
||||
num_iterations = self.config.trainer.max_num_iterations
|
||||
for step in range(self.start_step, self.start_step + num_iterations):
|
||||
# if the visualizer used, the rendering of the visualizer will be included in the iteration train time
|
||||
# if the viewer used, the rendering of the viewer will be included in the iteration train time
|
||||
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
|
||||
|
||||
# training callbacks before the training iteration
|
||||
@@ -141,17 +139,9 @@ class Trainer:
|
||||
for callback in self.callbacks:
|
||||
callback.run_callback_at_location(step, location=TrainingCallbackLocation.AFTER_TRAIN_ITERATION)
|
||||
|
||||
# NOTE(ethan): we should remove this when the viewer is disabled
|
||||
with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as vis_t:
|
||||
num_rays_per_batch = self.config.pipeline.datamanager.train_num_rays_per_batch
|
||||
try:
|
||||
self.visualizer_state.update_scene(step, self.pipeline.model, num_rays_per_batch)
|
||||
except RuntimeError:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
assert self.visualizer_state.vis is not None
|
||||
self.visualizer_state.vis["renderingState/log_errors"].write(
|
||||
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
|
||||
)
|
||||
vis_t = self._update_viewer_state(step)
|
||||
|
||||
self._update_viewer_rays_per_sec(train_t, vis_t, step)
|
||||
|
||||
# a batch of train rays
|
||||
if step_check(step, self.config.logging.steps_per_log, run_at_zero=True):
|
||||
@@ -183,12 +173,55 @@ class Trainer:
|
||||
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
|
||||
self._save_checkpoint(self.config.trainer.model_dir, step)
|
||||
|
||||
self._update_rays_per_sec(train_t, vis_t, step)
|
||||
self._update_viewer_rays_per_sec(train_t, vis_t, step)
|
||||
self._write_out_storage(step)
|
||||
|
||||
self._write_out_storage(num_iterations)
|
||||
|
||||
def _update_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int):
|
||||
def _check_viewer_warnings(self) -> None:
|
||||
"""Helper to print out any warnings regarding the way the viewer/loggers are enabled"""
|
||||
if self.config.viewer.enable:
|
||||
if self.config.logging.event_writer_config:
|
||||
string = (
|
||||
"[WARNING]: Tensorboard or Wandb enabled with Viewer will slow down Viewer. "
|
||||
"Please set `--logging.event_writer none` for faster rendering"
|
||||
)
|
||||
CONSOLE.print(f"[bold red]{string}")
|
||||
string = "[WARNING] Disabling eval iterations since viewer is enabled."
|
||||
CONSOLE.print(f"[bold red]{string}")
|
||||
|
||||
@check_viewer_enabled
|
||||
def _init_viewer_scene(self) -> None:
|
||||
"""Initializes viewer scene with given train dataset"""
|
||||
assert self.viewer_state and self.pipeline.datamanager.train_input_dataset
|
||||
self.viewer_state.init_scene(
|
||||
dataset=self.pipeline.datamanager.train_input_dataset,
|
||||
start_train=self.config.viewer.start_train,
|
||||
)
|
||||
|
||||
@check_viewer_enabled
|
||||
def _update_viewer_state(self, step: int) -> TimeWriter:
|
||||
"""Updates the viewer state by rendering out scene with current pipeline
|
||||
Returns the time taken to render scene.
|
||||
|
||||
Args:
|
||||
step: current train step
|
||||
"""
|
||||
assert self.viewer_state is not None
|
||||
with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as vis_t:
|
||||
num_rays_per_batch = self.config.pipeline.datamanager.train_num_rays_per_batch
|
||||
try:
|
||||
self.viewer_state.update_scene(step, self.pipeline.model, num_rays_per_batch)
|
||||
except RuntimeError:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
assert self.viewer_state.vis is not None
|
||||
self.viewer_state.vis["renderingState/log_errors"].write(
|
||||
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
|
||||
)
|
||||
return vis_t
|
||||
|
||||
@check_viewer_enabled
|
||||
def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int):
|
||||
"""Performs update on rays/sec calclation for training
|
||||
|
||||
Args:
|
||||
|
||||
Reference in new issue
Block a user