[wip] logging separated s.t. only one is enabled (#458)

* updated viewer/writer configs

* allow viewer + tb/wandb but with errors; moved localwriter to its own thing

* nits and all

* update test
This commit is contained in:
evonneng authored and GitHub committed 2022-09-17 17:50:11 -07:00
1 parent a79ef3fb11
commit 4e7ef375a2
11 files changed
+200 -166

No files matched your search

+60 -27
View File
@@ -26,6 +26,7 @@ from pathlib import Path
from typing import Any, Dict, List
import torch
from rich import console
from torch.cuda.amp.grad_scaler import GradScaler
from nerfactory.configs import base as cfg
@@ -37,12 +38,13 @@ from nerfactory.utils.callbacks import (
TrainingCallbackAttributes,
TrainingCallbackLocation,
)
from nerfactory.utils.decorators import check_main_thread
from nerfactory.utils.decorators import check_main_thread, check_viewer_enabled
from nerfactory.utils.misc import step_check
from nerfactory.utils.writer import EventName, TimeWriter
from nerfactory.viewer.server import viewer_utils
logging.getLogger("PIL").setLevel(logging.WARNING)
CONSOLE = console.Console()
def train_loop(local_rank: int, world_size: int, config: cfg.Config) -> Any:
@@ -81,21 +83,19 @@ class Trainer:
self.pipeline: Pipeline
self.optimizers: Optimizers
self.start_step = 0
# visualizer variable
banner_messages = None
self.visualizer_state = viewer_utils.VisualizerState(config.viewer, config_base_dir=self.config.base_dir)
if config.viewer.enable:
banner_messages = [f"Viewer at: {self.visualizer_state.viewer_url}"]
self.grad_scaler = GradScaler(enabled=self.mixed_precision)
# training callbacks
self.callbacks: List[TrainingCallback]
# logging variables
writer.setup_event_writers(
# optimizers
self.grad_scaler = GradScaler(enabled=self.mixed_precision)
# logging/viewer variables
self.viewer_state, banner_messages = viewer_utils.setup_viewer(config.viewer)
writer.setup_event_writer(config.logging)
writer.setup_local_writer(
config.logging, max_iter=config.trainer.max_num_iterations, banner_messages=banner_messages
)
profiler.setup_profiler(config.logging)
writer.put_config(name="config", config_dict=dataclasses.asdict(config), step=0)
self._check_viewer_warnings()
def setup(self, test_mode=False):
"""Setup the Trainer by calling other setup functions.
@@ -119,13 +119,11 @@ class Trainer:
"""Train the model."""
assert self.pipeline.datamanager.train_input_dataset is not None, "Missing DatsetInputs"
self.visualizer_state.init_scene(
dataset=self.pipeline.datamanager.train_input_dataset, start_train=self.config.viewer.start_train
)
self._init_viewer_scene()
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
num_iterations = self.config.trainer.max_num_iterations
for step in range(self.start_step, self.start_step + num_iterations):
# if the visualizer used, the rendering of the visualizer will be included in the iteration train time
# if the viewer used, the rendering of the viewer will be included in the iteration train time
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
# training callbacks before the training iteration
@@ -141,17 +139,9 @@ class Trainer:
for callback in self.callbacks:
callback.run_callback_at_location(step, location=TrainingCallbackLocation.AFTER_TRAIN_ITERATION)
# NOTE(ethan): we should remove this when the viewer is disabled
with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as vis_t:
num_rays_per_batch = self.config.pipeline.datamanager.train_num_rays_per_batch
try:
self.visualizer_state.update_scene(step, self.pipeline.model, num_rays_per_batch)
except RuntimeError:
time.sleep(0.03) # sleep to allow buffer to reset
assert self.visualizer_state.vis is not None
self.visualizer_state.vis["renderingState/log_errors"].write(
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
)
vis_t = self._update_viewer_state(step)
self._update_viewer_rays_per_sec(train_t, vis_t, step)
# a batch of train rays
if step_check(step, self.config.logging.steps_per_log, run_at_zero=True):
@@ -183,12 +173,55 @@ class Trainer:
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
self._save_checkpoint(self.config.trainer.model_dir, step)
self._update_rays_per_sec(train_t, vis_t, step)
self._update_viewer_rays_per_sec(train_t, vis_t, step)
self._write_out_storage(step)
self._write_out_storage(num_iterations)
def _update_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int):
def _check_viewer_warnings(self) -> None:
"""Helper to print out any warnings regarding the way the viewer/loggers are enabled"""
if self.config.viewer.enable:
if self.config.logging.event_writer_config:
string = (
"[WARNING]: Tensorboard or Wandb enabled with Viewer will slow down Viewer. "
"Please set `--logging.event_writer none` for faster rendering"
)
CONSOLE.print(f"[bold red]{string}")
string = "[WARNING] Disabling eval iterations since viewer is enabled."
CONSOLE.print(f"[bold red]{string}")
@check_viewer_enabled
def _init_viewer_scene(self) -> None:
"""Initializes viewer scene with given train dataset"""
assert self.viewer_state and self.pipeline.datamanager.train_input_dataset
self.viewer_state.init_scene(
dataset=self.pipeline.datamanager.train_input_dataset,
start_train=self.config.viewer.start_train,
)
@check_viewer_enabled
def _update_viewer_state(self, step: int) -> TimeWriter:
"""Updates the viewer state by rendering out scene with current pipeline
Returns the time taken to render scene.
Args:
step: current train step
"""
assert self.viewer_state is not None
with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as vis_t:
num_rays_per_batch = self.config.pipeline.datamanager.train_num_rays_per_batch
try:
self.viewer_state.update_scene(step, self.pipeline.model, num_rays_per_batch)
except RuntimeError:
time.sleep(0.03) # sleep to allow buffer to reset
assert self.viewer_state.vis is not None
self.viewer_state.vis["renderingState/log_errors"].write(
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
)
return vis_t
@check_viewer_enabled
def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int):
"""Performs update on rays/sec calclation for training
Args: