mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 21:33:56 +08:00
[wip] logging separated s.t. only one is enabled (#458)
* updated viewer/writer configs * allow viewer + tb/wandb but with errors; moved localwriter to its own thing * nits and all * update test
This commit is contained in:
1 parent
a79ef3fb11
commit
4e7ef375a2
11 files changed
+200
-166
No files matched your search
@@ -49,12 +49,12 @@ def check_profiler_enabled(func: Callable) -> Callable:
|
||||
return wrapper
|
||||
|
||||
|
||||
def check_visualizer_enabled(func: Callable) -> Callable:
|
||||
"""Decorator: check if visualizer is enabled"""
|
||||
def check_viewer_enabled(func: Callable) -> Callable:
|
||||
"""Decorator: check if viewer is enabled"""
|
||||
|
||||
def wrapper(self, *args, **kwargs):
|
||||
ret = None
|
||||
if self.vis:
|
||||
if self.config.viewer.enable:
|
||||
ret = func(self, *args, **kwargs)
|
||||
return ret
|
||||
|
||||
|
||||
+42
-32
@@ -24,8 +24,6 @@ from pathlib import Path
|
||||
from time import time
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import imageio
|
||||
import numpy as np
|
||||
import torch
|
||||
import wandb
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
@@ -47,7 +45,7 @@ class EventName(enum.Enum):
|
||||
|
||||
ITER_TRAIN_TIME = "Train Iter (time)"
|
||||
TOTAL_TRAIN_TIME = "Train Total (time)"
|
||||
ITER_VIS_TIME = "Visualizer Rendering (time)"
|
||||
ITER_VIS_TIME = "Viewer Rendering (time)"
|
||||
ETA = "ETA (time)"
|
||||
TRAIN_RAYS_PER_SEC = "Train Rays / Sec"
|
||||
TEST_RAYS_PER_SEC = "Test Rays / Sec"
|
||||
@@ -157,17 +155,18 @@ def put_time(name: str, duration: float, step: int, avg_over_steps: bool = True,
|
||||
def write_out_storage():
|
||||
"""Function that writes all the events in storage to all the writer locations"""
|
||||
for writer in EVENT_WRITERS:
|
||||
if isinstance(writer, LocalWriter) and len(EVENT_STORAGE) > 0:
|
||||
writer.write_stats_log(EVENT_STORAGE[0]["step"])
|
||||
continue
|
||||
for event in EVENT_STORAGE:
|
||||
write_func = getattr(writer, event["write_type"].value)
|
||||
write_func(event["name"], event["event"], event["step"])
|
||||
# NOTE(ethan): why is the below statement needed?
|
||||
if isinstance(writer, LocalWriter):
|
||||
break
|
||||
|
||||
EVENT_STORAGE.clear()
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def setup_event_writers(config: cfg.LoggingConfig, max_iter: int, banner_messages: Optional[List[str]] = None) -> None:
|
||||
def setup_local_writer(config: cfg.LoggingConfig, max_iter: int, banner_messages: Optional[List[str]] = None) -> None:
|
||||
"""Initialization of all event writers specified in config
|
||||
|
||||
Args:
|
||||
@@ -175,14 +174,11 @@ def setup_event_writers(config: cfg.LoggingConfig, max_iter: int, banner_message
|
||||
max_iter: maximum number of train iterations
|
||||
banner_messages: list of messages to always display at bottom of screen
|
||||
"""
|
||||
for writer_type_config in config.writer:
|
||||
if writer_type_config.enable:
|
||||
if isinstance(writer_type_config, cfg.LocalWriterConfig):
|
||||
curr_writer = writer_type_config.setup(banner_messages=banner_messages)
|
||||
else:
|
||||
curr_writer = writer_type_config.setup()
|
||||
EVENT_WRITERS.append(curr_writer)
|
||||
logging.info("logging info to: %s", writer_type_config.log_dir)
|
||||
if config.local_writer.enable:
|
||||
curr_writer = config.local_writer.setup(banner_messages=banner_messages)
|
||||
EVENT_WRITERS.append(curr_writer)
|
||||
else:
|
||||
logging.info("disabled local writer")
|
||||
|
||||
## configure all the global buffer basic information
|
||||
GLOBAL_BUFFER["max_iter"] = max_iter
|
||||
@@ -191,6 +187,22 @@ def setup_event_writers(config: cfg.LoggingConfig, max_iter: int, banner_message
|
||||
GLOBAL_BUFFER["events"] = {}
|
||||
|
||||
|
||||
def setup_event_writer(config: cfg.LoggingConfig) -> None:
|
||||
"""Initialization of all event writers specified in config
|
||||
|
||||
Args:
|
||||
config: configuration to instantiate loggers
|
||||
max_iter: maximum number of train iterations
|
||||
banner_messages: list of messages to always display at bottom of screen
|
||||
"""
|
||||
if config.event_writer_config:
|
||||
curr_writer = config.event_writer_config.setup()
|
||||
EVENT_WRITERS.append(curr_writer)
|
||||
logging.info("logging info to: %s", config.event_writer_config.log_dir)
|
||||
else:
|
||||
logging.info("disabled tensorboard/wandb event writers")
|
||||
|
||||
|
||||
class Writer:
|
||||
"""Writer class"""
|
||||
|
||||
@@ -266,7 +278,7 @@ class WandbWriter(Writer):
|
||||
|
||||
def __init__(self, config: cfg.WandbWriterConfig):
|
||||
super().__init__(config.log_dir)
|
||||
wandb.init(project="nerfactory-project", dir=config.log_dir, reinit=True)
|
||||
wandb.init(project="nerfactory-project", dir=str(config.log_dir), reinit=True)
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
image = torch.permute(image, (2, 0, 1))
|
||||
@@ -340,7 +352,7 @@ def _format_time(seconds):
|
||||
|
||||
|
||||
@decorate_all([check_main_thread])
|
||||
class LocalWriter(Writer):
|
||||
class LocalWriter:
|
||||
"""Local Writer Class
|
||||
TODO: migrate to prettyprint
|
||||
|
||||
@@ -350,9 +362,8 @@ class LocalWriter(Writer):
|
||||
"""
|
||||
|
||||
def __init__(self, config: cfg.LocalWriterConfig, banner_messages: Optional[List[str]] = None):
|
||||
super().__init__(config.log_dir)
|
||||
self.config = config
|
||||
self.stats_to_track = [name.value for name in config.stats_to_track]
|
||||
self.max_log_size = config.max_log_size
|
||||
self.keys = set()
|
||||
self.past_mssgs = ["", ""]
|
||||
self.banner_len = 0 if banner_messages is None else len(banner_messages) + 1
|
||||
@@ -361,19 +372,18 @@ class LocalWriter(Writer):
|
||||
self.past_mssgs.extend(banner_messages)
|
||||
self.has_printed = False
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
if name in self.stats_to_track and self.log_dir:
|
||||
image = to8b(image)
|
||||
image_path = self.log_dir / f"{name}.jpg"
|
||||
imageio.imwrite(image_path, np.uint8(image.cpu().numpy() * 255.0))
|
||||
def write_stats_log(self, step: int) -> None:
|
||||
"""Function to write out scalars to terminal
|
||||
|
||||
def write_scalar(self, name: str, scalar: float, step: int) -> None:
|
||||
Args:
|
||||
step: current train step
|
||||
"""
|
||||
if step > 0:
|
||||
if not self.has_printed and self.max_log_size:
|
||||
if not self.has_printed and self.config.max_log_size:
|
||||
logging.info(
|
||||
"\x1b[33;20mPrinting max of %d lines. Set flag `--logging.writer.2.max-log-size=0` "
|
||||
"\x1b[33;20mPrinting max of %d lines. Set flag `--logging.local_writer.max-log-size=0` "
|
||||
"to disable line wrapping.\x1b[0m",
|
||||
self.max_log_size,
|
||||
self.config.max_log_size,
|
||||
)
|
||||
latest_map, new_key = self._consolidate_events()
|
||||
self._update_header(latest_map, new_key)
|
||||
@@ -405,8 +415,8 @@ class LocalWriter(Writer):
|
||||
latest_map: the most recent dictionary of stats that have been recorded
|
||||
new_key: indicator whether or not there is a new key added to logger
|
||||
"""
|
||||
full_log_cond = not self.max_log_size and GLOBAL_BUFFER["step"] <= GLOBAL_BUFFER["steps_per_log"]
|
||||
capped_log_cond = self.max_log_size and (len(self.past_mssgs) - self.banner_len <= 2 or new_key)
|
||||
full_log_cond = not self.config.max_log_size and GLOBAL_BUFFER["step"] <= GLOBAL_BUFFER["steps_per_log"]
|
||||
capped_log_cond = self.config.max_log_size and (len(self.past_mssgs) - self.banner_len <= 2 or new_key)
|
||||
if full_log_cond or capped_log_cond:
|
||||
mssg = f"{'Step (% Done)':<20}"
|
||||
for name, _ in latest_map.items():
|
||||
@@ -441,13 +451,13 @@ class LocalWriter(Writer):
|
||||
curr_mssg += f"{v:<20} "
|
||||
|
||||
# update the history buffer
|
||||
if self.max_log_size:
|
||||
if self.config.max_log_size:
|
||||
if not self.has_printed:
|
||||
cursor_idx = len(self.past_mssgs) - self.banner_len
|
||||
self.has_printed = True
|
||||
else:
|
||||
cursor_idx = len(self.past_mssgs)
|
||||
if len(self.past_mssgs[2:]) - self.banner_len >= self.max_log_size:
|
||||
if len(self.past_mssgs[2:]) - self.banner_len >= self.config.max_log_size:
|
||||
self.past_mssgs.pop(2)
|
||||
self.past_mssgs.insert(len(self.past_mssgs) - self.banner_len, curr_mssg)
|
||||
_cursorup(cursor_idx)
|
||||
|
||||
Reference in new issue
Block a user