Remove clutter for logging in terminal (#40)

* added skeleton for terminal logger

* done skeleton terminal writer logic... todo: actual calls to the write classes

* moved lambda to top function

* added logic to support timing

* prettier printing

* prettier printing

* lint

* separated out timing and other stats variables. printing in dict format

* keeping logging history and wiping rest

* updated logic to calculate total times instead of average times
This commit is contained in:
evonneng authored and GitHub committed 2022-04-26 12:12:57 -07:00
1 parent 5d4ffa210a
commit 23b70634cb
3 files changed
+160 -51

No files matched your search

+15 -50
View File
@@ -1,7 +1,6 @@
"""
Code to train model.
"""
import datetime
import logging
import os
from time import time
@@ -20,7 +19,8 @@ from mattport.nerf.dataset.utils import get_dataset_inputs
from mattport.nerf.metrics import get_psnr
from mattport.nerf.optimizers import Optimizers
from mattport.utils.decorators import check_main_thread
from mattport.utils.writer import LocalWriter, TensorboardWriter
from mattport.utils.io import Stats, StatsTracker
from mattport.utils.writer import TensorboardWriter
logging.getLogger("PIL").setLevel(logging.WARNING)
@@ -44,11 +44,10 @@ class Trainer:
self.start_step = 0
# logging variables
self.is_main_thread = local_rank % world_size == 0
self.local_writer = LocalWriter(local_rank, world_size, save_dir=os.path.join(os.getcwd(), "writer"))
self.tensorboard_writer = TensorboardWriter(
local_rank, world_size, save_dir=os.path.join(os.getcwd(), "writer")
)
self.time_dict = {}
self.stats = StatsTracker(config, self.is_main_thread)
def setup_dataset(self):
"""_summary_"""
@@ -133,42 +132,6 @@ class Trainer:
ckpt_path,
)
def _update_time(self, name, start_time, end_time, num_iter=1, batch_size=None):
"""update the time dictionary with running averages/cumulative durations
Args:
name (_type_): name of function time we are logging
start_time (_type_): start time for the call
end_time (_type_): end time when the call finished executing
num_iter (int, optional): number of total iteration steps. Defaults to 1.
batch_size (_type_, optional): total number of rays in a batch;
if None, reports duration instead of batch per second. Defaults to None.
"""
val = end_time - start_time
if batch_size:
val = batch_size / val
self.time_dict[name] = (self.time_dict.get(name, 0) * num_iter + val) / (num_iter + 1)
if name == "single iter (time)":
remain_iter = self.config.max_num_iterations - num_iter
self.time_dict["ETA (time)"] = remain_iter * self.time_dict[name]
def _print_stats(self, step):
"""helper to print out the timing dictionary."""
mssg = ""
if step == 0:
for k in self.time_dict:
mssg += f"{k:<20} "
print(mssg)
else:
for k, v in self.time_dict.items():
if "(time)" in k:
v = str(datetime.timedelta(seconds=v))
else:
v = f"{v:0.4f}"
mssg += f"{v:<20} "
print(mssg, end="\r")
@classmethod
def get_aggregated_loss(cls, losses: Dict[str, torch.tensor]):
"""Returns the aggregated losses and the scalar for calling .backwards() on.
@@ -187,29 +150,31 @@ class Trainer:
train_start = time()
num_iterations = self.config.max_num_iterations
iter_dataset = iter(self.train_dataloader)
for step in range(self.start_step, self.start_step + num_iterations):
for i, step in enumerate(range(self.start_step, self.start_step + num_iterations)):
data_start = time()
batch = next(iter_dataset)
self._update_time("load data (time)", data_start, time(), num_iter=step)
self.stats.update_time(Stats.ITER_LOAD_TIME, data_start, time(), step=step)
iter_start = time()
loss_dict = self.train_iteration(batch, step)
self._update_time(
"avg. rays/s (1/s)", iter_start, time(), num_iter=step, batch_size=batch["indices"].shape[0]
self.stats.update_time(
Stats.RAYS_PER_SEC, iter_start, time(), step=step, batch_size=batch["indices"].shape[0]
)
self._update_time("single iter (time)", iter_start, time(), num_iter=step)
self.stats.update_time(Stats.ITER_TRAIN_TIME, iter_start, time(), step=step)
if step != 0 and step % self.config.steps_per_log == 0:
self.tensorboard_writer.write_scalar_dict(loss_dict, step, group="Loss", prefix="train-")
logging.info("{%d}/{%d} with losses {%s}", step, num_iterations, loss_dict)
# TODO: add the learning rates to tensorboard/logging
if step != 0 and self.config.steps_per_save and step % self.config.steps_per_save == 0:
self.save_checkpoint(self.config.model_dir, step)
logging.info("Saved ckpt at {%d}", step)
if step != 0 and step % self.config.steps_per_test == 0:
self.test_image(image_idx=0, step=step)
self.test_image(image_idx=10, step=step)
self.test_image(image_idx=20, step=step)
self._print_stats(step)
self._update_time("total train", train_start, time())
self.stats.print_stats(i / num_iterations)
self.stats.update_time(Stats.TOTAL_TRAIN_TIME, train_start, time(), step=-1)
self.stats.dump_stats()
def train_iteration(self, batch: dict, step: int):
"""Run one iteration with a batch of inputs."""
@@ -227,7 +192,6 @@ class Trainer:
def test_image(self, image_idx, step):
"""Test a specific image."""
logging.info("Running test iteration.")
image = self.train_image_dataset[image_idx]["image"] # ground truth
image_height, image_width, _ = image.shape
pixel_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij")
@@ -254,4 +218,5 @@ class Trainer:
)
fine_psnr = get_psnr(image, rgb_fine)
self.stats.update_value(Stats.CURR_TEST_PSNR, fine_psnr, step)
self.tensorboard_writer.write_scalar(f"image_idx_{image_idx}-fine_psnr", fine_psnr, step, group="test")
+14
View File
@@ -12,3 +12,17 @@ def check_main_thread(func: Callable) -> Callable:
func(self, *args, **kwargs)
return wrapper
def check_print_stats_step(func: Callable) -> Callable:
"""Declarator: check if it is time to print stats update"""
def wrapper(self, *args, **kwargs):
if (
self.step % self.config.steps_per_log == 0
or (self.config.steps_per_save and self.step % self.config.steps_per_save == 0)
or self.step % self.config.steps_per_test == 0
):
func(self, *args, **kwargs)
return wrapper
+131 -1
View File
@@ -1,9 +1,139 @@
"""
Input/output utils.
"""
import datetime
import enum
import json
import os
from typing import Optional
from omegaconf import DictConfig
from mattport.utils.decorators import check_main_thread, check_print_stats_step
class Stats(enum.Enum):
"""Possible Stats values for StatsTracker
Args:
enum (_type_): _description_
"""
ITER_LOAD_TIME = "Data Load (time)"
ITER_TRAIN_TIME = "Train Iter (time)"
TOTAL_TRAIN_TIME = "Train Total (time)"
RAYS_PER_SEC = "Rays Per Sec (1/s)"
CURR_TEST_PSNR = "Test PSNR"
ETA = "ETA (time)"
class StatsTracker:
"""Stats Tracker class"""
def __init__(self, config: DictConfig, is_main_thread: bool, max_history: Optional[int] = 20):
self.config = config
self.is_main_thread = is_main_thread
self.max_history = max_history
self.step = 0
self.stats_dict = {}
self.past_stats = []
self.new_key = False
def update_value(self, name: str, value: float, step: int):
"""update stats dictionary with key value pair
Args:
name (str): name of statistic we are logging
value (float): value to update.
step (int): number of total iteration steps.
"""
self.step = step
self.new_key = not name in self.stats_dict or self.new_key
self.stats_dict[name] = value
def update_time(self, name: str, start_time: float, end_time: float, step: int, batch_size: int = None):
"""update the stats dictionary with running averages/cumulative durations
Args:
name (str): name of statistic we are logging
start_time (float): start time for the call in seconds
end_time (float): end time when the call finished executing in seconds
step (int): number of total iteration steps.
batch_size (int, optional): total number of rays in a batch;
if None, reports duration instead of batch per second. Defaults to None.
"""
self.step = step
self.new_key = not name in self.stats_dict or self.new_key
val = end_time - start_time
if batch_size:
# calculate the batch per second stat
val = batch_size / val
if step == -1:
# logging total time instead of average
self.stats_dict[name] = val
else:
# calculate updated average
self.stats_dict[name] = (self.stats_dict.get(name, 0) * step + val) / (step + 1)
if name == Stats.ITER_TRAIN_TIME:
# update ETA if logging iteration train time
remain_iter = self.config.max_num_iterations - step
self.stats_dict[Stats.ETA] = remain_iter * self.stats_dict[name]
@check_print_stats_step
@check_main_thread
def print_stats(self, fraction_done: float):
"""helper to print out the stats dictionary.
Args:
fraction_done (float): fraction of steps executed in training iterations
"""
# print a new header line if there is a new key added
if self.step == 0 or self.new_key:
mssg = f"{'Step (% Done)':<20}"
for k in self.stats_dict:
mssg += f"{k.value:<20} "
if self.step > 0:
cursorup(len(self.past_stats) + 2)
print(mssg)
print("-" * len(mssg))
if self.step > 0:
for mssg in self.past_stats:
print(mssg)
# generate a new stats reporting message
if self.step > 0:
curr_mssg = f"{self.step} ({fraction_done*100:.02f}%)"
curr_mssg = f"{curr_mssg:<20}"
for k, v in self.stats_dict.items():
if "(time)" in k.value:
v = str(datetime.timedelta(seconds=v))
else:
v = f"{v:0.4f}"
curr_mssg += f"{v:<20} "
# update the history buffer
if len(self.past_stats) >= self.max_history:
self.past_stats.pop(0)
cursorup(len(self.past_stats) + 1)
for mssg in self.past_stats:
print(mssg)
print(curr_mssg)
self.past_stats.append(curr_mssg)
@check_main_thread
def dump_stats(self):
"""Dump stats locally to a json file"""
raise NotImplementedError
def cursorup(x: int):
"""utility tool to move the cursor up on the terminal
Args:
x (int): amount of lines to move cursor upward
"""
print(f"\r\033[{x}A", end="")
def load_from_json(filename: str):