diff --git a/mattport/nerf/trainer.py b/mattport/nerf/trainer.py index be52fa69b..cbcb65384 100644 --- a/mattport/nerf/trainer.py +++ b/mattport/nerf/trainer.py @@ -1,7 +1,6 @@ """ Code to train model. """ -import datetime import logging import os from time import time @@ -20,7 +19,8 @@ from mattport.nerf.dataset.utils import get_dataset_inputs from mattport.nerf.metrics import get_psnr from mattport.nerf.optimizers import Optimizers from mattport.utils.decorators import check_main_thread -from mattport.utils.writer import LocalWriter, TensorboardWriter +from mattport.utils.io import Stats, StatsTracker +from mattport.utils.writer import TensorboardWriter logging.getLogger("PIL").setLevel(logging.WARNING) @@ -44,11 +44,10 @@ class Trainer: self.start_step = 0 # logging variables self.is_main_thread = local_rank % world_size == 0 - self.local_writer = LocalWriter(local_rank, world_size, save_dir=os.path.join(os.getcwd(), "writer")) self.tensorboard_writer = TensorboardWriter( local_rank, world_size, save_dir=os.path.join(os.getcwd(), "writer") ) - self.time_dict = {} + self.stats = StatsTracker(config, self.is_main_thread) def setup_dataset(self): """_summary_""" @@ -133,42 +132,6 @@ class Trainer: ckpt_path, ) - def _update_time(self, name, start_time, end_time, num_iter=1, batch_size=None): - """update the time dictionary with running averages/cumulative durations - - Args: - name (_type_): name of function time we are logging - start_time (_type_): start time for the call - end_time (_type_): end time when the call finished executing - num_iter (int, optional): number of total iteration steps. Defaults to 1. - batch_size (_type_, optional): total number of rays in a batch; - if None, reports duration instead of batch per second. Defaults to None. - """ - val = end_time - start_time - if batch_size: - val = batch_size / val - - self.time_dict[name] = (self.time_dict.get(name, 0) * num_iter + val) / (num_iter + 1) - if name == "single iter (time)": - remain_iter = self.config.max_num_iterations - num_iter - self.time_dict["ETA (time)"] = remain_iter * self.time_dict[name] - - def _print_stats(self, step): - """helper to print out the timing dictionary.""" - mssg = "" - if step == 0: - for k in self.time_dict: - mssg += f"{k:<20} " - print(mssg) - else: - for k, v in self.time_dict.items(): - if "(time)" in k: - v = str(datetime.timedelta(seconds=v)) - else: - v = f"{v:0.4f}" - mssg += f"{v:<20} " - print(mssg, end="\r") - @classmethod def get_aggregated_loss(cls, losses: Dict[str, torch.tensor]): """Returns the aggregated losses and the scalar for calling .backwards() on. @@ -187,29 +150,31 @@ class Trainer: train_start = time() num_iterations = self.config.max_num_iterations iter_dataset = iter(self.train_dataloader) - for step in range(self.start_step, self.start_step + num_iterations): + for i, step in enumerate(range(self.start_step, self.start_step + num_iterations)): data_start = time() batch = next(iter_dataset) - self._update_time("load data (time)", data_start, time(), num_iter=step) + self.stats.update_time(Stats.ITER_LOAD_TIME, data_start, time(), step=step) + iter_start = time() loss_dict = self.train_iteration(batch, step) - self._update_time( - "avg. rays/s (1/s)", iter_start, time(), num_iter=step, batch_size=batch["indices"].shape[0] + self.stats.update_time( + Stats.RAYS_PER_SEC, iter_start, time(), step=step, batch_size=batch["indices"].shape[0] ) - self._update_time("single iter (time)", iter_start, time(), num_iter=step) + self.stats.update_time(Stats.ITER_TRAIN_TIME, iter_start, time(), step=step) + if step != 0 and step % self.config.steps_per_log == 0: self.tensorboard_writer.write_scalar_dict(loss_dict, step, group="Loss", prefix="train-") - logging.info("{%d}/{%d} with losses {%s}", step, num_iterations, loss_dict) # TODO: add the learning rates to tensorboard/logging if step != 0 and self.config.steps_per_save and step % self.config.steps_per_save == 0: self.save_checkpoint(self.config.model_dir, step) - logging.info("Saved ckpt at {%d}", step) if step != 0 and step % self.config.steps_per_test == 0: self.test_image(image_idx=0, step=step) self.test_image(image_idx=10, step=step) self.test_image(image_idx=20, step=step) - self._print_stats(step) - self._update_time("total train", train_start, time()) + self.stats.print_stats(i / num_iterations) + + self.stats.update_time(Stats.TOTAL_TRAIN_TIME, train_start, time(), step=-1) + self.stats.dump_stats() def train_iteration(self, batch: dict, step: int): """Run one iteration with a batch of inputs.""" @@ -227,7 +192,6 @@ class Trainer: def test_image(self, image_idx, step): """Test a specific image.""" - logging.info("Running test iteration.") image = self.train_image_dataset[image_idx]["image"] # ground truth image_height, image_width, _ = image.shape pixel_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij") @@ -254,4 +218,5 @@ class Trainer: ) fine_psnr = get_psnr(image, rgb_fine) + self.stats.update_value(Stats.CURR_TEST_PSNR, fine_psnr, step) self.tensorboard_writer.write_scalar(f"image_idx_{image_idx}-fine_psnr", fine_psnr, step, group="test") diff --git a/mattport/utils/decorators.py b/mattport/utils/decorators.py index 7a016996a..7f4baecb5 100644 --- a/mattport/utils/decorators.py +++ b/mattport/utils/decorators.py @@ -12,3 +12,17 @@ def check_main_thread(func: Callable) -> Callable: func(self, *args, **kwargs) return wrapper + + +def check_print_stats_step(func: Callable) -> Callable: + """Declarator: check if it is time to print stats update""" + + def wrapper(self, *args, **kwargs): + if ( + self.step % self.config.steps_per_log == 0 + or (self.config.steps_per_save and self.step % self.config.steps_per_save == 0) + or self.step % self.config.steps_per_test == 0 + ): + func(self, *args, **kwargs) + + return wrapper diff --git a/mattport/utils/io.py b/mattport/utils/io.py index 6ca89a126..4a63873b9 100644 --- a/mattport/utils/io.py +++ b/mattport/utils/io.py @@ -1,9 +1,139 @@ """ Input/output utils. """ - +import datetime +import enum import json import os +from typing import Optional + +from omegaconf import DictConfig + +from mattport.utils.decorators import check_main_thread, check_print_stats_step + + +class Stats(enum.Enum): + """Possible Stats values for StatsTracker + + Args: + enum (_type_): _description_ + """ + + ITER_LOAD_TIME = "Data Load (time)" + ITER_TRAIN_TIME = "Train Iter (time)" + TOTAL_TRAIN_TIME = "Train Total (time)" + RAYS_PER_SEC = "Rays Per Sec (1/s)" + CURR_TEST_PSNR = "Test PSNR" + ETA = "ETA (time)" + + +class StatsTracker: + """Stats Tracker class""" + + def __init__(self, config: DictConfig, is_main_thread: bool, max_history: Optional[int] = 20): + self.config = config + self.is_main_thread = is_main_thread + self.max_history = max_history + self.step = 0 + self.stats_dict = {} + self.past_stats = [] + self.new_key = False + + def update_value(self, name: str, value: float, step: int): + """update stats dictionary with key value pair + + Args: + name (str): name of statistic we are logging + value (float): value to update. + step (int): number of total iteration steps. + """ + self.step = step + self.new_key = not name in self.stats_dict or self.new_key + self.stats_dict[name] = value + + def update_time(self, name: str, start_time: float, end_time: float, step: int, batch_size: int = None): + """update the stats dictionary with running averages/cumulative durations + + Args: + name (str): name of statistic we are logging + start_time (float): start time for the call in seconds + end_time (float): end time when the call finished executing in seconds + step (int): number of total iteration steps. + batch_size (int, optional): total number of rays in a batch; + if None, reports duration instead of batch per second. Defaults to None. + """ + self.step = step + self.new_key = not name in self.stats_dict or self.new_key + val = end_time - start_time + if batch_size: + # calculate the batch per second stat + val = batch_size / val + + if step == -1: + # logging total time instead of average + self.stats_dict[name] = val + else: + # calculate updated average + self.stats_dict[name] = (self.stats_dict.get(name, 0) * step + val) / (step + 1) + + if name == Stats.ITER_TRAIN_TIME: + # update ETA if logging iteration train time + remain_iter = self.config.max_num_iterations - step + self.stats_dict[Stats.ETA] = remain_iter * self.stats_dict[name] + + @check_print_stats_step + @check_main_thread + def print_stats(self, fraction_done: float): + """helper to print out the stats dictionary. + + Args: + fraction_done (float): fraction of steps executed in training iterations + """ + # print a new header line if there is a new key added + if self.step == 0 or self.new_key: + mssg = f"{'Step (% Done)':<20}" + for k in self.stats_dict: + mssg += f"{k.value:<20} " + if self.step > 0: + cursorup(len(self.past_stats) + 2) + print(mssg) + print("-" * len(mssg)) + if self.step > 0: + for mssg in self.past_stats: + print(mssg) + + # generate a new stats reporting message + if self.step > 0: + curr_mssg = f"{self.step} ({fraction_done*100:.02f}%)" + curr_mssg = f"{curr_mssg:<20}" + for k, v in self.stats_dict.items(): + if "(time)" in k.value: + v = str(datetime.timedelta(seconds=v)) + else: + v = f"{v:0.4f}" + curr_mssg += f"{v:<20} " + # update the history buffer + if len(self.past_stats) >= self.max_history: + self.past_stats.pop(0) + cursorup(len(self.past_stats) + 1) + for mssg in self.past_stats: + print(mssg) + print(curr_mssg) + self.past_stats.append(curr_mssg) + + @check_main_thread + def dump_stats(self): + """Dump stats locally to a json file""" + raise NotImplementedError + + +def cursorup(x: int): + """utility tool to move the cursor up on the terminal + + Args: + x (int): amount of lines to move cursor upward + """ + print(f"\r\033[{x}A", end="") def load_from_json(filename: str):