mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 05:14:15 +08:00
Remove clutter for logging in terminal (#40)
* added skeleton for terminal logger * done skeleton terminal writer logic... todo: actual calls to the write classes * moved lambda to top function * added logic to support timing * prettier printing * prettier printing * lint * separated out timing and other stats variables. printing in dict format * keeping logging history and wiping rest * updated logic to calculate total times instead of average times
This commit is contained in:
1 parent
5d4ffa210a
commit
23b70634cb
3 files changed
+160
-51
No files matched your search
+15
-50
@@ -1,7 +1,6 @@
|
||||
"""
|
||||
Code to train model.
|
||||
"""
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
from time import time
|
||||
@@ -20,7 +19,8 @@ from mattport.nerf.dataset.utils import get_dataset_inputs
|
||||
from mattport.nerf.metrics import get_psnr
|
||||
from mattport.nerf.optimizers import Optimizers
|
||||
from mattport.utils.decorators import check_main_thread
|
||||
from mattport.utils.writer import LocalWriter, TensorboardWriter
|
||||
from mattport.utils.io import Stats, StatsTracker
|
||||
from mattport.utils.writer import TensorboardWriter
|
||||
|
||||
logging.getLogger("PIL").setLevel(logging.WARNING)
|
||||
|
||||
@@ -44,11 +44,10 @@ class Trainer:
|
||||
self.start_step = 0
|
||||
# logging variables
|
||||
self.is_main_thread = local_rank % world_size == 0
|
||||
self.local_writer = LocalWriter(local_rank, world_size, save_dir=os.path.join(os.getcwd(), "writer"))
|
||||
self.tensorboard_writer = TensorboardWriter(
|
||||
local_rank, world_size, save_dir=os.path.join(os.getcwd(), "writer")
|
||||
)
|
||||
self.time_dict = {}
|
||||
self.stats = StatsTracker(config, self.is_main_thread)
|
||||
|
||||
def setup_dataset(self):
|
||||
"""_summary_"""
|
||||
@@ -133,42 +132,6 @@ class Trainer:
|
||||
ckpt_path,
|
||||
)
|
||||
|
||||
def _update_time(self, name, start_time, end_time, num_iter=1, batch_size=None):
|
||||
"""update the time dictionary with running averages/cumulative durations
|
||||
|
||||
Args:
|
||||
name (_type_): name of function time we are logging
|
||||
start_time (_type_): start time for the call
|
||||
end_time (_type_): end time when the call finished executing
|
||||
num_iter (int, optional): number of total iteration steps. Defaults to 1.
|
||||
batch_size (_type_, optional): total number of rays in a batch;
|
||||
if None, reports duration instead of batch per second. Defaults to None.
|
||||
"""
|
||||
val = end_time - start_time
|
||||
if batch_size:
|
||||
val = batch_size / val
|
||||
|
||||
self.time_dict[name] = (self.time_dict.get(name, 0) * num_iter + val) / (num_iter + 1)
|
||||
if name == "single iter (time)":
|
||||
remain_iter = self.config.max_num_iterations - num_iter
|
||||
self.time_dict["ETA (time)"] = remain_iter * self.time_dict[name]
|
||||
|
||||
def _print_stats(self, step):
|
||||
"""helper to print out the timing dictionary."""
|
||||
mssg = ""
|
||||
if step == 0:
|
||||
for k in self.time_dict:
|
||||
mssg += f"{k:<20} "
|
||||
print(mssg)
|
||||
else:
|
||||
for k, v in self.time_dict.items():
|
||||
if "(time)" in k:
|
||||
v = str(datetime.timedelta(seconds=v))
|
||||
else:
|
||||
v = f"{v:0.4f}"
|
||||
mssg += f"{v:<20} "
|
||||
print(mssg, end="\r")
|
||||
|
||||
@classmethod
|
||||
def get_aggregated_loss(cls, losses: Dict[str, torch.tensor]):
|
||||
"""Returns the aggregated losses and the scalar for calling .backwards() on.
|
||||
@@ -187,29 +150,31 @@ class Trainer:
|
||||
train_start = time()
|
||||
num_iterations = self.config.max_num_iterations
|
||||
iter_dataset = iter(self.train_dataloader)
|
||||
for step in range(self.start_step, self.start_step + num_iterations):
|
||||
for i, step in enumerate(range(self.start_step, self.start_step + num_iterations)):
|
||||
data_start = time()
|
||||
batch = next(iter_dataset)
|
||||
self._update_time("load data (time)", data_start, time(), num_iter=step)
|
||||
self.stats.update_time(Stats.ITER_LOAD_TIME, data_start, time(), step=step)
|
||||
|
||||
iter_start = time()
|
||||
loss_dict = self.train_iteration(batch, step)
|
||||
self._update_time(
|
||||
"avg. rays/s (1/s)", iter_start, time(), num_iter=step, batch_size=batch["indices"].shape[0]
|
||||
self.stats.update_time(
|
||||
Stats.RAYS_PER_SEC, iter_start, time(), step=step, batch_size=batch["indices"].shape[0]
|
||||
)
|
||||
self._update_time("single iter (time)", iter_start, time(), num_iter=step)
|
||||
self.stats.update_time(Stats.ITER_TRAIN_TIME, iter_start, time(), step=step)
|
||||
|
||||
if step != 0 and step % self.config.steps_per_log == 0:
|
||||
self.tensorboard_writer.write_scalar_dict(loss_dict, step, group="Loss", prefix="train-")
|
||||
logging.info("{%d}/{%d} with losses {%s}", step, num_iterations, loss_dict)
|
||||
# TODO: add the learning rates to tensorboard/logging
|
||||
if step != 0 and self.config.steps_per_save and step % self.config.steps_per_save == 0:
|
||||
self.save_checkpoint(self.config.model_dir, step)
|
||||
logging.info("Saved ckpt at {%d}", step)
|
||||
if step != 0 and step % self.config.steps_per_test == 0:
|
||||
self.test_image(image_idx=0, step=step)
|
||||
self.test_image(image_idx=10, step=step)
|
||||
self.test_image(image_idx=20, step=step)
|
||||
self._print_stats(step)
|
||||
self._update_time("total train", train_start, time())
|
||||
self.stats.print_stats(i / num_iterations)
|
||||
|
||||
self.stats.update_time(Stats.TOTAL_TRAIN_TIME, train_start, time(), step=-1)
|
||||
self.stats.dump_stats()
|
||||
|
||||
def train_iteration(self, batch: dict, step: int):
|
||||
"""Run one iteration with a batch of inputs."""
|
||||
@@ -227,7 +192,6 @@ class Trainer:
|
||||
|
||||
def test_image(self, image_idx, step):
|
||||
"""Test a specific image."""
|
||||
logging.info("Running test iteration.")
|
||||
image = self.train_image_dataset[image_idx]["image"] # ground truth
|
||||
image_height, image_width, _ = image.shape
|
||||
pixel_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij")
|
||||
@@ -254,4 +218,5 @@ class Trainer:
|
||||
)
|
||||
|
||||
fine_psnr = get_psnr(image, rgb_fine)
|
||||
self.stats.update_value(Stats.CURR_TEST_PSNR, fine_psnr, step)
|
||||
self.tensorboard_writer.write_scalar(f"image_idx_{image_idx}-fine_psnr", fine_psnr, step, group="test")
|
||||
@@ -12,3 +12,17 @@ def check_main_thread(func: Callable) -> Callable:
|
||||
func(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def check_print_stats_step(func: Callable) -> Callable:
|
||||
"""Declarator: check if it is time to print stats update"""
|
||||
|
||||
def wrapper(self, *args, **kwargs):
|
||||
if (
|
||||
self.step % self.config.steps_per_log == 0
|
||||
or (self.config.steps_per_save and self.step % self.config.steps_per_save == 0)
|
||||
or self.step % self.config.steps_per_test == 0
|
||||
):
|
||||
func(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
+131
-1
@@ -1,9 +1,139 @@
|
||||
"""
|
||||
Input/output utils.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import enum
|
||||
import json
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from mattport.utils.decorators import check_main_thread, check_print_stats_step
|
||||
|
||||
|
||||
class Stats(enum.Enum):
|
||||
"""Possible Stats values for StatsTracker
|
||||
|
||||
Args:
|
||||
enum (_type_): _description_
|
||||
"""
|
||||
|
||||
ITER_LOAD_TIME = "Data Load (time)"
|
||||
ITER_TRAIN_TIME = "Train Iter (time)"
|
||||
TOTAL_TRAIN_TIME = "Train Total (time)"
|
||||
RAYS_PER_SEC = "Rays Per Sec (1/s)"
|
||||
CURR_TEST_PSNR = "Test PSNR"
|
||||
ETA = "ETA (time)"
|
||||
|
||||
|
||||
class StatsTracker:
|
||||
"""Stats Tracker class"""
|
||||
|
||||
def __init__(self, config: DictConfig, is_main_thread: bool, max_history: Optional[int] = 20):
|
||||
self.config = config
|
||||
self.is_main_thread = is_main_thread
|
||||
self.max_history = max_history
|
||||
self.step = 0
|
||||
self.stats_dict = {}
|
||||
self.past_stats = []
|
||||
self.new_key = False
|
||||
|
||||
def update_value(self, name: str, value: float, step: int):
|
||||
"""update stats dictionary with key value pair
|
||||
|
||||
Args:
|
||||
name (str): name of statistic we are logging
|
||||
value (float): value to update.
|
||||
step (int): number of total iteration steps.
|
||||
"""
|
||||
self.step = step
|
||||
self.new_key = not name in self.stats_dict or self.new_key
|
||||
self.stats_dict[name] = value
|
||||
|
||||
def update_time(self, name: str, start_time: float, end_time: float, step: int, batch_size: int = None):
|
||||
"""update the stats dictionary with running averages/cumulative durations
|
||||
|
||||
Args:
|
||||
name (str): name of statistic we are logging
|
||||
start_time (float): start time for the call in seconds
|
||||
end_time (float): end time when the call finished executing in seconds
|
||||
step (int): number of total iteration steps.
|
||||
batch_size (int, optional): total number of rays in a batch;
|
||||
if None, reports duration instead of batch per second. Defaults to None.
|
||||
"""
|
||||
self.step = step
|
||||
self.new_key = not name in self.stats_dict or self.new_key
|
||||
val = end_time - start_time
|
||||
if batch_size:
|
||||
# calculate the batch per second stat
|
||||
val = batch_size / val
|
||||
|
||||
if step == -1:
|
||||
# logging total time instead of average
|
||||
self.stats_dict[name] = val
|
||||
else:
|
||||
# calculate updated average
|
||||
self.stats_dict[name] = (self.stats_dict.get(name, 0) * step + val) / (step + 1)
|
||||
|
||||
if name == Stats.ITER_TRAIN_TIME:
|
||||
# update ETA if logging iteration train time
|
||||
remain_iter = self.config.max_num_iterations - step
|
||||
self.stats_dict[Stats.ETA] = remain_iter * self.stats_dict[name]
|
||||
|
||||
@check_print_stats_step
|
||||
@check_main_thread
|
||||
def print_stats(self, fraction_done: float):
|
||||
"""helper to print out the stats dictionary.
|
||||
|
||||
Args:
|
||||
fraction_done (float): fraction of steps executed in training iterations
|
||||
"""
|
||||
# print a new header line if there is a new key added
|
||||
if self.step == 0 or self.new_key:
|
||||
mssg = f"{'Step (% Done)':<20}"
|
||||
for k in self.stats_dict:
|
||||
mssg += f"{k.value:<20} "
|
||||
if self.step > 0:
|
||||
cursorup(len(self.past_stats) + 2)
|
||||
print(mssg)
|
||||
print("-" * len(mssg))
|
||||
if self.step > 0:
|
||||
for mssg in self.past_stats:
|
||||
print(mssg)
|
||||
|
||||
# generate a new stats reporting message
|
||||
if self.step > 0:
|
||||
curr_mssg = f"{self.step} ({fraction_done*100:.02f}%)"
|
||||
curr_mssg = f"{curr_mssg:<20}"
|
||||
for k, v in self.stats_dict.items():
|
||||
if "(time)" in k.value:
|
||||
v = str(datetime.timedelta(seconds=v))
|
||||
else:
|
||||
v = f"{v:0.4f}"
|
||||
curr_mssg += f"{v:<20} "
|
||||
# update the history buffer
|
||||
if len(self.past_stats) >= self.max_history:
|
||||
self.past_stats.pop(0)
|
||||
cursorup(len(self.past_stats) + 1)
|
||||
for mssg in self.past_stats:
|
||||
print(mssg)
|
||||
print(curr_mssg)
|
||||
self.past_stats.append(curr_mssg)
|
||||
|
||||
@check_main_thread
|
||||
def dump_stats(self):
|
||||
"""Dump stats locally to a json file"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def cursorup(x: int):
|
||||
"""utility tool to move the cursor up on the terminal
|
||||
|
||||
Args:
|
||||
x (int): amount of lines to move cursor upward
|
||||
"""
|
||||
print(f"\r\033[{x}A", end="")
|
||||
|
||||
|
||||
def load_from_json(filename: str):
|
||||
|
||||
Reference in new issue
Block a user