Adios mattport

This commit is contained in:
Matt Tancik committed 2022-05-30 09:24:12 -07:00
1 parent 9d2b34ab60
commit b26408eb7a
86 files changed
+176 -176

No files matched your search

View File
Whitespace-only changes.
+21
View File
@@ -0,0 +1,21 @@
"""
Callback functions for training iterations
"""
from radiance.nerf.graph.base import Graph
from radiance.nerf.occupancy_grid import OccupancyGrid
def update_occupancy(graph: Graph):
"""Update the occupancy grid depending on iteration count after every train step"""
if not hasattr(graph, "occupancy_pointers"):
occupancy_grids = []
for _, module in graph.__dict__["_modules"].items():
if isinstance(module, OccupancyGrid):
occupancy_grids.append(module)
graph.occupancy_pointers = occupancy_grids
for grid in graph.occupancy_pointers:
grid.iteration_count += 1
if grid.iteration_count % grid.update_every_num_iters == 0:
if grid.density_fn is not None:
grid.update_occupancy_grid(grid.density_fn)
+72
View File
@@ -0,0 +1,72 @@
"""functionality to handle multiprocessing syncing and communicating"""
import torch
import torch.distributed as dist
_LOCAL_PROCESS_GROUP = None
def get_world_size() -> int:
"""Get total number of available gpus"""
if not dist.is_available():
return 1
if not dist.is_initialized():
return 1
return dist.get_world_size()
def get_rank() -> int:
"""Get global rank of current thread"""
if not dist.is_available():
return 0
if not dist.is_initialized():
return 0
return dist.get_rank()
def get_local_rank() -> int:
"""The rank of the current process within the local (per-machine) process group."""
if not dist.is_available():
return 0
if not dist.is_initialized():
return 0
assert (
_LOCAL_PROCESS_GROUP is not None
), "Local process group is not created! Please use launch() to spawn processes!"
return dist.get_rank(group=_LOCAL_PROCESS_GROUP)
def get_local_size() -> int:
"""
The size of the per-machine process group,
i.e. the number of processes per machine.
"""
if not dist.is_available():
return 1
if not dist.is_initialized():
return 1
return dist.get_world_size(group=_LOCAL_PROCESS_GROUP)
def is_main_process() -> bool:
"""check to see if you are currently on the main process"""
return get_rank() == 0
def synchronize(world_size):
"""
Helper function to synchronize (barrier) among all processes when
using distributed training
"""
if not dist.is_available():
return
if not dist.is_initialized():
return
world_size = dist.get_world_size()
if world_size == 1:
return
if dist.get_backend() == dist.Backend.NCCL:
# This argument is needed to avoid warnings.
# It's valid only for NCCL backend.
dist.barrier(device_ids=[torch.cuda.current_device()])
else:
dist.barrier()
+43
View File
@@ -0,0 +1,43 @@
"""
Decorator definitions
"""
from typing import Callable, List
from radiance.utils import comms
def decorate_all(decorators: List[Callable]) -> Callable:
"""A decorator to decorate all member functions of a class"""
def decorate(cls):
for attr in cls.__dict__:
if callable(getattr(cls, attr)) and attr != "__init__":
for decorator in decorators:
setattr(cls, attr, decorator(getattr(cls, attr)))
return cls
return decorate
def check_profiler_enabled(func: Callable) -> Callable:
"""Decorator: check if profiler is enabled"""
def wrapper(self, *args, **kwargs):
ret = None
if self.config.enable_profiler:
ret = func(self, *args, **kwargs)
return ret
return wrapper
def check_main_thread(func: Callable) -> Callable:
"""Decorator: check if you are on main thread"""
def wrapper(*args, **kwargs):
ret = None
if comms.is_main_process():
ret = func(*args, **kwargs)
return ret
return wrapper
+99
View File
@@ -0,0 +1,99 @@
"""
Input/output utils.
"""
import json
import os
import logging
def load_from_json(filename: str):
"""_summary_
Args:
filename (str): _description_
Returns:
_type_: _description_
"""
assert filename.endswith(".json")
with open(filename, "r", encoding="UTF-8") as file:
return json.load(file)
def write_to_json(filename: str, content: dict):
"""_summary_
Args:
filename (str): _description_
content (dict): _description_
"""
assert filename.endswith(".json")
with open(filename, "w", encoding="UTF-8") as file:
json.dump(content, file)
def get_git_root(path, dirs=(".git",), default=None):
"""_summary_
"https://stackoverflow.com/questions/22081209/find-the-root-of-the-git-repository-where-the-file-lives
Args:
path (_type_): _description_
dirs (tuple, optional): _description_. Defaults to (".git",).
default (_type_, optional): _description_. Defaults to None.
Returns:
_type_: _description_
"""
prev, test = None, os.path.abspath(path)
while prev != test:
if any(os.path.isdir(os.path.join(test, directory)) for directory in dirs):
return test
prev, test = test, os.path.abspath(os.path.join(test, os.pardir))
return default
def get_project_root(path):
"""Return the project root directory from an environment variable.
# TODO: handle this better and report user error
"""
project_root = os.getenv("PROJECT_ROOT")
if project_root is None:
logging.info(
(
"Please set PROJECT_ROOT to the root directory of this repo. "
"Going to try calling get_git_root(path) instead."
)
)
project_root = get_git_root(path)
return project_root
def get_absolute_path(path, proj_root_func=get_project_root):
"""
Returns the full, absolute path.
Relative paths are assumed to start at the repo directory.
"""
if path == "":
return ""
absolute_path = path
if absolute_path[0] != "/":
absolute_path = os.path.join(proj_root_func(path), absolute_path)
return absolute_path
def make_dir(filename_or_folder):
"""Make the directory for either the filename or folder.
Note that filename_or_folder currently needs to end in / for it to be recognized as a folder.
"""
if filename_or_folder[-1] != "/" and filename_or_folder.find(".") < 0:
folder = os.path.dirname(filename_or_folder + "/")
else:
folder = os.path.dirname(filename_or_folder)
if not os.path.exists(folder):
try:
os.makedirs(folder)
except Exception as e: # pylint: disable=broad-except
print(f"Couldn't create folder: {folder}. Maybe due to a parallel process?")
print(e)
return filename_or_folder
+69
View File
@@ -0,0 +1,69 @@
""" Math Helper Functions """
import torch
from torchtyping import TensorType
def components_from_spherical_harmonics(levels: int, directions: TensorType[..., 3]) -> TensorType[..., "components"]:
"""
Args:
levels (int): Number of spherical harmonic levels to compute.
coeffs (TensorType[..., &quot;coeffs&quot;]): Spherical hamonic coefficients
Returns:
TensorType[..., "components"]: Value for each component.
"""
num_components = levels**2
components = torch.zeros((*directions.shape[:-1], num_components)).to(directions.device)
assert 1 <= levels <= 5, f"SH levels must be in [1,4], got {levels}"
assert directions.shape[-1] == 3, f"Direction input should have three dimensions. Got {directions.shape[-1]}"
x = directions[..., 0]
y = directions[..., 1]
z = directions[..., 2]
xx = x**2
yy = y**2
zz = z**2
# l0
components[..., 0] = 0.28209479177387814
# l1
if levels > 1:
components[..., 1] = 0.4886025119029199 * y
components[..., 2] = 0.4886025119029199 * z
components[..., 3] = 0.4886025119029199 * x
# l2
if levels > 2:
components[..., 4] = 1.0925484305920792 * x * y
components[..., 5] = 1.0925484305920792 * y * z
components[..., 6] = 0.9461746957575601 * zz
components[..., 7] = 1.0925484305920792 * x * z
components[..., 8] = 0.5462742152960396 * (xx - yy)
# l3
if levels > 3:
components[..., 9] = 0.5900435899266435 * y * (3 * xx - yy)
components[..., 10] = 2.890611442640554 * x * y * z
components[..., 11] = 0.4570457994644658 * y * (5 * zz - 1)
components[..., 12] = 0.3731763325901154 * z * (5 * zz - 3)
components[..., 13] = 0.4570457994644658 * x * (5 * zz - 1)
components[..., 14] = 1.445305721320277 * z * (xx - yy)
components[..., 15] = 0.5900435899266435 * x * (xx - 3 * yy)
# l4
if levels > 4:
components[..., 16] = 2.5033429417967046 * x * y * (xx - yy)
components[..., 17] = 1.7701307697799304 * y * z * (3 * xx - yy)
components[..., 18] = 0.9461746957575601 * x * y * (7 * zz - 1)
components[..., 19] = 0.6690465435572892 * y * (7 * zz - 3 * z)
components[..., 20] = 0.10578554691520431 * (35 * zz * zz - 30 * zz + 3)
components[..., 21] = 0.6690465435572892 * x * z * (7 * zz - 3)
components[..., 22] = 0.47308734787878004 * (xx - yy) * (7 * zz - 1)
components[..., 23] = 1.7701307697799304 * x * z * (xx - 3 * yy)
components[..., 24] = 0.4425326924449826 * (xx * (xx - 3 * yy) - yy * (3 * xx - yy))
return components
+53
View File
@@ -0,0 +1,53 @@
"""
Miscellaneous helper code.
"""
import torch
class DotDict(dict):
"""
dot.notation access to dictionary attributes
"""
def __getattr__(self, attr):
return self[attr]
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
def get_dict_to_torch(stuff, device="cpu"):
"""Set everything in the dict to the specified torch device."""
if isinstance(stuff, dict):
for k, v in stuff.items():
stuff[k] = get_dict_to_torch(v, device)
return stuff
if isinstance(stuff, torch.Tensor):
return stuff.to(device)
return stuff
def get_dict_to_cpu(stuff):
"""Set everything in the dict to CPU."""
if isinstance(stuff, dict):
for k, v in stuff.items():
stuff[k] = get_dict_to_cpu(v)
return stuff
if isinstance(stuff, torch.Tensor):
return stuff.detach().cpu()
return stuff
def is_not_none(var):
"""Return True if the variable var is None."""
return not isinstance(var, type(None))
def get_masked_dict(d, mask):
"""Return a masked dictionary.
TODO(ethan): add more asserts/checks so this doesn't have unpredictable behavior."""
masked_dict = {}
for key, value in d.items():
masked_dict[key] = value[mask]
return masked_dict
+70
View File
@@ -0,0 +1,70 @@
"""
Profiler base class and functionality
"""
import logging
import time
from typing import Callable
from omegaconf import DictConfig
from radiance.utils import comms
from radiance.utils.decorators import check_main_thread, check_profiler_enabled, decorate_all
PROFILER = []
def time_function(func: Callable) -> Callable:
"""Decorator: time a function call"""
def wrapper(*args, **kwargs):
start = time.time()
ret = func(*args, **kwargs)
if PROFILER:
class_str = func.__qualname__
PROFILER[0].update_time(class_str, start, time.time())
return ret
return wrapper
def flush_profiler(config: DictConfig):
"""Method that checks if profiler is enabled before flushing"""
if config.enable_profiler and PROFILER:
PROFILER[0].print_profile()
def setup_profiler(config: DictConfig):
"""Initialization of profilers"""
if comms.is_main_process():
PROFILER.append(Profiler(config))
@decorate_all([check_profiler_enabled, check_main_thread])
class Profiler:
"""Profiler class"""
def __init__(self, config: DictConfig):
self.config = config
self.profiler_dict = {}
def update_time(self, func_name: str, start_time: float, end_time: float):
"""update the profiler dictionary with running averages of durations
Args:
func_name (str): the function name that is being profiled
start_time (float): the start time when function is called
end_time (float): the end time when function terminated
"""
val = end_time - start_time
func_dict = self.profiler_dict.get(func_name, {"val": 0, "step": 0})
prev_val = func_dict["val"]
prev_step = func_dict["step"]
self.profiler_dict[func_name] = {"val": (prev_val * prev_step + val) / (prev_step + 1), "step": prev_step + 1}
def print_profile(self):
"""helper to print out the profiler stats"""
logging.info("Printing profiling stats, from longest to shortest duration in seconds")
sorted_keys = [k for k, _ in sorted(self.profiler_dict.items(), key=lambda item: item[1]["val"], reverse=True)]
for k in sorted_keys:
val = f"{self.profiler_dict[k]['val']:0.4f}"
print(f"{k:<20}: {val:<20}")
+61
View File
@@ -0,0 +1,61 @@
""" Helper functions for visualizing outputs """
from typing import Optional
import torch
from matplotlib import cm
from torchtyping import TensorType
def apply_colormap(image: TensorType[..., 1], cmap="viridis") -> TensorType[..., 3]:
"""Convert single channel to a color image.
Args:
image (TensorType[..., 1]): Single channel image.
cmap (str, optional): Colormap for image. Defaults to 'turbo'.
Returns:
TensorType[..., 3]: Colored image
"""
colormap = cm.get_cmap(cmap)
colormap = torch.tensor(colormap.colors).to(image.device)
image_long = (image * 255).long()
image_long_min = torch.min(image_long)
image_long_max = torch.max(image_long)
assert image_long_min >= 0, f"the min value is {image_long_min}"
assert image_long_max <= 255, f"the max value is {image_long_max}"
return colormap[image_long[..., 0]]
def apply_depth_colormap(
depth: TensorType[..., 1],
accumulation: Optional[TensorType[..., 1]] = None,
near_plane: Optional[float] = None,
far_plane: Optional[float] = None,
cmap="turbo",
) -> TensorType[..., 3]:
"""Converts a depth image to color for easier analysis.
Args:
depth (TensorType[..., 1]): Depth image.
accumulation (TensorType[..., 1], optional): Ray accumulation used for masking vis. Defaults to None.
near_plane (float, optional): Closest depth to consider. If None, use min image value. Defaults to None.
far_plane (float, optional): Furthest depth to consider. If None, use max image value. Defaults to None.
cmap (str, optional): Colormap to apply. Defaults to "turbo".
Returns:
TensorType[..., 3]: Colored depth image
"""
near_plane = near_plane or torch.min(depth)
far_plane = far_plane or torch.max(depth)
depth = (depth - near_plane) / (far_plane - near_plane + 1e-10)
depth = torch.clip(depth, 0, 1)
colored_image = apply_colormap(depth, cmap=cmap)
if accumulation is not None:
colored_image = colored_image * accumulation + (1 - accumulation)
return colored_image
+372
View File
@@ -0,0 +1,372 @@
"""
Generic Writer class
"""
import enum
import os
from abc import abstractmethod
from typing import Dict
import imageio
import numpy as np
import torch
from omegaconf import DictConfig, ListConfig
from torch.utils.tensorboard import SummaryWriter
from torchtyping import TensorType
import radiance.utils.writer
from radiance.utils.decorators import check_main_thread, decorate_all
to8b = lambda x: (255 * torch.clamp(x, min=0, max=1)).to(torch.uint8)
EVENT_WRITERS = []
EVENT_STORAGE = []
GLOBAL_BUFFER = {}
class EventName(enum.Enum):
"""Names of possible events that can be logged via Local Writer for convenience.
see config/logging/default_logging.yaml"""
ITER_LOAD_TIME = "Data Load (time)"
ITER_TRAIN_TIME = "Train Iter (time)"
TOTAL_TRAIN_TIME = "Train Total (time)"
ETA = "ETA (time)"
RAYS_PER_SEC = "Rays Per Sec"
CURR_TEST_PSNR = "Test PSNR"
class EventType(enum.Enum):
"""Possible Event types and their associated write function"""
IMAGE = "write_image"
SCALAR = "write_scalar"
DICT = "write_scalar_dict"
@check_main_thread
def put_image(name, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None):
"""Setter function to place images into the queue to be written out"""
if isinstance(name, EventName):
name = name.value
EVENT_STORAGE.append(
{"name": name, "write_type": EventType.IMAGE, "event": image, "step": step, "group": group, "prefix": prefix}
)
@check_main_thread
def put_scalar(name: str, scalar: float, step: int, group: str = None, prefix: str = None):
"""Setter function to place scalars into the queue to be written out"""
if isinstance(name, EventName):
name = name.value
EVENT_STORAGE.append(
{"name": name, "write_type": EventType.SCALAR, "event": scalar, "step": step, "group": group, "prefix": prefix}
)
@check_main_thread
def put_dict(name: str, scalar_dict: float, step: int, group: str = None, prefix: str = None):
"""Setter function to place a dictionary of scalars into the queue to be written out"""
EVENT_STORAGE.append(
{
"name": name,
"write_type": EventType.DICT,
"event": scalar_dict,
"step": step,
"group": group,
"prefix": prefix,
}
)
@check_main_thread
def put_time(
name: str,
start_time: float,
end_time: float,
step: int,
group: str = None,
prefix: str = None,
avg_over_iters: bool = False,
avg_over_batch: int = None,
update_eta: bool = False,
):
"""Setter function to place a time element into the queue to be written out
Processes the time info according to the options:
avg_over_iters (bool): if True, calculate and record a running average of the times
avg_over_batch (int): if set, the size of the batch for which we take the average over (batch/second)
update_eta (bool): if True, update the ETA. should only be set for the training iterations/s
"""
if isinstance(name, EventName):
name = name.value
GLOBAL_BUFFER["step"] = step
val = end_time - start_time
if avg_over_batch:
val = avg_over_batch / val
if avg_over_iters:
curr_event = GLOBAL_BUFFER["events"].get(name, {"buffer": [], "avg": 0})
curr_buffer = curr_event["buffer"]
curr_avg = curr_event["avg"]
if len(curr_buffer) >= GLOBAL_BUFFER["max_buffer_size"]:
curr_buffer.pop(0)
curr_buffer.append(val)
curr_avg = sum(curr_buffer) / len(curr_buffer)
put_scalar(name, curr_avg, step, group, prefix)
GLOBAL_BUFFER["events"][name] = {"buffer": curr_buffer, "avg": curr_avg}
else:
put_scalar(name, val, step, group, prefix)
if update_eta:
## NOTE: eta should be called with avg train iteration time
remain_iter = GLOBAL_BUFFER["max_iter"] - step
remain_time = remain_iter * GLOBAL_BUFFER["events"][name]["avg"]
put_scalar(EventName.ETA, remain_time, step, group, prefix)
@check_main_thread
def write_out_storage():
"""Function that writes all the events in storage to all the writer locations"""
for writer in EVENT_WRITERS:
for event in EVENT_STORAGE:
write_func = getattr(writer, event["write_type"].value)
if event["write_type"] == EventType.DICT:
write_func(event["event"], event["step"], event["group"], event["prefix"])
if isinstance(writer, LocalWriter):
continue
else:
write_func(event["name"], event["event"], event["step"], event["group"], event["prefix"])
if isinstance(writer, LocalWriter):
break
EVENT_STORAGE.clear()
@check_main_thread
def setup_event_writers(config: DictConfig) -> None:
"""Initialization of all event writers specified in config"""
logging_configs = config.logging.writer
for writer_type in logging_configs:
writer_class = getattr(radiance.utils.writer, writer_type)
writer_config = logging_configs[writer_type]
if writer_type == "LocalWriter":
curr_writer = writer_class(writer_config.save_dir, writer_config.stats_to_track, writer_config.max_log_size)
else:
curr_writer = writer_class(writer_config.save_dir)
EVENT_WRITERS.append(curr_writer)
## configure all the global buffer basic information
GLOBAL_BUFFER["max_iter"] = config.graph.max_num_iterations
GLOBAL_BUFFER["max_buffer_size"] = config.logging.max_buffer_size
GLOBAL_BUFFER["steps_per_log"] = config.logging.steps_per_log
GLOBAL_BUFFER["events"] = {}
def get_tensorboard_name(name: str, group: str = None, prefix: str = None):
"""Returns a string for tensorboard with an optional group and prefix.
Where tensorboard_name has the form `group/prefix-name`.
"""
group_string = f"{group}/" if group else ""
prefix_string = f"{prefix}" if prefix else ""
tensorboard_name = f"{group_string}{prefix_string}{name}"
return tensorboard_name
class Writer:
"""Writer class"""
def __init__(self, save_dir: str):
self.save_dir = save_dir
@abstractmethod
def write_image(
self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None
) -> None:
"""_summary_
Args:
name (str): data identifier
image (TensorType["H", "W", "C"]): rendered image to write
step (int): the time step to log
group (str): the group e.g., "Loss", "Accuracy", "Time"
prefix (str): the prefix e.g., "train-", "test-"
"""
raise NotImplementedError
@abstractmethod
def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None:
"""Required method to write a single scalar value to the logger
Args:
name (str): data identifier
step (int): the time step to log
group (str): the group e.g., "Loss", "Accuracy", "Time"
prefix (str): the prefix e.g., "train-", "test-"
"""
raise NotImplementedError
@check_main_thread
def write_scalar_dict(
self, scalar_dict: Dict[str, float], step: int, group: str = None, prefix: str = None
) -> None:
"""Function that writes out all scalars from a given dictionary to the logger
Args:
scalar_dict (dict): dictionary containing all scalar values with key names and quantities
step (int): the time step to log
group (str): the group e.g., "Loss", "Accuracy", "Time"
prefix (str): the prefix e.g., "train-", "test-"
"""
for name, scalar in scalar_dict.items():
self.write_scalar(name, scalar, step, group=group, prefix=prefix)
@decorate_all([check_main_thread])
class TensorboardWriter(Writer):
"""Tensorboard Writer Class"""
def __init__(self, save_dir: str):
super().__init__(save_dir)
self.tb_writer = SummaryWriter(log_dir=self.save_dir)
def write_image(
self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None
) -> None:
"""_summary_
Args:
name (str): data identifier
image (TensorType["H", "W", "C"]): rendered image to write
"""
image = to8b(image)
tensorboard_name = get_tensorboard_name(name, group=group, prefix=prefix)
self.tb_writer.add_image(tensorboard_name, image, step, dataformats="HWC")
def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None:
"""Tensorboard method to write a single scalar value to the logger
Args:
name (str): data identifier
x (float): x value to write
y (float): y value to write
group (str)): a prefix to group tensorboard scalars
"""
tensorboard_name = get_tensorboard_name(name, group=group, prefix=prefix)
self.tb_writer.add_scalar(tensorboard_name, scalar, step)
def _cursorup(x: int):
"""utility tool to move the cursor up on the terminal
Args:
x (int): amount of lines to move cursor upward
"""
print(f"\r\033[{x}A", end="\x1b[1K\r")
def _format_time(seconds):
"""utility tool to format time in human readable form given seconds"""
ms = seconds % 1
ms = ms * 1e3
seconds = int(seconds)
days, seconds = divmod(seconds, 86400)
hours, seconds = divmod(seconds, 3600)
minutes, seconds = divmod(seconds, 60)
if days > 0:
return f"{days} d, {hours} h, {minutes} m, {seconds} s"
if hours > 0:
return f"{hours} h, {minutes} m, {seconds} s"
if minutes > 0:
return f"{minutes} m, {seconds} s"
if seconds > 0:
return f"{seconds} s, {ms:0.3f} ms"
return f"{ms:0.3f} ms"
@decorate_all([check_main_thread])
class LocalWriter(Writer):
"""Local Writer Class"""
def __init__(self, save_dir: str, stats_to_track: ListConfig, max_log_size: int = 0):
"""
Args:
stats_to_track (ListConfig): the names of stats that should be logged.
max_log size (int): max number of lines that will be logged to teminal.
"""
super().__init__(save_dir)
self.stats_to_track = [EventName[name].value for name in stats_to_track]
self.max_log_size = max_log_size
self.keys = set()
self.past_mssgs = ["", ""]
def write_image(
self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None
) -> None:
if name in self.stats_to_track:
image = to8b(image)
image_path = os.path.join(self.save_dir, f"{name}.jpg")
imageio.imwrite(image_path, np.uint8(image.cpu().numpy() * 255.0))
def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None:
if step > 0:
latest_map, new_key = self._consolidate_events()
self._update_header(latest_map, new_key)
self._print_stats(latest_map)
def _consolidate_events(self):
latest_map = {}
new_key = False
for event in EVENT_STORAGE:
name = event["name"]
if name not in self.keys:
self.keys.add(name)
new_key = True
latest_map[name] = event["event"]
return latest_map, new_key
def _update_header(self, latest_map, new_key):
"""helper to handle the printing of the header labels"""
full_log_cond = not self.max_log_size and GLOBAL_BUFFER["step"] <= GLOBAL_BUFFER["steps_per_log"]
capped_log_cond = self.max_log_size and (len(self.past_mssgs) <= 2 or new_key)
if full_log_cond or capped_log_cond:
mssg = f"{'Step (% Done)':<20}"
for name, _ in latest_map.items():
if name in self.stats_to_track:
mssg += f"{name:<20} "
self.past_mssgs[0] = mssg
self.past_mssgs[1] = "-" * len(mssg)
if full_log_cond:
print(mssg)
print("-" * len(mssg))
def _print_stats(self, latest_map, padding=" "):
"""helper to print out the stats in a readable format"""
step = GLOBAL_BUFFER["step"]
fraction_done = step / GLOBAL_BUFFER["max_iter"]
curr_mssg = f"{step} ({fraction_done*100:.02f}%)"
curr_mssg = f"{curr_mssg:<20}"
for name, v in latest_map.items():
if name in self.stats_to_track:
if "(time)" in name:
v = _format_time(v)
else:
v = f"{v:0.4f}"
curr_mssg += f"{v:<20} "
# update the history buffer
if self.max_log_size:
cursor_idx = len(self.past_mssgs)
if len(self.past_mssgs[2:]) >= self.max_log_size:
self.past_mssgs.pop(2)
self.past_mssgs.append(curr_mssg)
_cursorup(cursor_idx)
for mssg in self.past_mssgs:
pad_len = len(self.past_mssgs[0])
print(f"{mssg:{padding}<{pad_len}}")
else:
print(curr_mssg)