mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 07:05:44 +08:00
Adios mattport
This commit is contained in:
1 parent
9d2b34ab60
commit
b26408eb7a
86 files changed
+176
-176
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
Callback functions for training iterations
|
||||
"""
|
||||
|
||||
from radiance.nerf.graph.base import Graph
|
||||
from radiance.nerf.occupancy_grid import OccupancyGrid
|
||||
|
||||
|
||||
def update_occupancy(graph: Graph):
|
||||
"""Update the occupancy grid depending on iteration count after every train step"""
|
||||
if not hasattr(graph, "occupancy_pointers"):
|
||||
occupancy_grids = []
|
||||
for _, module in graph.__dict__["_modules"].items():
|
||||
if isinstance(module, OccupancyGrid):
|
||||
occupancy_grids.append(module)
|
||||
graph.occupancy_pointers = occupancy_grids
|
||||
for grid in graph.occupancy_pointers:
|
||||
grid.iteration_count += 1
|
||||
if grid.iteration_count % grid.update_every_num_iters == 0:
|
||||
if grid.density_fn is not None:
|
||||
grid.update_occupancy_grid(grid.density_fn)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""functionality to handle multiprocessing syncing and communicating"""
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
_LOCAL_PROCESS_GROUP = None
|
||||
|
||||
|
||||
def get_world_size() -> int:
|
||||
"""Get total number of available gpus"""
|
||||
if not dist.is_available():
|
||||
return 1
|
||||
if not dist.is_initialized():
|
||||
return 1
|
||||
return dist.get_world_size()
|
||||
|
||||
|
||||
def get_rank() -> int:
|
||||
"""Get global rank of current thread"""
|
||||
if not dist.is_available():
|
||||
return 0
|
||||
if not dist.is_initialized():
|
||||
return 0
|
||||
return dist.get_rank()
|
||||
|
||||
|
||||
def get_local_rank() -> int:
|
||||
"""The rank of the current process within the local (per-machine) process group."""
|
||||
if not dist.is_available():
|
||||
return 0
|
||||
if not dist.is_initialized():
|
||||
return 0
|
||||
assert (
|
||||
_LOCAL_PROCESS_GROUP is not None
|
||||
), "Local process group is not created! Please use launch() to spawn processes!"
|
||||
return dist.get_rank(group=_LOCAL_PROCESS_GROUP)
|
||||
|
||||
|
||||
def get_local_size() -> int:
|
||||
"""
|
||||
The size of the per-machine process group,
|
||||
i.e. the number of processes per machine.
|
||||
"""
|
||||
if not dist.is_available():
|
||||
return 1
|
||||
if not dist.is_initialized():
|
||||
return 1
|
||||
return dist.get_world_size(group=_LOCAL_PROCESS_GROUP)
|
||||
|
||||
|
||||
def is_main_process() -> bool:
|
||||
"""check to see if you are currently on the main process"""
|
||||
return get_rank() == 0
|
||||
|
||||
|
||||
def synchronize(world_size):
|
||||
"""
|
||||
Helper function to synchronize (barrier) among all processes when
|
||||
using distributed training
|
||||
"""
|
||||
if not dist.is_available():
|
||||
return
|
||||
if not dist.is_initialized():
|
||||
return
|
||||
world_size = dist.get_world_size()
|
||||
if world_size == 1:
|
||||
return
|
||||
if dist.get_backend() == dist.Backend.NCCL:
|
||||
# This argument is needed to avoid warnings.
|
||||
# It's valid only for NCCL backend.
|
||||
dist.barrier(device_ids=[torch.cuda.current_device()])
|
||||
else:
|
||||
dist.barrier()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Decorator definitions
|
||||
"""
|
||||
from typing import Callable, List
|
||||
|
||||
from radiance.utils import comms
|
||||
|
||||
|
||||
def decorate_all(decorators: List[Callable]) -> Callable:
|
||||
"""A decorator to decorate all member functions of a class"""
|
||||
|
||||
def decorate(cls):
|
||||
for attr in cls.__dict__:
|
||||
if callable(getattr(cls, attr)) and attr != "__init__":
|
||||
for decorator in decorators:
|
||||
setattr(cls, attr, decorator(getattr(cls, attr)))
|
||||
return cls
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
def check_profiler_enabled(func: Callable) -> Callable:
|
||||
"""Decorator: check if profiler is enabled"""
|
||||
|
||||
def wrapper(self, *args, **kwargs):
|
||||
ret = None
|
||||
if self.config.enable_profiler:
|
||||
ret = func(self, *args, **kwargs)
|
||||
return ret
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def check_main_thread(func: Callable) -> Callable:
|
||||
"""Decorator: check if you are on main thread"""
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
ret = None
|
||||
if comms.is_main_process():
|
||||
ret = func(*args, **kwargs)
|
||||
return ret
|
||||
|
||||
return wrapper
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
Input/output utils.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
def load_from_json(filename: str):
|
||||
"""_summary_
|
||||
|
||||
Args:
|
||||
filename (str): _description_
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
assert filename.endswith(".json")
|
||||
with open(filename, "r", encoding="UTF-8") as file:
|
||||
return json.load(file)
|
||||
|
||||
|
||||
def write_to_json(filename: str, content: dict):
|
||||
"""_summary_
|
||||
|
||||
Args:
|
||||
filename (str): _description_
|
||||
content (dict): _description_
|
||||
"""
|
||||
assert filename.endswith(".json")
|
||||
with open(filename, "w", encoding="UTF-8") as file:
|
||||
json.dump(content, file)
|
||||
|
||||
|
||||
def get_git_root(path, dirs=(".git",), default=None):
|
||||
"""_summary_
|
||||
"https://stackoverflow.com/questions/22081209/find-the-root-of-the-git-repository-where-the-file-lives
|
||||
|
||||
Args:
|
||||
path (_type_): _description_
|
||||
dirs (tuple, optional): _description_. Defaults to (".git",).
|
||||
default (_type_, optional): _description_. Defaults to None.
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
prev, test = None, os.path.abspath(path)
|
||||
while prev != test:
|
||||
if any(os.path.isdir(os.path.join(test, directory)) for directory in dirs):
|
||||
return test
|
||||
prev, test = test, os.path.abspath(os.path.join(test, os.pardir))
|
||||
return default
|
||||
|
||||
|
||||
def get_project_root(path):
|
||||
"""Return the project root directory from an environment variable.
|
||||
# TODO: handle this better and report user error
|
||||
"""
|
||||
project_root = os.getenv("PROJECT_ROOT")
|
||||
if project_root is None:
|
||||
logging.info(
|
||||
(
|
||||
"Please set PROJECT_ROOT to the root directory of this repo. "
|
||||
"Going to try calling get_git_root(path) instead."
|
||||
)
|
||||
)
|
||||
project_root = get_git_root(path)
|
||||
return project_root
|
||||
|
||||
|
||||
def get_absolute_path(path, proj_root_func=get_project_root):
|
||||
"""
|
||||
Returns the full, absolute path.
|
||||
Relative paths are assumed to start at the repo directory.
|
||||
"""
|
||||
if path == "":
|
||||
return ""
|
||||
absolute_path = path
|
||||
if absolute_path[0] != "/":
|
||||
absolute_path = os.path.join(proj_root_func(path), absolute_path)
|
||||
return absolute_path
|
||||
|
||||
|
||||
def make_dir(filename_or_folder):
|
||||
"""Make the directory for either the filename or folder.
|
||||
Note that filename_or_folder currently needs to end in / for it to be recognized as a folder.
|
||||
"""
|
||||
if filename_or_folder[-1] != "/" and filename_or_folder.find(".") < 0:
|
||||
folder = os.path.dirname(filename_or_folder + "/")
|
||||
else:
|
||||
folder = os.path.dirname(filename_or_folder)
|
||||
if not os.path.exists(folder):
|
||||
try:
|
||||
os.makedirs(folder)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
print(f"Couldn't create folder: {folder}. Maybe due to a parallel process?")
|
||||
print(e)
|
||||
return filename_or_folder
|
||||
@@ -0,0 +1,69 @@
|
||||
""" Math Helper Functions """
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
|
||||
|
||||
def components_from_spherical_harmonics(levels: int, directions: TensorType[..., 3]) -> TensorType[..., "components"]:
|
||||
"""
|
||||
Args:
|
||||
levels (int): Number of spherical harmonic levels to compute.
|
||||
coeffs (TensorType[..., "coeffs"]): Spherical hamonic coefficients
|
||||
|
||||
Returns:
|
||||
TensorType[..., "components"]: Value for each component.
|
||||
"""
|
||||
num_components = levels**2
|
||||
components = torch.zeros((*directions.shape[:-1], num_components)).to(directions.device)
|
||||
|
||||
assert 1 <= levels <= 5, f"SH levels must be in [1,4], got {levels}"
|
||||
assert directions.shape[-1] == 3, f"Direction input should have three dimensions. Got {directions.shape[-1]}"
|
||||
|
||||
x = directions[..., 0]
|
||||
y = directions[..., 1]
|
||||
z = directions[..., 2]
|
||||
|
||||
xx = x**2
|
||||
yy = y**2
|
||||
zz = z**2
|
||||
|
||||
# l0
|
||||
components[..., 0] = 0.28209479177387814
|
||||
|
||||
# l1
|
||||
if levels > 1:
|
||||
components[..., 1] = 0.4886025119029199 * y
|
||||
components[..., 2] = 0.4886025119029199 * z
|
||||
components[..., 3] = 0.4886025119029199 * x
|
||||
|
||||
# l2
|
||||
if levels > 2:
|
||||
components[..., 4] = 1.0925484305920792 * x * y
|
||||
components[..., 5] = 1.0925484305920792 * y * z
|
||||
components[..., 6] = 0.9461746957575601 * zz
|
||||
components[..., 7] = 1.0925484305920792 * x * z
|
||||
components[..., 8] = 0.5462742152960396 * (xx - yy)
|
||||
|
||||
# l3
|
||||
if levels > 3:
|
||||
components[..., 9] = 0.5900435899266435 * y * (3 * xx - yy)
|
||||
components[..., 10] = 2.890611442640554 * x * y * z
|
||||
components[..., 11] = 0.4570457994644658 * y * (5 * zz - 1)
|
||||
components[..., 12] = 0.3731763325901154 * z * (5 * zz - 3)
|
||||
components[..., 13] = 0.4570457994644658 * x * (5 * zz - 1)
|
||||
components[..., 14] = 1.445305721320277 * z * (xx - yy)
|
||||
components[..., 15] = 0.5900435899266435 * x * (xx - 3 * yy)
|
||||
|
||||
# l4
|
||||
if levels > 4:
|
||||
components[..., 16] = 2.5033429417967046 * x * y * (xx - yy)
|
||||
components[..., 17] = 1.7701307697799304 * y * z * (3 * xx - yy)
|
||||
components[..., 18] = 0.9461746957575601 * x * y * (7 * zz - 1)
|
||||
components[..., 19] = 0.6690465435572892 * y * (7 * zz - 3 * z)
|
||||
components[..., 20] = 0.10578554691520431 * (35 * zz * zz - 30 * zz + 3)
|
||||
components[..., 21] = 0.6690465435572892 * x * z * (7 * zz - 3)
|
||||
components[..., 22] = 0.47308734787878004 * (xx - yy) * (7 * zz - 1)
|
||||
components[..., 23] = 1.7701307697799304 * x * z * (xx - 3 * yy)
|
||||
components[..., 24] = 0.4425326924449826 * (xx * (xx - 3 * yy) - yy * (3 * xx - yy))
|
||||
|
||||
return components
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Miscellaneous helper code.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class DotDict(dict):
|
||||
"""
|
||||
dot.notation access to dictionary attributes
|
||||
"""
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
__setattr__ = dict.__setitem__
|
||||
__delattr__ = dict.__delitem__
|
||||
|
||||
|
||||
def get_dict_to_torch(stuff, device="cpu"):
|
||||
"""Set everything in the dict to the specified torch device."""
|
||||
if isinstance(stuff, dict):
|
||||
for k, v in stuff.items():
|
||||
stuff[k] = get_dict_to_torch(v, device)
|
||||
return stuff
|
||||
if isinstance(stuff, torch.Tensor):
|
||||
return stuff.to(device)
|
||||
return stuff
|
||||
|
||||
|
||||
def get_dict_to_cpu(stuff):
|
||||
"""Set everything in the dict to CPU."""
|
||||
if isinstance(stuff, dict):
|
||||
for k, v in stuff.items():
|
||||
stuff[k] = get_dict_to_cpu(v)
|
||||
return stuff
|
||||
if isinstance(stuff, torch.Tensor):
|
||||
return stuff.detach().cpu()
|
||||
return stuff
|
||||
|
||||
|
||||
def is_not_none(var):
|
||||
"""Return True if the variable var is None."""
|
||||
return not isinstance(var, type(None))
|
||||
|
||||
|
||||
def get_masked_dict(d, mask):
|
||||
"""Return a masked dictionary.
|
||||
TODO(ethan): add more asserts/checks so this doesn't have unpredictable behavior."""
|
||||
masked_dict = {}
|
||||
for key, value in d.items():
|
||||
masked_dict[key] = value[mask]
|
||||
return masked_dict
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Profiler base class and functionality
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from radiance.utils import comms
|
||||
from radiance.utils.decorators import check_main_thread, check_profiler_enabled, decorate_all
|
||||
|
||||
PROFILER = []
|
||||
|
||||
|
||||
def time_function(func: Callable) -> Callable:
|
||||
"""Decorator: time a function call"""
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
start = time.time()
|
||||
ret = func(*args, **kwargs)
|
||||
if PROFILER:
|
||||
class_str = func.__qualname__
|
||||
PROFILER[0].update_time(class_str, start, time.time())
|
||||
return ret
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def flush_profiler(config: DictConfig):
|
||||
"""Method that checks if profiler is enabled before flushing"""
|
||||
if config.enable_profiler and PROFILER:
|
||||
PROFILER[0].print_profile()
|
||||
|
||||
|
||||
def setup_profiler(config: DictConfig):
|
||||
"""Initialization of profilers"""
|
||||
if comms.is_main_process():
|
||||
PROFILER.append(Profiler(config))
|
||||
|
||||
|
||||
@decorate_all([check_profiler_enabled, check_main_thread])
|
||||
class Profiler:
|
||||
"""Profiler class"""
|
||||
|
||||
def __init__(self, config: DictConfig):
|
||||
self.config = config
|
||||
self.profiler_dict = {}
|
||||
|
||||
def update_time(self, func_name: str, start_time: float, end_time: float):
|
||||
"""update the profiler dictionary with running averages of durations
|
||||
|
||||
Args:
|
||||
func_name (str): the function name that is being profiled
|
||||
start_time (float): the start time when function is called
|
||||
end_time (float): the end time when function terminated
|
||||
"""
|
||||
val = end_time - start_time
|
||||
func_dict = self.profiler_dict.get(func_name, {"val": 0, "step": 0})
|
||||
prev_val = func_dict["val"]
|
||||
prev_step = func_dict["step"]
|
||||
self.profiler_dict[func_name] = {"val": (prev_val * prev_step + val) / (prev_step + 1), "step": prev_step + 1}
|
||||
|
||||
def print_profile(self):
|
||||
"""helper to print out the profiler stats"""
|
||||
logging.info("Printing profiling stats, from longest to shortest duration in seconds")
|
||||
sorted_keys = [k for k, _ in sorted(self.profiler_dict.items(), key=lambda item: item[1]["val"], reverse=True)]
|
||||
for k in sorted_keys:
|
||||
val = f"{self.profiler_dict[k]['val']:0.4f}"
|
||||
print(f"{k:<20}: {val:<20}")
|
||||
@@ -0,0 +1,61 @@
|
||||
""" Helper functions for visualizing outputs """
|
||||
|
||||
from typing import Optional
|
||||
import torch
|
||||
from matplotlib import cm
|
||||
from torchtyping import TensorType
|
||||
|
||||
|
||||
def apply_colormap(image: TensorType[..., 1], cmap="viridis") -> TensorType[..., 3]:
|
||||
"""Convert single channel to a color image.
|
||||
|
||||
Args:
|
||||
image (TensorType[..., 1]): Single channel image.
|
||||
cmap (str, optional): Colormap for image. Defaults to 'turbo'.
|
||||
|
||||
Returns:
|
||||
TensorType[..., 3]: Colored image
|
||||
"""
|
||||
|
||||
colormap = cm.get_cmap(cmap)
|
||||
colormap = torch.tensor(colormap.colors).to(image.device)
|
||||
image_long = (image * 255).long()
|
||||
image_long_min = torch.min(image_long)
|
||||
image_long_max = torch.max(image_long)
|
||||
assert image_long_min >= 0, f"the min value is {image_long_min}"
|
||||
assert image_long_max <= 255, f"the max value is {image_long_max}"
|
||||
return colormap[image_long[..., 0]]
|
||||
|
||||
|
||||
def apply_depth_colormap(
|
||||
depth: TensorType[..., 1],
|
||||
accumulation: Optional[TensorType[..., 1]] = None,
|
||||
near_plane: Optional[float] = None,
|
||||
far_plane: Optional[float] = None,
|
||||
cmap="turbo",
|
||||
) -> TensorType[..., 3]:
|
||||
"""Converts a depth image to color for easier analysis.
|
||||
|
||||
Args:
|
||||
depth (TensorType[..., 1]): Depth image.
|
||||
accumulation (TensorType[..., 1], optional): Ray accumulation used for masking vis. Defaults to None.
|
||||
near_plane (float, optional): Closest depth to consider. If None, use min image value. Defaults to None.
|
||||
far_plane (float, optional): Furthest depth to consider. If None, use max image value. Defaults to None.
|
||||
cmap (str, optional): Colormap to apply. Defaults to "turbo".
|
||||
|
||||
Returns:
|
||||
TensorType[..., 3]: Colored depth image
|
||||
"""
|
||||
|
||||
near_plane = near_plane or torch.min(depth)
|
||||
far_plane = far_plane or torch.max(depth)
|
||||
|
||||
depth = (depth - near_plane) / (far_plane - near_plane + 1e-10)
|
||||
depth = torch.clip(depth, 0, 1)
|
||||
|
||||
colored_image = apply_colormap(depth, cmap=cmap)
|
||||
|
||||
if accumulation is not None:
|
||||
colored_image = colored_image * accumulation + (1 - accumulation)
|
||||
|
||||
return colored_image
|
||||
@@ -0,0 +1,372 @@
|
||||
"""
|
||||
Generic Writer class
|
||||
"""
|
||||
|
||||
|
||||
import enum
|
||||
import os
|
||||
from abc import abstractmethod
|
||||
from typing import Dict
|
||||
|
||||
import imageio
|
||||
import numpy as np
|
||||
import torch
|
||||
from omegaconf import DictConfig, ListConfig
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
from torchtyping import TensorType
|
||||
|
||||
import radiance.utils.writer
|
||||
from radiance.utils.decorators import check_main_thread, decorate_all
|
||||
|
||||
to8b = lambda x: (255 * torch.clamp(x, min=0, max=1)).to(torch.uint8)
|
||||
EVENT_WRITERS = []
|
||||
EVENT_STORAGE = []
|
||||
GLOBAL_BUFFER = {}
|
||||
|
||||
|
||||
class EventName(enum.Enum):
|
||||
"""Names of possible events that can be logged via Local Writer for convenience.
|
||||
see config/logging/default_logging.yaml"""
|
||||
|
||||
ITER_LOAD_TIME = "Data Load (time)"
|
||||
ITER_TRAIN_TIME = "Train Iter (time)"
|
||||
TOTAL_TRAIN_TIME = "Train Total (time)"
|
||||
ETA = "ETA (time)"
|
||||
RAYS_PER_SEC = "Rays Per Sec"
|
||||
CURR_TEST_PSNR = "Test PSNR"
|
||||
|
||||
|
||||
class EventType(enum.Enum):
|
||||
"""Possible Event types and their associated write function"""
|
||||
|
||||
IMAGE = "write_image"
|
||||
SCALAR = "write_scalar"
|
||||
DICT = "write_scalar_dict"
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def put_image(name, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None):
|
||||
"""Setter function to place images into the queue to be written out"""
|
||||
if isinstance(name, EventName):
|
||||
name = name.value
|
||||
|
||||
EVENT_STORAGE.append(
|
||||
{"name": name, "write_type": EventType.IMAGE, "event": image, "step": step, "group": group, "prefix": prefix}
|
||||
)
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def put_scalar(name: str, scalar: float, step: int, group: str = None, prefix: str = None):
|
||||
"""Setter function to place scalars into the queue to be written out"""
|
||||
if isinstance(name, EventName):
|
||||
name = name.value
|
||||
|
||||
EVENT_STORAGE.append(
|
||||
{"name": name, "write_type": EventType.SCALAR, "event": scalar, "step": step, "group": group, "prefix": prefix}
|
||||
)
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def put_dict(name: str, scalar_dict: float, step: int, group: str = None, prefix: str = None):
|
||||
"""Setter function to place a dictionary of scalars into the queue to be written out"""
|
||||
EVENT_STORAGE.append(
|
||||
{
|
||||
"name": name,
|
||||
"write_type": EventType.DICT,
|
||||
"event": scalar_dict,
|
||||
"step": step,
|
||||
"group": group,
|
||||
"prefix": prefix,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def put_time(
|
||||
name: str,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
step: int,
|
||||
group: str = None,
|
||||
prefix: str = None,
|
||||
avg_over_iters: bool = False,
|
||||
avg_over_batch: int = None,
|
||||
update_eta: bool = False,
|
||||
):
|
||||
"""Setter function to place a time element into the queue to be written out
|
||||
|
||||
Processes the time info according to the options:
|
||||
avg_over_iters (bool): if True, calculate and record a running average of the times
|
||||
avg_over_batch (int): if set, the size of the batch for which we take the average over (batch/second)
|
||||
update_eta (bool): if True, update the ETA. should only be set for the training iterations/s
|
||||
"""
|
||||
if isinstance(name, EventName):
|
||||
name = name.value
|
||||
|
||||
GLOBAL_BUFFER["step"] = step
|
||||
val = end_time - start_time
|
||||
if avg_over_batch:
|
||||
val = avg_over_batch / val
|
||||
|
||||
if avg_over_iters:
|
||||
curr_event = GLOBAL_BUFFER["events"].get(name, {"buffer": [], "avg": 0})
|
||||
curr_buffer = curr_event["buffer"]
|
||||
curr_avg = curr_event["avg"]
|
||||
if len(curr_buffer) >= GLOBAL_BUFFER["max_buffer_size"]:
|
||||
curr_buffer.pop(0)
|
||||
curr_buffer.append(val)
|
||||
curr_avg = sum(curr_buffer) / len(curr_buffer)
|
||||
put_scalar(name, curr_avg, step, group, prefix)
|
||||
GLOBAL_BUFFER["events"][name] = {"buffer": curr_buffer, "avg": curr_avg}
|
||||
else:
|
||||
put_scalar(name, val, step, group, prefix)
|
||||
|
||||
if update_eta:
|
||||
## NOTE: eta should be called with avg train iteration time
|
||||
remain_iter = GLOBAL_BUFFER["max_iter"] - step
|
||||
remain_time = remain_iter * GLOBAL_BUFFER["events"][name]["avg"]
|
||||
put_scalar(EventName.ETA, remain_time, step, group, prefix)
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def write_out_storage():
|
||||
"""Function that writes all the events in storage to all the writer locations"""
|
||||
for writer in EVENT_WRITERS:
|
||||
for event in EVENT_STORAGE:
|
||||
write_func = getattr(writer, event["write_type"].value)
|
||||
if event["write_type"] == EventType.DICT:
|
||||
write_func(event["event"], event["step"], event["group"], event["prefix"])
|
||||
if isinstance(writer, LocalWriter):
|
||||
continue
|
||||
else:
|
||||
write_func(event["name"], event["event"], event["step"], event["group"], event["prefix"])
|
||||
if isinstance(writer, LocalWriter):
|
||||
break
|
||||
EVENT_STORAGE.clear()
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def setup_event_writers(config: DictConfig) -> None:
|
||||
"""Initialization of all event writers specified in config"""
|
||||
logging_configs = config.logging.writer
|
||||
for writer_type in logging_configs:
|
||||
writer_class = getattr(radiance.utils.writer, writer_type)
|
||||
writer_config = logging_configs[writer_type]
|
||||
if writer_type == "LocalWriter":
|
||||
curr_writer = writer_class(writer_config.save_dir, writer_config.stats_to_track, writer_config.max_log_size)
|
||||
else:
|
||||
curr_writer = writer_class(writer_config.save_dir)
|
||||
EVENT_WRITERS.append(curr_writer)
|
||||
|
||||
## configure all the global buffer basic information
|
||||
GLOBAL_BUFFER["max_iter"] = config.graph.max_num_iterations
|
||||
GLOBAL_BUFFER["max_buffer_size"] = config.logging.max_buffer_size
|
||||
GLOBAL_BUFFER["steps_per_log"] = config.logging.steps_per_log
|
||||
GLOBAL_BUFFER["events"] = {}
|
||||
|
||||
|
||||
def get_tensorboard_name(name: str, group: str = None, prefix: str = None):
|
||||
"""Returns a string for tensorboard with an optional group and prefix.
|
||||
Where tensorboard_name has the form `group/prefix-name`.
|
||||
"""
|
||||
group_string = f"{group}/" if group else ""
|
||||
prefix_string = f"{prefix}" if prefix else ""
|
||||
tensorboard_name = f"{group_string}{prefix_string}{name}"
|
||||
return tensorboard_name
|
||||
|
||||
|
||||
class Writer:
|
||||
"""Writer class"""
|
||||
|
||||
def __init__(self, save_dir: str):
|
||||
self.save_dir = save_dir
|
||||
|
||||
@abstractmethod
|
||||
def write_image(
|
||||
self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None
|
||||
) -> None:
|
||||
"""_summary_
|
||||
|
||||
Args:
|
||||
name (str): data identifier
|
||||
image (TensorType["H", "W", "C"]): rendered image to write
|
||||
step (int): the time step to log
|
||||
group (str): the group e.g., "Loss", "Accuracy", "Time"
|
||||
prefix (str): the prefix e.g., "train-", "test-"
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None:
|
||||
"""Required method to write a single scalar value to the logger
|
||||
|
||||
Args:
|
||||
name (str): data identifier
|
||||
step (int): the time step to log
|
||||
group (str): the group e.g., "Loss", "Accuracy", "Time"
|
||||
prefix (str): the prefix e.g., "train-", "test-"
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@check_main_thread
|
||||
def write_scalar_dict(
|
||||
self, scalar_dict: Dict[str, float], step: int, group: str = None, prefix: str = None
|
||||
) -> None:
|
||||
"""Function that writes out all scalars from a given dictionary to the logger
|
||||
|
||||
Args:
|
||||
scalar_dict (dict): dictionary containing all scalar values with key names and quantities
|
||||
step (int): the time step to log
|
||||
group (str): the group e.g., "Loss", "Accuracy", "Time"
|
||||
prefix (str): the prefix e.g., "train-", "test-"
|
||||
"""
|
||||
for name, scalar in scalar_dict.items():
|
||||
self.write_scalar(name, scalar, step, group=group, prefix=prefix)
|
||||
|
||||
|
||||
@decorate_all([check_main_thread])
|
||||
class TensorboardWriter(Writer):
|
||||
"""Tensorboard Writer Class"""
|
||||
|
||||
def __init__(self, save_dir: str):
|
||||
super().__init__(save_dir)
|
||||
self.tb_writer = SummaryWriter(log_dir=self.save_dir)
|
||||
|
||||
def write_image(
|
||||
self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None
|
||||
) -> None:
|
||||
"""_summary_
|
||||
|
||||
Args:
|
||||
name (str): data identifier
|
||||
image (TensorType["H", "W", "C"]): rendered image to write
|
||||
"""
|
||||
image = to8b(image)
|
||||
tensorboard_name = get_tensorboard_name(name, group=group, prefix=prefix)
|
||||
self.tb_writer.add_image(tensorboard_name, image, step, dataformats="HWC")
|
||||
|
||||
def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None:
|
||||
"""Tensorboard method to write a single scalar value to the logger
|
||||
|
||||
Args:
|
||||
name (str): data identifier
|
||||
x (float): x value to write
|
||||
y (float): y value to write
|
||||
group (str)): a prefix to group tensorboard scalars
|
||||
"""
|
||||
tensorboard_name = get_tensorboard_name(name, group=group, prefix=prefix)
|
||||
self.tb_writer.add_scalar(tensorboard_name, scalar, step)
|
||||
|
||||
|
||||
def _cursorup(x: int):
|
||||
"""utility tool to move the cursor up on the terminal
|
||||
|
||||
Args:
|
||||
x (int): amount of lines to move cursor upward
|
||||
"""
|
||||
print(f"\r\033[{x}A", end="\x1b[1K\r")
|
||||
|
||||
|
||||
def _format_time(seconds):
|
||||
"""utility tool to format time in human readable form given seconds"""
|
||||
ms = seconds % 1
|
||||
ms = ms * 1e3
|
||||
seconds = int(seconds)
|
||||
days, seconds = divmod(seconds, 86400)
|
||||
hours, seconds = divmod(seconds, 3600)
|
||||
minutes, seconds = divmod(seconds, 60)
|
||||
if days > 0:
|
||||
return f"{days} d, {hours} h, {minutes} m, {seconds} s"
|
||||
if hours > 0:
|
||||
return f"{hours} h, {minutes} m, {seconds} s"
|
||||
if minutes > 0:
|
||||
return f"{minutes} m, {seconds} s"
|
||||
if seconds > 0:
|
||||
return f"{seconds} s, {ms:0.3f} ms"
|
||||
|
||||
return f"{ms:0.3f} ms"
|
||||
|
||||
|
||||
@decorate_all([check_main_thread])
|
||||
class LocalWriter(Writer):
|
||||
"""Local Writer Class"""
|
||||
|
||||
def __init__(self, save_dir: str, stats_to_track: ListConfig, max_log_size: int = 0):
|
||||
"""
|
||||
Args:
|
||||
stats_to_track (ListConfig): the names of stats that should be logged.
|
||||
max_log size (int): max number of lines that will be logged to teminal.
|
||||
"""
|
||||
super().__init__(save_dir)
|
||||
self.stats_to_track = [EventName[name].value for name in stats_to_track]
|
||||
self.max_log_size = max_log_size
|
||||
self.keys = set()
|
||||
self.past_mssgs = ["", ""]
|
||||
|
||||
def write_image(
|
||||
self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None
|
||||
) -> None:
|
||||
if name in self.stats_to_track:
|
||||
image = to8b(image)
|
||||
image_path = os.path.join(self.save_dir, f"{name}.jpg")
|
||||
imageio.imwrite(image_path, np.uint8(image.cpu().numpy() * 255.0))
|
||||
|
||||
def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None:
|
||||
if step > 0:
|
||||
latest_map, new_key = self._consolidate_events()
|
||||
self._update_header(latest_map, new_key)
|
||||
self._print_stats(latest_map)
|
||||
|
||||
def _consolidate_events(self):
|
||||
latest_map = {}
|
||||
new_key = False
|
||||
for event in EVENT_STORAGE:
|
||||
name = event["name"]
|
||||
if name not in self.keys:
|
||||
self.keys.add(name)
|
||||
new_key = True
|
||||
latest_map[name] = event["event"]
|
||||
return latest_map, new_key
|
||||
|
||||
def _update_header(self, latest_map, new_key):
|
||||
"""helper to handle the printing of the header labels"""
|
||||
full_log_cond = not self.max_log_size and GLOBAL_BUFFER["step"] <= GLOBAL_BUFFER["steps_per_log"]
|
||||
capped_log_cond = self.max_log_size and (len(self.past_mssgs) <= 2 or new_key)
|
||||
if full_log_cond or capped_log_cond:
|
||||
mssg = f"{'Step (% Done)':<20}"
|
||||
for name, _ in latest_map.items():
|
||||
if name in self.stats_to_track:
|
||||
mssg += f"{name:<20} "
|
||||
self.past_mssgs[0] = mssg
|
||||
self.past_mssgs[1] = "-" * len(mssg)
|
||||
if full_log_cond:
|
||||
print(mssg)
|
||||
print("-" * len(mssg))
|
||||
|
||||
def _print_stats(self, latest_map, padding=" "):
|
||||
"""helper to print out the stats in a readable format"""
|
||||
step = GLOBAL_BUFFER["step"]
|
||||
fraction_done = step / GLOBAL_BUFFER["max_iter"]
|
||||
curr_mssg = f"{step} ({fraction_done*100:.02f}%)"
|
||||
curr_mssg = f"{curr_mssg:<20}"
|
||||
for name, v in latest_map.items():
|
||||
if name in self.stats_to_track:
|
||||
if "(time)" in name:
|
||||
v = _format_time(v)
|
||||
else:
|
||||
v = f"{v:0.4f}"
|
||||
curr_mssg += f"{v:<20} "
|
||||
|
||||
# update the history buffer
|
||||
if self.max_log_size:
|
||||
cursor_idx = len(self.past_mssgs)
|
||||
if len(self.past_mssgs[2:]) >= self.max_log_size:
|
||||
self.past_mssgs.pop(2)
|
||||
self.past_mssgs.append(curr_mssg)
|
||||
_cursorup(cursor_idx)
|
||||
|
||||
for mssg in self.past_mssgs:
|
||||
pad_len = len(self.past_mssgs[0])
|
||||
print(f"{mssg:{padding}<{pad_len}}")
|
||||
else:
|
||||
print(curr_mssg)
|
||||
Reference in new issue
Block a user