refactoring configs; pulled network out of graph; created trainer heirarchy (#51)

* changed config graph structure

* removed network udner graphs; added new heirarchy trainer

* fixed callback in occupancy graph

* lint

* changed register callback to setter method
This commit is contained in:
evonneng authored and GitHub committed 2022-06-15 09:06:43 -07:00
1 parent 41073540c1
commit 91ef54963c
19 files changed
+163 -134

No files matched your search

+16 -17
View File
@@ -17,7 +17,7 @@ Code to train model.
"""
import logging
import os
from typing import Callable, Dict, List
from typing import Dict
import torch
import torch.distributed as dist
@@ -30,7 +30,6 @@ from pyrad.data.dataloader import EvalDataloader, setup_dataset_eval, setup_data
from pyrad.graphs.base import setup_graph
from pyrad.optimizers.optimizers import setup_optimizers
from pyrad.utils import profiler, writer
from pyrad.utils.callbacks import update_occupancy
from pyrad.utils.decorators import check_main_thread
from pyrad.utils.writer import EventName, TimeWriter
@@ -70,13 +69,15 @@ class Trainer:
self.graph = setup_graph(self.config.graph, dataset_inputs_train, device=self.device)
self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups())
if self.config.graph.resume_train.load_dir:
if self.config.trainer.resume_train.load_dir:
self._load_checkpoint()
if self.world_size > 1:
self.graph = DDP(self.graph, device_ids=[self.local_rank])
dist.barrier(device_ids=[self.local_rank])
self.graph.register_callbacks()
@classmethod
def get_aggregated_loss(cls, loss_dict: Dict[str, torch.tensor]):
"""Returns the aggregated losses and the scalar for calling .backwards() on.
@@ -91,21 +92,21 @@ class Trainer:
def train(self) -> None:
"""_summary_"""
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
num_iterations = self.config.graph.max_num_iterations
num_iterations = self.config.trainer.max_num_iterations
iter_dataloader_train = iter(self.dataloader_train)
for step in range(self.start_step, self.start_step + num_iterations):
with TimeWriter(writer, EventName.ITER_LOAD_TIME, step=step):
ray_indices, batch = next(iter_dataloader_train)
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as t:
loss_dict = self.train_iteration(ray_indices, batch, step, _callback=[update_occupancy])
loss_dict = self.train_iteration(ray_indices, batch, step)
writer.put_scalar(name=EventName.RAYS_PER_SEC, scalar=ray_indices.shape[0] / t.duration, step=step)
if step != 0 and step % self.config.logging.steps_per_log == 0:
writer.put_dict(name="Loss/train-loss_dict", scalar_dict=loss_dict, step=step)
if step != 0 and self.config.graph.steps_per_save and step % self.config.graph.steps_per_save == 0:
self._save_checkpoint(self.config.graph.model_dir, step)
if step % self.config.graph.steps_per_test == 0:
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
self._save_checkpoint(self.config.trainer.model_dir, step)
if step % self.config.trainer.steps_per_test == 0:
self.eval_with_dataloader(self.dataloader_eval, step=step)
self._write_out_storage(step)
@@ -119,9 +120,9 @@ class Trainer:
"""
if (
step % self.config.logging.steps_per_log == 0
or (self.config.graph.steps_per_save and step % self.config.graph.steps_per_save == 0)
or step % self.config.graph.steps_per_test == 0
or step == self.config.graph.max_num_iterations
or (self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0)
or step % self.config.trainer.steps_per_test == 0
or step == self.config.trainer.max_num_iterations
):
writer.write_out_storage()
@@ -158,9 +159,7 @@ class Trainer:
)
@profiler.time_function
def train_iteration(
self, ray_indices: TensorType["num_rays", 3], batch: dict, step: int, _callback: List[Callable] = None
) -> Dict[str, float]:
def train_iteration(self, ray_indices: TensorType["num_rays", 3], batch: dict, step: int) -> Dict[str, float]:
"""Run one iteration with a batch of inputs.
Args:
@@ -177,9 +176,9 @@ class Trainer:
loss.backward()
self.optimizers.optimizer_step_all()
self.optimizers.scheduler_step_all(step)
if _callback:
for _func in _callback:
_func(self.graph)
if self.graph.callbacks:
for _func in self.graph.callbacks:
_func.after_step(step)
return loss_dict
@profiler.time_function
+3 -3
View File
@@ -38,9 +38,9 @@ class Field(nn.Module):
frustums=Frustums(
origins=positions,
directions=torch.ones_like(positions),
frustum_starts=torch.zeros((*positions.shape[:-1], 1)),
frustum_ends=torch.zeros((*positions.shape[:-1], 1)),
pixel_area=torch.ones((*positions.shape[:-1], 1)),
frustum_starts=torch.zeros_like((positions[..., :1])),
frustum_ends=torch.zeros_like((positions[..., :1])),
pixel_area=torch.ones_like((positions[..., :1])),
)
)
density, _ = self.get_density(point_samples)
@@ -28,7 +28,7 @@ class OccupancyGrid(nn.Module):
"""Module to keep track of the density and occupancy."""
def __init__(
self, num_cascades: int = 1, resolution: int = 128, aabb=None, density_fn=None, update_every_num_iters: int = 16
self, num_cascades: int = 1, resolution: int = 128, aabb=None, update_every_num_iters: int = 16
) -> None:
super().__init__()
assert is_not_none(aabb), "The axis-aligned bounding box aabb is not defined!"
@@ -42,9 +42,7 @@ class OccupancyGrid(nn.Module):
self.register_buffer("occupancy_grid", occupancy_grid)
self.mean_density = 0.0
self.density_fn = density_fn
self.update_every_num_iters = update_every_num_iters
self.iteration_count = 0
def reset(self):
"""Zero out the occupancy grid."""
@@ -67,7 +65,6 @@ class OccupancyGrid(nn.Module):
"""
# TODO(ethan): remove this pylint disable
# pylint: disable=invalid-name
device = self.occupancy_grid.device
assert self.num_cascades == 1, "Still need to implement code for multiple cascades."
+6 -1
View File
@@ -40,7 +40,7 @@ def setup_graph(config: DictConfig, dataset_inputs: DatasetInputs, device: str)
Args:
dataset_inputs (DatasetInputs): The inputs which will be used to define the camera parameters.
"""
graph = instantiate_from_dict_config(config.network, **dataset_inputs.as_dict())
graph = instantiate_from_dict_config(config, **dataset_inputs.as_dict())
graph.to(device)
return graph
@@ -99,6 +99,11 @@ class Graph(AbstractGraph):
self.populate_collider()
self.populate_fields()
self.populate_misc_modules() # populate the modules
self.callbacks = None
def register_callbacks(self): # pylint:disable=no-self-use
"""Option to register callback for training functions"""
self.callbacks = []
def populate_collider(self):
"""Set the scene bounds collider to use."""
+11
View File
@@ -33,6 +33,7 @@ from pyrad.graphs.modules.ray_sampler import PDFSampler, UniformSampler
from pyrad.utils import colors
from pyrad.cameras.rays import RayBundle
from pyrad.utils import visualization, writer
from pyrad.utils.callbacks import Callback
class NGPGraph(Graph):
@@ -45,6 +46,16 @@ class NGPGraph(Graph):
self.field = None
super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs)
def register_callbacks(self) -> None:
"""defining callbacks to run after every training iteration"""
self.callbacks = [
Callback(
self.occupancy_grid.update_every_num_iters,
self.occupancy_grid.update_occupancy_grid,
density_fn=self.field.density_fn,
)
]
def populate_fields(self):
"""Set the fields."""
# torch or tiny-cuda-nn version
+26 -15
View File
@@ -16,20 +16,31 @@
Callback functions for training iterations
"""
from pyrad.graphs.base import Graph
from pyrad.fields.occupancy_fields.occupancy_grid import OccupancyGrid
from typing import Callable
def update_occupancy(graph: Graph):
"""Update the occupancy grid depending on iteration count after every train step"""
if not hasattr(graph, "occupancy_pointers"):
occupancy_grids = []
for _, module in graph.__dict__["_modules"].items():
if isinstance(module, OccupancyGrid):
occupancy_grids.append(module)
graph.occupancy_pointers = occupancy_grids
for grid in graph.occupancy_pointers:
grid.iteration_count += 1
if grid.iteration_count % grid.update_every_num_iters == 0:
if grid.density_fn is not None:
grid.update_occupancy_grid(grid.density_fn)
class Callback:
"""Callback Object"""
def __init__(self, update_every_num_iters: int, func: Callable, *args, **kwargs):
self.update_every_num_iters = update_every_num_iters
self.func = func
self.args = args
self.kwargs = kwargs
def before_step(self, step: int):
"""callback to run before training step
Args:
step (int): current iteration step
"""
raise NotImplementedError
def after_step(self, step: int):
"""callback to run after training step
Args:
step (int): current iteration step
"""
if step % self.update_every_num_iters == 0:
self.func(*self.args, **self.kwargs)
+14 -14
View File
@@ -141,13 +141,13 @@ def setup_event_writers(config: DictConfig) -> None:
writer_class = getattr(sys.modules[__name__], writer_type)
writer_config = logging_configs[writer_type]
if writer_type == "LocalWriter":
curr_writer = writer_class(writer_config.save_dir, writer_config.stats_to_track, writer_config.max_log_size)
curr_writer = writer_class(writer_config.log_dir, writer_config.stats_to_track, writer_config.max_log_size)
else:
curr_writer = writer_class(writer_config.save_dir)
curr_writer = writer_class(writer_config.log_dir)
EVENT_WRITERS.append(curr_writer)
## configure all the global buffer basic information
GLOBAL_BUFFER["max_iter"] = config.graph.max_num_iterations
GLOBAL_BUFFER["max_iter"] = config.trainer.max_num_iterations
GLOBAL_BUFFER["max_buffer_size"] = config.logging.max_buffer_size
GLOBAL_BUFFER["steps_per_log"] = config.logging.steps_per_log
GLOBAL_BUFFER["events"] = {}
@@ -156,8 +156,8 @@ def setup_event_writers(config: DictConfig) -> None:
class Writer:
"""Writer class"""
def __init__(self, save_dir: str):
self.save_dir = save_dir
def __init__(self, log_dir: str):
self.log_dir = log_dir
@abstractmethod
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
@@ -225,9 +225,9 @@ class TimeWriter:
class WandbWriter(Writer):
"""WandDB Writer Class"""
def __init__(self, save_dir: str):
super().__init__(save_dir)
wandb.init(dir=save_dir)
def __init__(self, log_dir: str):
super().__init__(log_dir)
wandb.init(dir=log_dir)
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
"""_summary_
@@ -253,9 +253,9 @@ class WandbWriter(Writer):
class TensorboardWriter(Writer):
"""Tensorboard Writer Class"""
def __init__(self, save_dir: str):
super().__init__(save_dir)
self.tb_writer = SummaryWriter(log_dir=self.save_dir)
def __init__(self, log_dir: str):
super().__init__(log_dir)
self.tb_writer = SummaryWriter(log_dir=self.log_dir)
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
"""_summary_
@@ -310,13 +310,13 @@ def _format_time(seconds):
class LocalWriter(Writer):
"""Local Writer Class"""
def __init__(self, save_dir: str, stats_to_track: ListConfig, max_log_size: int = 0):
def __init__(self, log_dir: str, stats_to_track: ListConfig, max_log_size: int = 0):
"""
Args:
stats_to_track (ListConfig): the names of stats that should be logged.
max_log size (int): max number of lines that will be logged to teminal.
"""
super().__init__(save_dir)
super().__init__(log_dir)
self.stats_to_track = [EventName[name].value for name in stats_to_track]
self.max_log_size = max_log_size
self.keys = set()
@@ -325,7 +325,7 @@ class LocalWriter(Writer):
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
if name in self.stats_to_track:
image = to8b(image)
image_path = os.path.join(self.save_dir, f"{name}.jpg")
image_path = os.path.join(self.log_dir, f"{name}.jpg")
imageio.imwrite(image_path, np.uint8(image.cpu().numpy() * 255.0))
def write_scalar(self, name: str, scalar: float, step: int) -> None: