mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 07:05:44 +08:00
refactoring configs; pulled network out of graph; created trainer heirarchy (#51)
* changed config graph structure * removed network udner graphs; added new heirarchy trainer * fixed callback in occupancy graph * lint * changed register callback to setter method
This commit is contained in:
1 parent
41073540c1
commit
91ef54963c
19 files changed
+163
-134
No files matched your search
+16
-17
@@ -17,7 +17,7 @@ Code to train model.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from typing import Callable, Dict, List
|
||||
from typing import Dict
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -30,7 +30,6 @@ from pyrad.data.dataloader import EvalDataloader, setup_dataset_eval, setup_data
|
||||
from pyrad.graphs.base import setup_graph
|
||||
from pyrad.optimizers.optimizers import setup_optimizers
|
||||
from pyrad.utils import profiler, writer
|
||||
from pyrad.utils.callbacks import update_occupancy
|
||||
from pyrad.utils.decorators import check_main_thread
|
||||
from pyrad.utils.writer import EventName, TimeWriter
|
||||
|
||||
@@ -70,13 +69,15 @@ class Trainer:
|
||||
self.graph = setup_graph(self.config.graph, dataset_inputs_train, device=self.device)
|
||||
self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups())
|
||||
|
||||
if self.config.graph.resume_train.load_dir:
|
||||
if self.config.trainer.resume_train.load_dir:
|
||||
self._load_checkpoint()
|
||||
|
||||
if self.world_size > 1:
|
||||
self.graph = DDP(self.graph, device_ids=[self.local_rank])
|
||||
dist.barrier(device_ids=[self.local_rank])
|
||||
|
||||
self.graph.register_callbacks()
|
||||
|
||||
@classmethod
|
||||
def get_aggregated_loss(cls, loss_dict: Dict[str, torch.tensor]):
|
||||
"""Returns the aggregated losses and the scalar for calling .backwards() on.
|
||||
@@ -91,21 +92,21 @@ class Trainer:
|
||||
def train(self) -> None:
|
||||
"""_summary_"""
|
||||
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
|
||||
num_iterations = self.config.graph.max_num_iterations
|
||||
num_iterations = self.config.trainer.max_num_iterations
|
||||
iter_dataloader_train = iter(self.dataloader_train)
|
||||
for step in range(self.start_step, self.start_step + num_iterations):
|
||||
with TimeWriter(writer, EventName.ITER_LOAD_TIME, step=step):
|
||||
ray_indices, batch = next(iter_dataloader_train)
|
||||
|
||||
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as t:
|
||||
loss_dict = self.train_iteration(ray_indices, batch, step, _callback=[update_occupancy])
|
||||
loss_dict = self.train_iteration(ray_indices, batch, step)
|
||||
writer.put_scalar(name=EventName.RAYS_PER_SEC, scalar=ray_indices.shape[0] / t.duration, step=step)
|
||||
|
||||
if step != 0 and step % self.config.logging.steps_per_log == 0:
|
||||
writer.put_dict(name="Loss/train-loss_dict", scalar_dict=loss_dict, step=step)
|
||||
if step != 0 and self.config.graph.steps_per_save and step % self.config.graph.steps_per_save == 0:
|
||||
self._save_checkpoint(self.config.graph.model_dir, step)
|
||||
if step % self.config.graph.steps_per_test == 0:
|
||||
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
|
||||
self._save_checkpoint(self.config.trainer.model_dir, step)
|
||||
if step % self.config.trainer.steps_per_test == 0:
|
||||
self.eval_with_dataloader(self.dataloader_eval, step=step)
|
||||
self._write_out_storage(step)
|
||||
|
||||
@@ -119,9 +120,9 @@ class Trainer:
|
||||
"""
|
||||
if (
|
||||
step % self.config.logging.steps_per_log == 0
|
||||
or (self.config.graph.steps_per_save and step % self.config.graph.steps_per_save == 0)
|
||||
or step % self.config.graph.steps_per_test == 0
|
||||
or step == self.config.graph.max_num_iterations
|
||||
or (self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0)
|
||||
or step % self.config.trainer.steps_per_test == 0
|
||||
or step == self.config.trainer.max_num_iterations
|
||||
):
|
||||
writer.write_out_storage()
|
||||
|
||||
@@ -158,9 +159,7 @@ class Trainer:
|
||||
)
|
||||
|
||||
@profiler.time_function
|
||||
def train_iteration(
|
||||
self, ray_indices: TensorType["num_rays", 3], batch: dict, step: int, _callback: List[Callable] = None
|
||||
) -> Dict[str, float]:
|
||||
def train_iteration(self, ray_indices: TensorType["num_rays", 3], batch: dict, step: int) -> Dict[str, float]:
|
||||
"""Run one iteration with a batch of inputs.
|
||||
|
||||
Args:
|
||||
@@ -177,9 +176,9 @@ class Trainer:
|
||||
loss.backward()
|
||||
self.optimizers.optimizer_step_all()
|
||||
self.optimizers.scheduler_step_all(step)
|
||||
if _callback:
|
||||
for _func in _callback:
|
||||
_func(self.graph)
|
||||
if self.graph.callbacks:
|
||||
for _func in self.graph.callbacks:
|
||||
_func.after_step(step)
|
||||
return loss_dict
|
||||
|
||||
@profiler.time_function
|
||||
|
||||
@@ -38,9 +38,9 @@ class Field(nn.Module):
|
||||
frustums=Frustums(
|
||||
origins=positions,
|
||||
directions=torch.ones_like(positions),
|
||||
frustum_starts=torch.zeros((*positions.shape[:-1], 1)),
|
||||
frustum_ends=torch.zeros((*positions.shape[:-1], 1)),
|
||||
pixel_area=torch.ones((*positions.shape[:-1], 1)),
|
||||
frustum_starts=torch.zeros_like((positions[..., :1])),
|
||||
frustum_ends=torch.zeros_like((positions[..., :1])),
|
||||
pixel_area=torch.ones_like((positions[..., :1])),
|
||||
)
|
||||
)
|
||||
density, _ = self.get_density(point_samples)
|
||||
|
||||
@@ -28,7 +28,7 @@ class OccupancyGrid(nn.Module):
|
||||
"""Module to keep track of the density and occupancy."""
|
||||
|
||||
def __init__(
|
||||
self, num_cascades: int = 1, resolution: int = 128, aabb=None, density_fn=None, update_every_num_iters: int = 16
|
||||
self, num_cascades: int = 1, resolution: int = 128, aabb=None, update_every_num_iters: int = 16
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert is_not_none(aabb), "The axis-aligned bounding box aabb is not defined!"
|
||||
@@ -42,9 +42,7 @@ class OccupancyGrid(nn.Module):
|
||||
self.register_buffer("occupancy_grid", occupancy_grid)
|
||||
self.mean_density = 0.0
|
||||
|
||||
self.density_fn = density_fn
|
||||
self.update_every_num_iters = update_every_num_iters
|
||||
self.iteration_count = 0
|
||||
|
||||
def reset(self):
|
||||
"""Zero out the occupancy grid."""
|
||||
@@ -67,7 +65,6 @@ class OccupancyGrid(nn.Module):
|
||||
"""
|
||||
# TODO(ethan): remove this pylint disable
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
device = self.occupancy_grid.device
|
||||
|
||||
assert self.num_cascades == 1, "Still need to implement code for multiple cascades."
|
||||
|
||||
@@ -40,7 +40,7 @@ def setup_graph(config: DictConfig, dataset_inputs: DatasetInputs, device: str)
|
||||
Args:
|
||||
dataset_inputs (DatasetInputs): The inputs which will be used to define the camera parameters.
|
||||
"""
|
||||
graph = instantiate_from_dict_config(config.network, **dataset_inputs.as_dict())
|
||||
graph = instantiate_from_dict_config(config, **dataset_inputs.as_dict())
|
||||
graph.to(device)
|
||||
return graph
|
||||
|
||||
@@ -99,6 +99,11 @@ class Graph(AbstractGraph):
|
||||
self.populate_collider()
|
||||
self.populate_fields()
|
||||
self.populate_misc_modules() # populate the modules
|
||||
self.callbacks = None
|
||||
|
||||
def register_callbacks(self): # pylint:disable=no-self-use
|
||||
"""Option to register callback for training functions"""
|
||||
self.callbacks = []
|
||||
|
||||
def populate_collider(self):
|
||||
"""Set the scene bounds collider to use."""
|
||||
|
||||
@@ -33,6 +33,7 @@ from pyrad.graphs.modules.ray_sampler import PDFSampler, UniformSampler
|
||||
from pyrad.utils import colors
|
||||
from pyrad.cameras.rays import RayBundle
|
||||
from pyrad.utils import visualization, writer
|
||||
from pyrad.utils.callbacks import Callback
|
||||
|
||||
|
||||
class NGPGraph(Graph):
|
||||
@@ -45,6 +46,16 @@ class NGPGraph(Graph):
|
||||
self.field = None
|
||||
super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs)
|
||||
|
||||
def register_callbacks(self) -> None:
|
||||
"""defining callbacks to run after every training iteration"""
|
||||
self.callbacks = [
|
||||
Callback(
|
||||
self.occupancy_grid.update_every_num_iters,
|
||||
self.occupancy_grid.update_occupancy_grid,
|
||||
density_fn=self.field.density_fn,
|
||||
)
|
||||
]
|
||||
|
||||
def populate_fields(self):
|
||||
"""Set the fields."""
|
||||
# torch or tiny-cuda-nn version
|
||||
|
||||
+26
-15
@@ -16,20 +16,31 @@
|
||||
Callback functions for training iterations
|
||||
"""
|
||||
|
||||
from pyrad.graphs.base import Graph
|
||||
from pyrad.fields.occupancy_fields.occupancy_grid import OccupancyGrid
|
||||
from typing import Callable
|
||||
|
||||
|
||||
def update_occupancy(graph: Graph):
|
||||
"""Update the occupancy grid depending on iteration count after every train step"""
|
||||
if not hasattr(graph, "occupancy_pointers"):
|
||||
occupancy_grids = []
|
||||
for _, module in graph.__dict__["_modules"].items():
|
||||
if isinstance(module, OccupancyGrid):
|
||||
occupancy_grids.append(module)
|
||||
graph.occupancy_pointers = occupancy_grids
|
||||
for grid in graph.occupancy_pointers:
|
||||
grid.iteration_count += 1
|
||||
if grid.iteration_count % grid.update_every_num_iters == 0:
|
||||
if grid.density_fn is not None:
|
||||
grid.update_occupancy_grid(grid.density_fn)
|
||||
class Callback:
|
||||
"""Callback Object"""
|
||||
|
||||
def __init__(self, update_every_num_iters: int, func: Callable, *args, **kwargs):
|
||||
self.update_every_num_iters = update_every_num_iters
|
||||
self.func = func
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
def before_step(self, step: int):
|
||||
"""callback to run before training step
|
||||
|
||||
Args:
|
||||
step (int): current iteration step
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def after_step(self, step: int):
|
||||
"""callback to run after training step
|
||||
|
||||
Args:
|
||||
step (int): current iteration step
|
||||
"""
|
||||
if step % self.update_every_num_iters == 0:
|
||||
self.func(*self.args, **self.kwargs)
|
||||
+14
-14
@@ -141,13 +141,13 @@ def setup_event_writers(config: DictConfig) -> None:
|
||||
writer_class = getattr(sys.modules[__name__], writer_type)
|
||||
writer_config = logging_configs[writer_type]
|
||||
if writer_type == "LocalWriter":
|
||||
curr_writer = writer_class(writer_config.save_dir, writer_config.stats_to_track, writer_config.max_log_size)
|
||||
curr_writer = writer_class(writer_config.log_dir, writer_config.stats_to_track, writer_config.max_log_size)
|
||||
else:
|
||||
curr_writer = writer_class(writer_config.save_dir)
|
||||
curr_writer = writer_class(writer_config.log_dir)
|
||||
EVENT_WRITERS.append(curr_writer)
|
||||
|
||||
## configure all the global buffer basic information
|
||||
GLOBAL_BUFFER["max_iter"] = config.graph.max_num_iterations
|
||||
GLOBAL_BUFFER["max_iter"] = config.trainer.max_num_iterations
|
||||
GLOBAL_BUFFER["max_buffer_size"] = config.logging.max_buffer_size
|
||||
GLOBAL_BUFFER["steps_per_log"] = config.logging.steps_per_log
|
||||
GLOBAL_BUFFER["events"] = {}
|
||||
@@ -156,8 +156,8 @@ def setup_event_writers(config: DictConfig) -> None:
|
||||
class Writer:
|
||||
"""Writer class"""
|
||||
|
||||
def __init__(self, save_dir: str):
|
||||
self.save_dir = save_dir
|
||||
def __init__(self, log_dir: str):
|
||||
self.log_dir = log_dir
|
||||
|
||||
@abstractmethod
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
@@ -225,9 +225,9 @@ class TimeWriter:
|
||||
class WandbWriter(Writer):
|
||||
"""WandDB Writer Class"""
|
||||
|
||||
def __init__(self, save_dir: str):
|
||||
super().__init__(save_dir)
|
||||
wandb.init(dir=save_dir)
|
||||
def __init__(self, log_dir: str):
|
||||
super().__init__(log_dir)
|
||||
wandb.init(dir=log_dir)
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
"""_summary_
|
||||
@@ -253,9 +253,9 @@ class WandbWriter(Writer):
|
||||
class TensorboardWriter(Writer):
|
||||
"""Tensorboard Writer Class"""
|
||||
|
||||
def __init__(self, save_dir: str):
|
||||
super().__init__(save_dir)
|
||||
self.tb_writer = SummaryWriter(log_dir=self.save_dir)
|
||||
def __init__(self, log_dir: str):
|
||||
super().__init__(log_dir)
|
||||
self.tb_writer = SummaryWriter(log_dir=self.log_dir)
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
"""_summary_
|
||||
@@ -310,13 +310,13 @@ def _format_time(seconds):
|
||||
class LocalWriter(Writer):
|
||||
"""Local Writer Class"""
|
||||
|
||||
def __init__(self, save_dir: str, stats_to_track: ListConfig, max_log_size: int = 0):
|
||||
def __init__(self, log_dir: str, stats_to_track: ListConfig, max_log_size: int = 0):
|
||||
"""
|
||||
Args:
|
||||
stats_to_track (ListConfig): the names of stats that should be logged.
|
||||
max_log size (int): max number of lines that will be logged to teminal.
|
||||
"""
|
||||
super().__init__(save_dir)
|
||||
super().__init__(log_dir)
|
||||
self.stats_to_track = [EventName[name].value for name in stats_to_track]
|
||||
self.max_log_size = max_log_size
|
||||
self.keys = set()
|
||||
@@ -325,7 +325,7 @@ class LocalWriter(Writer):
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
if name in self.stats_to_track:
|
||||
image = to8b(image)
|
||||
image_path = os.path.join(self.save_dir, f"{name}.jpg")
|
||||
image_path = os.path.join(self.log_dir, f"{name}.jpg")
|
||||
imageio.imwrite(image_path, np.uint8(image.cpu().numpy() * 255.0))
|
||||
|
||||
def write_scalar(self, name: str, scalar: float, step: int) -> None:
|
||||
|
||||
Reference in new issue
Block a user