mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 15:15:43 +08:00
Pipeline v2 (#258)
* Model class done, pipeline class still needs more visualization stuffs * Renaming the BaseDataContainer to DataloaderOutputs and adding documentation * Pushing more documentation, removing some trivial functions * Updating dict typing * Removing config dicts for collider * Passing tests * Pushing shell new dataloader * Saving work, need to fix NGP model's get_loss_dict and Pipeline * V1 of the ngp pipeline should be done. Need train script and debugging * Updating with latest instant-ngp code * Syntax updates from latest ngp graph code * Linter, removing unneeded code in new NGP graph code * Style * Should be good now. Linter is happy and the code is verbatum current instant-ngp * Everything done except novel view testing code * Fixing some nitpicks * Debugging dataloader * merged master * WOP. integrating pipeline with configs and notebook demo * removing model folder * a lot of name changing from Graph to Model * in progress... changing trainer to use pipeline... broken right now * pipeline code runs * switched to val dataset * makes run_eval.py work * updated configs for pipeline change * fixing test_run_train script * fixing black and linting * fix docs * small updates to trainer Co-authored-by: Jake Austin <jake-austin@berkeley.edu>
This commit is contained in:
1 parent
670c2a01db
commit
0b042be956
55 files changed
+8595
-8263
No files matched your search
@@ -19,24 +19,16 @@ import functools
|
||||
import logging
|
||||
import os
|
||||
import typing
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, List
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
from torch.nn.parallel import DistributedDataParallel as DDP
|
||||
from torchtyping import TensorType
|
||||
|
||||
from nerfactory.cameras.rays import RayBundle
|
||||
from nerfactory.data.dataloader import (
|
||||
EvalDataloader,
|
||||
TrainDataloader,
|
||||
setup_dataset_eval,
|
||||
setup_dataset_train,
|
||||
)
|
||||
from nerfactory.data.structs import DatasetInputs
|
||||
from nerfactory.graphs.base import Graph, setup_graph
|
||||
from nerfactory.dataloaders.structs import DatasetInputs
|
||||
from nerfactory.optimizers.optimizers import Optimizers, setup_optimizers
|
||||
from nerfactory.pipelines.base import Pipeline, setup_pipeline
|
||||
from nerfactory.utils import profiler, writer
|
||||
from nerfactory.utils.callbacks import Callback
|
||||
from nerfactory.utils.config import Config
|
||||
@@ -67,10 +59,8 @@ class Trainer:
|
||||
logging.warning("Mixed precision is disabled for CPU training.")
|
||||
# dataset variables
|
||||
self.dataset_inputs_train: DatasetInputs
|
||||
self.dataloader_train: TrainDataloader
|
||||
self.dataloader_eval: EvalDataloader
|
||||
# model variables
|
||||
self.graph: Graph
|
||||
self.pipeline: Pipeline
|
||||
self.optimizers: Optimizers
|
||||
self.start_step = 0
|
||||
# logging variables
|
||||
@@ -88,55 +78,45 @@ class Trainer:
|
||||
Args:
|
||||
test_mode (bool, optional): Whether to setup for testing. Defaults to False.
|
||||
"""
|
||||
self.dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device)
|
||||
_, self.dataloader_eval = setup_dataset_eval(self.config.data, test_mode=test_mode, device=self.device)
|
||||
self.graph = setup_graph(self.config.graph, self.dataset_inputs_train, device=self.device)
|
||||
if not isinstance(self.graph, Graph):
|
||||
raise ValueError("Graph was improperly initialized.")
|
||||
self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups())
|
||||
self.pipeline: Pipeline = setup_pipeline(self.config.pipeline, device=self.device, test_mode=test_mode)
|
||||
self.optimizers = setup_optimizers(self.config.optimizers, self.pipeline.get_param_groups())
|
||||
|
||||
self._load_checkpoint()
|
||||
|
||||
if self.world_size > 1:
|
||||
self.graph = typing.cast(Graph, DDP(self.graph, device_ids=[self.local_rank]))
|
||||
self.pipeline = typing.cast(
|
||||
Pipeline, typing.cast(Pipeline, DDP(self.pipeline, device_ids=[self.local_rank]))
|
||||
)
|
||||
dist.barrier(device_ids=[self.local_rank])
|
||||
|
||||
self.callbacks = self.graph.get_training_callbacks()
|
||||
|
||||
@classmethod
|
||||
def get_aggregated_loss(cls, loss_dict: Dict[str, torch.Tensor]) -> torch.Tensor:
|
||||
"""Returns the aggregated losses and the scalar for calling .backwards() on.
|
||||
# TODO: move this out to another file/class/etc.
|
||||
"""
|
||||
# TODO(ethan): add loss weightings here from a config
|
||||
# e.g. weighted_losses = map(lambda k: some_weight_dict[k] * loss_dict[k], loss_dict.keys())
|
||||
weighted_losses = loss_dict.values()
|
||||
return functools.reduce(torch.add, weighted_losses)
|
||||
# TODO(ethan): do this for pipeline, not pipeline.model
|
||||
self.callbacks = self.pipeline.model.get_training_callbacks()
|
||||
|
||||
def train(self) -> None:
|
||||
"""Train the model."""
|
||||
self.visualizer_state.init_scene(
|
||||
image_dataset=self.dataloader_train.image_sampler.dataset, dataset_inputs=self.dataset_inputs_train
|
||||
image_dataset=self.pipeline.dataloader.train_image_dataset,
|
||||
dataset_inputs=self.pipeline.dataloader.train_datasetinputs,
|
||||
)
|
||||
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
|
||||
num_iterations = self.config.trainer.max_num_iterations
|
||||
iter_dataloader_train = iter(self.dataloader_train)
|
||||
for step in range(self.start_step, self.start_step + num_iterations):
|
||||
with TimeWriter(writer, EventName.ITER_LOAD_TIME, step=step):
|
||||
ray_indices, batch = next(iter_dataloader_train)
|
||||
# with TimeWriter(writer, EventName.ITER_LOAD_TIME, step=step):
|
||||
# ray_indices, batch = next(iter_dataloader_train)
|
||||
|
||||
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as t:
|
||||
loss_metric_dict = self.train_iteration(ray_indices, batch, step)
|
||||
writer.put_scalar(name=EventName.RAYS_PER_SEC, scalar=ray_indices.shape[0] / t.duration, step=step)
|
||||
loss_metric_dict = self.train_iteration(step)
|
||||
train_num_rays_per_batch = self.pipeline.dataloader.train_num_rays_per_batch
|
||||
writer.put_scalar(name=EventName.RAYS_PER_SEC, scalar=train_num_rays_per_batch / t.duration, step=step)
|
||||
|
||||
if step != 0 and step % self.config.logging.steps_per_log == 0:
|
||||
writer.put_dict(name="Loss/train-loss_dict", scalar_dict=loss_metric_dict, step=step)
|
||||
writer.put_dict(name="Loss/train-loss_metrics_dict", scalar_dict=loss_metric_dict, step=step)
|
||||
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
|
||||
self._save_checkpoint(self.config.trainer.model_dir, step)
|
||||
if step % self.config.trainer.steps_per_test == 0:
|
||||
self.eval_with_dataloader(self.dataloader_eval, step=step)
|
||||
self.pipeline.get_eval_loss_dict(step=step)
|
||||
self._write_out_storage(step)
|
||||
self.visualizer_state.update_scene(step, self.graph)
|
||||
self.visualizer_state.update_scene(step, self.pipeline.model)
|
||||
|
||||
self._write_out_storage(num_iterations)
|
||||
|
||||
@@ -155,17 +135,17 @@ class Trainer:
|
||||
writer.write_out_storage()
|
||||
|
||||
def _load_checkpoint(self) -> None:
|
||||
"""Helper function to load graph and optimizer from prespecified checkpoint"""
|
||||
"""Helper function to load pipeline and optimizer from prespecified checkpoint"""
|
||||
load_config = self.config.trainer.resume_train
|
||||
if load_config.load_dir is not None and load_config.load_step is not None:
|
||||
load_path = os.path.join(load_config.load_dir, f"step-{load_config.load_step:09d}.ckpt")
|
||||
assert os.path.exists(load_path), f"Checkpoint {load_path} does not exist"
|
||||
loaded_state = torch.load(load_path, map_location="cpu")
|
||||
self.start_step = loaded_state["step"] + 1
|
||||
# load the checkpoints for graph and optimizer
|
||||
self.graph.load_graph(loaded_state)
|
||||
self.optimizers.load_optimizers(loaded_state)
|
||||
self.grad_scaler.load_state_dict(loaded_state["scaler"])
|
||||
# load the checkpoints for pipeline, optimizers, and gradient scalar
|
||||
self.pipeline.load_pipeline(loaded_state["pipeline"])
|
||||
self.optimizers.load_optimizers(loaded_state["optimizers"])
|
||||
self.grad_scaler.load_state_dict(loaded_state["scalers"])
|
||||
logging.info("done loading checkpoint from %s", load_path)
|
||||
else:
|
||||
logging.info("No checkpoints to load, training from scratch")
|
||||
@@ -181,14 +161,14 @@ class Trainer:
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
ckpt_path = os.path.join(output_dir, f"step-{step:09d}.ckpt")
|
||||
if hasattr(self.graph, "module"):
|
||||
model = self.graph.module.state_dict() # type: ignore
|
||||
if hasattr(self.pipeline, "module"):
|
||||
pipeline = self.pipeline.module.state_dict() # type: ignore
|
||||
else:
|
||||
model = self.graph.state_dict()
|
||||
pipeline = self.pipeline.state_dict()
|
||||
torch.save(
|
||||
{
|
||||
"step": step,
|
||||
"model": model,
|
||||
"pipeline": pipeline,
|
||||
"optimizers": {k: v.state_dict() for (k, v) in self.optimizers.optimizers.items()},
|
||||
"scalers": self.grad_scaler.state_dict(),
|
||||
},
|
||||
@@ -196,22 +176,19 @@ class Trainer:
|
||||
)
|
||||
|
||||
@profiler.time_function
|
||||
def train_iteration(
|
||||
self, ray_indices: TensorType["num_rays", 3], batch: Dict[str, torch.Tensor], step: int
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
def train_iteration(self, step: int) -> Dict[str, torch.Tensor]:
|
||||
"""Run one iteration with a batch of inputs.
|
||||
|
||||
Args:
|
||||
ray_indices: Contains camera, row, and col indicies for target rays.
|
||||
batch: Batch of training data.
|
||||
step: Current training step.
|
||||
|
||||
Returns:
|
||||
Dict[str, float]: Dictionary of model losses and metrics.
|
||||
Dictionary of model losses.
|
||||
"""
|
||||
self.optimizers.zero_grad_all()
|
||||
with torch.autocast(device_type=ray_indices.device.type, enabled=self.mixed_precision):
|
||||
_, loss_dict, metrics_dict = self.graph.forward(ray_indices=ray_indices, batch=batch)
|
||||
cpu_or_cuda_str = self.device.split(":")[0]
|
||||
with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision):
|
||||
_, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step)
|
||||
loss = functools.reduce(torch.add, loss_dict.values())
|
||||
self.grad_scaler.scale(loss).backward() # type: ignore
|
||||
self.optimizers.optimizer_scaler_step_all(self.grad_scaler)
|
||||
@@ -225,34 +202,3 @@ class Trainer:
|
||||
loss_dict["loss"] = loss
|
||||
loss_dict.update(metrics_dict)
|
||||
return loss_dict
|
||||
|
||||
@profiler.time_function
|
||||
def test_image(self, camera_ray_bundle: RayBundle, batch: dict, step: Optional[int] = None) -> float:
|
||||
"""Test a specific image.
|
||||
|
||||
Args:
|
||||
camera_ray_bundle: Bundle of test rays.
|
||||
batch: Batch of data.
|
||||
step: Current training step.
|
||||
|
||||
Returns:
|
||||
float: PSNR
|
||||
"""
|
||||
self.graph.eval()
|
||||
if camera_ray_bundle.camera_indices is None:
|
||||
raise ValueError("camera_ray_bundle.camera_indices is None during testing")
|
||||
image_idx = int(camera_ray_bundle.camera_indices[0, 0])
|
||||
outputs = self.graph.get_outputs_for_camera_ray_bundle(camera_ray_bundle)
|
||||
psnr = self.graph.log_test_image_outputs(image_idx, step, batch, outputs)
|
||||
self.graph.train()
|
||||
return psnr
|
||||
|
||||
def eval_with_dataloader(self, dataloader: EvalDataloader, step: Optional[int] = None) -> None:
|
||||
"""Run evaluation with a given dataloader.
|
||||
|
||||
Args:
|
||||
dataloader: Evaluation dataloader.
|
||||
step: Current training iteration.
|
||||
"""
|
||||
for camera_ray_bundle, batch in dataloader:
|
||||
self.test_image(camera_ray_bundle, batch, step=step)
|
||||
Reference in new issue
Block a user