From 1d142631277cb1eab910a99febeaa3b8c87340f5 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Tue, 20 Sep 2022 20:07:35 -0700 Subject: [PATCH] eval progress bar with more metrics (#499) updated eval --- nerfactory/configs/base.py | 10 +---- nerfactory/configs/base_configs.py | 18 ++++----- nerfactory/datamanagers/base.py | 61 +++++++++++------------------ nerfactory/engine/trainer.py | 18 +-------- nerfactory/models/instant_ngp.py | 6 +++ nerfactory/pipelines/base.py | 62 ++++++++++++++++++++---------- nerfactory/utils/misc.py | 16 ++++++++ nerfactory/utils/writer.py | 3 +- scripts/run_eval.py | 54 ++------------------------ tests/test_run_train.py | 7 +--- 10 files changed, 102 insertions(+), 153 deletions(-) diff --git a/nerfactory/configs/base.py b/nerfactory/configs/base.py index 8a9ce33d6..03e20fada 100644 --- a/nerfactory/configs/base.py +++ b/nerfactory/configs/base.py @@ -196,20 +196,14 @@ class VanillaDataManagerConfig(InstantiateConfig): train/eval data at each iteration """ - # Note: eval_dataparser is annotated with Fixed[] to prevent dcargs from trying to - # convert Optional[InstantiateConfig] into subcommands for choosing between None and - # InstantiateConfig. - _target: Type = VanillaDataManager """target class to instantiate""" - train_dataparser: AnnotatedDataParserUnion = BlenderDataParserConfig() + dataparser: AnnotatedDataParserUnion = BlenderDataParserConfig() """specifies the dataparser used to unpack the data""" train_num_rays_per_batch: int = 1024 """number of rays per batch to use per training iteration""" train_num_images_to_sample_from: int = -1 """number of images to sample during training iteration""" - eval_dataparser: dcargs.conf.Fixed[Optional[InstantiateConfig]] = None - """optionally specify different dataparser to use during eval; if None, uses train_dataparser""" eval_num_rays_per_batch: int = 1024 """number of rays per batch to use per eval iteration""" eval_num_images_to_sample_from: int = -1 @@ -409,7 +403,7 @@ class Config(PrintableConfig): def set_experiment_name(self) -> None: """Dynamically set the experiment name""" if self.experiment_name is None: - self.experiment_name = str(self.pipeline.datamanager.train_dataparser.data_directory).replace("/", "-") + self.experiment_name = str(self.pipeline.datamanager.dataparser.data_directory).replace("/", "-") def get_base_dir(self) -> Path: """Retrieve the base directory to set relative paths""" diff --git a/nerfactory/configs/base_configs.py b/nerfactory/configs/base_configs.py index f517b0aca..cf5934c8a 100644 --- a/nerfactory/configs/base_configs.py +++ b/nerfactory/configs/base_configs.py @@ -51,7 +51,7 @@ base_configs["compound"] = Config( method_name="compound", trainer=TrainerConfig(mixed_precision=True), pipeline=PipelineConfig( - datamanager=VanillaDataManagerConfig(train_dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), + datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), model=CompoundModelConfig(eval_num_rays_per_chunk=8192), ), optimizers={ @@ -68,7 +68,7 @@ base_configs["instant-ngp"] = Config( method_name="instant-ngp", trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, mixed_precision=True), pipeline=PipelineConfig( - datamanager=VanillaDataManagerConfig(train_dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), + datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192), ), optimizers={ @@ -85,9 +85,7 @@ base_configs["mipnerf-360"] = Config( method_name="mipnerf-360", trainer=TrainerConfig(steps_per_eval_batch=200), pipeline=PipelineConfig( - datamanager=VanillaDataManagerConfig( - train_dataparser=MipNerf360DataParserConfig(), train_num_rays_per_batch=8192 - ), + datamanager=VanillaDataManagerConfig(dataparser=MipNerf360DataParserConfig(), train_num_rays_per_batch=8192), model=VanillaModelConfig( _target=MipNerf360Model, collider_params={"near_plane": 0.5, "far_plane": 20.0}, @@ -108,7 +106,7 @@ base_configs["mipnerf-360"] = Config( base_configs["mipnerf"] = Config( method_name="mipnerf", pipeline=PipelineConfig( - datamanager=VanillaDataManagerConfig(train_dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), + datamanager=VanillaDataManagerConfig(dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=8192), model=VanillaModelConfig( _target=MipNerfModel, loss_coefficients={"rgb_loss_coarse": 0.1, "rgb_loss_fine": 1.0}, @@ -129,7 +127,7 @@ base_configs["nerfw"] = Config( method_name="nerfw", pipeline=PipelineConfig( datamanager=VanillaDataManagerConfig( - train_dataparser=FriendsDataParserConfig(), + dataparser=FriendsDataParserConfig(), ), model=NerfWModelConfig(), ), @@ -146,7 +144,7 @@ base_configs["semantic-nerf"] = Config( method_name="semantic-nerf", pipeline=PipelineConfig( datamanager=VanillaDataManagerConfig( - train_dataparser=FriendsDataParserConfig(), + dataparser=FriendsDataParserConfig(), ), model=VanillaModelConfig( _target=SemanticNerfModel, @@ -167,7 +165,7 @@ base_configs["vanilla-nerf"] = Config( method_name="vanilla-nerf", pipeline=PipelineConfig( datamanager=VanillaDataManagerConfig( - train_dataparser=BlenderDataParserConfig(), + dataparser=BlenderDataParserConfig(), ), model=VanillaModelConfig(_target=NeRFModel), ), @@ -184,7 +182,7 @@ base_configs["tensorf"] = Config( trainer=TrainerConfig(mixed_precision=True), pipeline=PipelineConfig( datamanager=VanillaDataManagerConfig( - train_dataparser=BlenderDataParserConfig(), + dataparser=BlenderDataParserConfig(), ), model=TensoRFModelConfig(), ), diff --git a/nerfactory/datamanagers/base.py b/nerfactory/datamanagers/base.py index 84d7f7bf6..a2463abd9 100644 --- a/nerfactory/datamanagers/base.py +++ b/nerfactory/datamanagers/base.py @@ -17,7 +17,6 @@ Data loader. """ from __future__ import annotations -import logging from abc import abstractmethod from typing import Dict, List, Optional, Tuple, Union @@ -232,13 +231,9 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self.local_rank = local_rank self.sampler = None - if config.eval_dataparser is None: - logging.info("No eval dataset specified so using train dataset for eval.") - config.eval_dataparser = config.train_dataparser - - self.train_input_dataset = InputDataset(config.train_dataparser.setup().get_dataset_inputs(split="train")) + self.train_input_dataset = InputDataset(config.dataparser.setup().get_dataset_inputs(split="train")) self.eval_input_dataset = InputDataset( - config.eval_dataparser.setup().get_dataset_inputs(split="val" if not test_mode else "test") + config.dataparser.setup().get_dataset_inputs(split="val" if not test_mode else "test") ) super().__init__() @@ -249,22 +244,16 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method sampler = DistributedSampler( self.train_input_dataset, num_replicas=self.world_size, rank=self.local_rank, shuffle=True, seed=42 ) - self.train_image_dataloader = CacheImageDataloader( - self.train_input_dataset, - num_images_to_sample_from=self.config.train_num_images_to_sample_from, - device=self.device, - num_workers=self.world_size * 4, - pin_memory=True, - sampler=sampler, - ) # TODO(ethan): pass this in else: - self.train_image_dataloader = CacheImageDataloader( - self.train_input_dataset, - num_images_to_sample_from=self.config.train_num_images_to_sample_from, - device=self.device, - num_workers=self.world_size * 4, - pin_memory=True, - ) + sampler = None + self.train_image_dataloader = CacheImageDataloader( + self.train_input_dataset, + num_images_to_sample_from=self.config.train_num_images_to_sample_from, + device=self.device, + num_workers=self.world_size * 4, + pin_memory=True, + sampler=sampler, + ) self.iter_train_image_dataloader = iter(self.train_image_dataloader) self.train_pixel_sampler = PixelSampler(self.config.train_num_rays_per_batch) self.train_ray_generator = RayGenerator(self.train_input_dataset.dataset_inputs.cameras.to(self.device)) @@ -276,22 +265,16 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method sampler = DistributedSampler( self.eval_input_dataset, num_replicas=self.world_size, rank=self.local_rank, shuffle=True, seed=42 ) - self.eval_image_dataloader = CacheImageDataloader( - self.eval_input_dataset, - num_images_to_sample_from=self.config.eval_num_images_to_sample_from, - device=self.device, - num_workers=self.world_size * 4, - pin_memory=True, - sampler=sampler, - ) # TODO(ethan): pass this in else: - self.eval_image_dataloader = CacheImageDataloader( - self.eval_input_dataset, - num_images_to_sample_from=self.config.eval_num_images_to_sample_from, - device=self.device, - num_workers=self.world_size * 4, - pin_memory=True, - ) + sampler = None + self.eval_image_dataloader = CacheImageDataloader( + self.eval_input_dataset, + num_images_to_sample_from=self.config.eval_num_images_to_sample_from, + device=self.device, + num_workers=self.world_size * 4, + pin_memory=True, + sampler=sampler, + ) self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) self.eval_pixel_sampler = PixelSampler(self.config.eval_num_rays_per_batch) self.eval_ray_generator = RayGenerator(self.eval_input_dataset.dataset_inputs.cameras.to(self.device)) @@ -299,13 +282,13 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader( input_dataset=self.eval_input_dataset, device=self.device, - num_workers=0 if self.world_size == 1 else self.world_size * 4, + num_workers=self.world_size * 4, ) self.eval_dataloader = RandIndicesEvalDataloader( input_dataset=self.eval_input_dataset, image_indices=self.config.eval_image_indices, device=self.device, - num_workers=0 if self.world_size == 1 else self.world_size * 4, + num_workers=self.world_size * 4, ) # TODO: eval dataloader should be separate from train diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py index 502c5652d..457b39f46 100644 --- a/nerfactory/engine/trainer.py +++ b/nerfactory/engine/trainer.py @@ -207,9 +207,7 @@ class Trainer: if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0: self._save_checkpoint(step) - self._write_out_storage(step) - - self._write_out_storage(num_iterations) + writer.write_out_storage() def _check_viewer_warnings(self) -> None: """Helper to print out any warnings regarding the way the viewer/loggers are enabled""" @@ -272,20 +270,6 @@ class Trainer: avg_over_steps=True, ) - def _write_out_storage(self, step: int) -> None: - """Perform writes only during appropriate time steps - - Args: - step: Current training step. - """ - if ( - step % self.config.logging.steps_per_log == 0 - or step % self.config.trainer.steps_per_eval_batch == 0 - or step % self.config.trainer.steps_per_eval_image == 0 - or step % self.config.trainer.steps_per_eval_all_images == 0 - ): - writer.write_out_storage() - def _load_checkpoint(self) -> None: """Helper function to load pipeline and optimizer from prespecified checkpoint""" load_dir = self.config.trainer.load_dir diff --git a/nerfactory/models/instant_ngp.py b/nerfactory/models/instant_ngp.py index 4c7eee8f1..9f2cb7df6 100644 --- a/nerfactory/models/instant_ngp.py +++ b/nerfactory/models/instant_ngp.py @@ -177,6 +177,12 @@ class NGPModel(Model): } return outputs + def get_metrics_dict(self, outputs, batch): + image = batch["image"].to(self.device) + metrics_dict = {} + metrics_dict["psnr"] = self.psnr(outputs["rgb"], image) + return metrics_dict + def get_loss_dict(self, outputs, batch, metrics_dict=None): image = batch["image"].to(self.device) mask = outputs["alive_ray_mask"] diff --git a/nerfactory/pipelines/base.py b/nerfactory/pipelines/base.py index c5afa5314..cd7ae2638 100644 --- a/nerfactory/pipelines/base.py +++ b/nerfactory/pipelines/base.py @@ -19,10 +19,18 @@ from __future__ import annotations import typing from abc import abstractmethod -from typing import Any, Dict, List, Optional +from time import time +from typing import Any, Dict, List, Optional, Union, cast import torch import torch.distributed as dist +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + TextColumn, + TimeElapsedColumn, +) from torch import nn from torch.nn import Parameter from torch.nn.parallel import DistributedDataParallel as DDP @@ -34,13 +42,13 @@ from nerfactory.utils import profiler from nerfactory.utils.callbacks import TrainingCallback, TrainingCallbackAttributes -def module_wrapper(module: nn.Module) -> nn.Module: +def module_wrapper(ddp_or_model: Union[DDP, Model]) -> Model: """ If DDP, then return the .module. Otherwise, return the model. """ - if isinstance(module, DDP): - return module.module - return module + if isinstance(ddp_or_model, DDP): + return cast(Model, ddp_or_model.module) + return ddp_or_model class Pipeline(nn.Module): @@ -124,9 +132,6 @@ class Pipeline(nn.Module): Args: step: current iteration step to update sampler if using DDP (distributed) """ - if self.world_size > 1 and step: - assert self.datamanager.train_sampler is not None - self.datamanager.train_sampler.set_epoch(step) ray_bundle, batch = self.datamanager.next_train(step) model_outputs = self.model(ray_bundle, batch) metrics_dict = self.model.get_metrics_dict(model_outputs, batch) @@ -144,9 +149,6 @@ class Pipeline(nn.Module): step: current iteration step """ self.eval() - if self.world_size > 1: - assert self.datamanager.eval_sampler is not None - self.datamanager.eval_sampler.set_epoch(step) ray_bundle, batch = self.datamanager.next_eval(step) model_outputs = self.model(ray_bundle, batch) loss_dict = self.model.get_loss_dict(model_outputs, batch) @@ -177,14 +179,36 @@ class Pipeline(nn.Module): @abstractmethod @profiler.time_function def get_average_eval_image_metrics(self, step: Optional[int] = None): - """Iterate over all the images in the eval dataset and get the average.""" + """Iterate over all the images in the eval dataset and get the average. + + Returns: + metrics_dict: dictionary of metrics + """ self.eval() metrics_dict_list = [] - # TODO: add something like tqdm but so that it doesn't interfere with logging - for camera_ray_bundle, batch in self.datamanager.fixed_indices_eval_dataloader: - outputs = module_wrapper(self.model).get_outputs_for_camera_ray_bundle(camera_ray_bundle) - metrics_dict, _ = module_wrapper(self.model).get_image_metrics_and_images(outputs, batch) - metrics_dict_list.append(metrics_dict) + num_images = len(self.datamanager.fixed_indices_eval_dataloader) + with Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TimeElapsedColumn(), + MofNCompleteColumn(), + transient=True, + ) as progress: + task = progress.add_task("[green]Evaluating all eval images...", total=num_images) + for camera_ray_bundle, batch in self.datamanager.fixed_indices_eval_dataloader: + # time this the following line + inner_start = time() + height, width = camera_ray_bundle.shape + num_rays = height * width + outputs = module_wrapper(self.model).get_outputs_for_camera_ray_bundle(camera_ray_bundle) + metrics_dict, _ = module_wrapper(self.model).get_image_metrics_and_images(outputs, batch) + assert "num_rays_per_sec" not in metrics_dict + metrics_dict["num_rays_per_sec"] = num_rays / (time() - inner_start) + fps_str = f"fps_at_{height}x{width}" + assert fps_str not in metrics_dict + metrics_dict[fps_str] = metrics_dict["num_rays_per_sec"] / (height * width) + metrics_dict_list.append(metrics_dict) + progress.advance(task) # average the metrics list metrics_dict = {} for key in metrics_dict_list[0].keys(): @@ -194,10 +218,6 @@ class Pipeline(nn.Module): self.train() return metrics_dict - @abstractmethod - def log_test_image_outputs(self) -> None: - """Log the test image outputs""" - def load_pipeline(self, loaded_state: Dict[str, Any]) -> None: """Load the checkpoint from the given path diff --git a/nerfactory/utils/misc.py b/nerfactory/utils/misc.py index f18df1706..5abd850cd 100644 --- a/nerfactory/utils/misc.py +++ b/nerfactory/utils/misc.py @@ -163,4 +163,20 @@ def scale_dict(dictionary: Dict[Any, Any], coefficients: Dict[str, float]) -> Di def step_check(step, step_size, run_at_zero=False) -> bool: """Returns true based on current step and step interval.""" + if step_size == 0: + return False return (run_at_zero or step != 0) and step % step_size == 0 + + +def _update_avg(prev_avg: float, new_val: float, step: int) -> float: + """helper to calculate the running average + + Args: + prev_avg (float): previous average value + new_val (float): new value to update the average with + step (int): current step number + + Returns: + float: new updated average + """ + return (step * prev_avg + new_val) / (step + 1) diff --git a/nerfactory/utils/writer.py b/nerfactory/utils/writer.py index a801f65d8..999b38597 100644 --- a/nerfactory/utils/writer.py +++ b/nerfactory/utils/writer.py @@ -376,7 +376,8 @@ class LocalWriter: Args: step: current train step """ - if step > 0: + valid_step = step > 0 and step % GLOBAL_BUFFER["steps_per_log"] == 0 + if valid_step: if not self.has_printed and self.config.max_log_size: logging.info( "\x1b[33;20mPrinting max of %d lines. Set flag `--logging.local-writer.max-log-size=0` " diff --git a/scripts/run_eval.py b/scripts/run_eval.py index 920ef90bf..f5df81c84 100755 --- a/scripts/run_eval.py +++ b/scripts/run_eval.py @@ -10,7 +10,7 @@ import logging import os from dataclasses import dataclass from pathlib import Path -from typing import Dict, Literal, Tuple, Union +from typing import Literal, Tuple, Union import dcargs import mediapy as media @@ -20,33 +20,16 @@ from rich.console import Console from rich.progress import track from typing_extensions import assert_never -# pylint: disable=unused-import from nerfactory.cameras.camera_paths import get_path_from_json, get_spiral_path from nerfactory.cameras.cameras import Cameras from nerfactory.configs import base as cfg from nerfactory.pipelines.base import Pipeline -from nerfactory.utils.misc import human_format -from nerfactory.utils.writer import TimeWriter console = Console(width=120) logging.basicConfig(format="[%(filename)s:%(lineno)d] %(message)s", level=logging.INFO) -def _update_avg(prev_avg: float, new_val: float, step: int) -> float: - """helper to calculate the running average - - Args: - prev_avg (float): previous average value - new_val (float): new value to update the average with - step (int): current step number - - Returns: - float: new updated average - """ - return (step * prev_avg + new_val) / (step + 1) - - def _load_checkpoint(config: cfg.TrainerConfig, pipeline: Pipeline) -> Path: """Helper function to load checkpointed pipeline @@ -69,31 +52,6 @@ def _load_checkpoint(config: cfg.TrainerConfig, pipeline: Pipeline) -> Path: return load_path -def _render_stats_dict(pipeline: Pipeline) -> Dict[str, float]: - """Helper function to evaluate the pipeline on a DataManager. - - Args: - pipeline (Pipeline): Pipeline to evaluate - - Returns: - dict: returns the average psnr and average rays per second - """ - avg_psnr = 0 - avg_rays_per_sec = 0 - avg_fps = 0 - for step, (camera_ray_bundle, batch) in track(enumerate(pipeline.datamanager.eval_dataloader)): - with TimeWriter(writer=None, name=None, write=False) as t: - with torch.no_grad(): - image_idx = int(camera_ray_bundle.camera_indices[0, 0]) - outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) - metrics_dict = pipeline.model.log_test_image_outputs(image_idx, step, batch, outputs) - psnr = metrics_dict["psnr"] - avg_rays_per_sec = _update_avg(avg_rays_per_sec, camera_ray_bundle.origins.shape[0] / t.duration, step) - avg_psnr = _update_avg(avg_psnr, psnr, step) - avg_fps = _update_avg(avg_fps, 1 / t.duration, step) - return {"avg psnr": avg_psnr, "avg rays per sec": avg_rays_per_sec, "avg fps": avg_fps} - - def _render_trajectory_video( pipeline: Pipeline, cameras: Cameras, @@ -173,20 +131,14 @@ class ComputePSNR: config, pipeline, checkpoint_path = _eval_setup(self.load_config) assert self.output_path.suffix == ".json" - stats_dict = _render_stats_dict(pipeline) - avg_psnr = stats_dict["avg psnr"] - avg_rays_per_sec = stats_dict["avg rays per sec"] - avg_fps = stats_dict["avg fps"] - console.print(f"Avg. PSNR: {avg_psnr:0.4f}") - console.print(f"Avg. Rays / Sec: {human_format(avg_rays_per_sec)}") - console.print(f"Avg. FPS: {avg_fps:0.4f}") + metrics_dict = pipeline.get_average_eval_image_metrics() # save output to some file self.output_path.parent.mkdir(parents=True, exist_ok=True) benchmark_info = { "experiment_name": config.experiment_name, "method_name": config.method_name, "checkpoint": str(checkpoint_path), - "results": stats_dict, + "results": metrics_dict, } self.output_path.write_text(json.dumps(benchmark_info, indent=2), "utf8") console.print(f"Saved results to: {self.output_path}") diff --git a/tests/test_run_train.py b/tests/test_run_train.py index a9892bac6..f76fa3f09 100644 --- a/tests/test_run_train.py +++ b/tests/test_run_train.py @@ -21,14 +21,9 @@ def set_reduced_config(config: cfg.Config): config.machine.num_gpus = 0 config.trainer.max_num_iterations = 2 # reduce dataset factors; set dataset to test - config.pipeline.datamanager.train_dataparser = cfg.BlenderDataParserConfig( - data_directory=Path("tests/data/lego_test") - ) + config.pipeline.datamanager.dataparser = cfg.BlenderDataParserConfig(data_directory=Path("tests/data/lego_test")) config.pipeline.datamanager.train_num_images_to_sample_from = 1 config.pipeline.datamanager.train_num_rays_per_batch = 4 - config.pipeline.datamanager.eval_dataparser = cfg.BlenderDataParserConfig( - data_directory=Path("tests/data/lego_test") - ) # use tensorboard logging instead of wandb config.logging.event_writer = "tb"