From 38f83de64b4aacf12adf521cd79f3e1b02190c93 Mon Sep 17 00:00:00 2001 From: evonneng Date: Tue, 20 Sep 2022 14:56:48 -0700 Subject: [PATCH] removed post init method in config setup (#495) * mrege * nits --- nerfactory/configs/base.py | 73 ++++-------------------- nerfactory/configs/base_configs.py | 1 - nerfactory/engine/trainer.py | 36 +++++++----- nerfactory/utils/writer.py | 26 ++++----- nerfactory/viewer/server/viewer_utils.py | 13 +++-- scripts/run_train.py | 7 ++- tests/test_run_train.py | 7 ++- 7 files changed, 60 insertions(+), 103 deletions(-) diff --git a/nerfactory/configs/base.py b/nerfactory/configs/base.py index 8f9269cdc..e58d8ec3d 100644 --- a/nerfactory/configs/base.py +++ b/nerfactory/configs/base.py @@ -19,7 +19,7 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Type import dcargs import torch @@ -84,27 +84,6 @@ class MachineConfig(PrintableConfig): """distributed connection point (for DDP)""" -# Logging related configs -@dataclass -class TensorboardWriterConfig(InstantiateConfig): - """Tensorboard Writer config""" - - _target: Type = writer.TensorboardWriter - """target class to instantiate""" - log_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING - """[Do not set] Auto-populated absolute path to saved tensorboard events""" - - -@dataclass -class WandbWriterConfig(InstantiateConfig): - """WandDB Writer config""" - - _target: Type = writer.WandbWriter - """target class to instantiate""" - log_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING - """[Do not set] Auto-populated absolute path to saved wandb events""" - - @dataclass class LocalWriterConfig(InstantiateConfig): """Local Writer config""" @@ -146,8 +125,6 @@ class LoggingConfig(PrintableConfig): e.g. if 20, averages will be computed over past 20 occurances.""" event_writer: Literal["tb", "wandb", "none"] = "wandb" """specify which writer to use (tensorboard or wandb)""" - event_writer_config: dcargs.conf.Fixed[Optional[Union[TensorboardWriterConfig, WandbWriterConfig]]] = dcargs.MISSING - """[Do not set] Writer config based on the specified event_writer. Set automatically.""" local_writer: LocalWriterConfig = LocalWriterConfig(enable=True) """if provided, will print stats locally. if None, will disable printing""" enable_profiler: bool = True @@ -174,8 +151,6 @@ class TrainerConfig(PrintableConfig): """whether or not to use mixed precision for training""" relative_model_dir: Path = Path("nerfactory_models/") """relative path to save all checkpoints""" - model_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING - """[Do not set] Auto-populated absolute path to saved checkpoints. Set automatically.""" # optional parameters if we want to resume training load_dir: Optional[Path] = None """optionally specify a pre-trained model directory to load from""" @@ -347,8 +322,6 @@ class ViewerConfig(PrintableConfig): """whether to enable viewer""" relative_log_filename: str = "viewer_log_filename.txt" """Filename to use for the log file.""" - log_filename: dcargs.conf.Fixed[Path] = dcargs.MISSING - """[Do not set] Absolute path to the log file. Set automatically.""" start_train: bool = True """whether to immediately start training upon loading viewer if False, will just visualize dataset but you can toggle training in viewer""" @@ -401,8 +374,8 @@ class Config(PrintableConfig): """Method name. Required to set in python or via cli""" experiment_name: Optional[str] = None """Experiment name. If None, will automatically be set to dataset name""" - base_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING - """[Do not set] Experiment base directory. Set automatically.""" + timestamp: str = datetime.now().strftime("%Y-%m-%d_%H%M%S") + """Experiment timestamp.""" machine: MachineConfig = MachineConfig() logging: LoggingConfig = LoggingConfig() viewer: ViewerConfig = ViewerConfig() @@ -417,38 +390,14 @@ class Config(PrintableConfig): } ) - def __post_init__(self): - """Make paths more specific using the current timestamp.""" - self.populate_dynamic_fields() - - def populate_dynamic_fields(self, timestamp: Optional[str] = None) -> None: - """Make paths in our config more specific using a timestamp. - - Args: - timestamp: Timestamp to use, as a string. If None, defaults to the current - time in the form YYYY-MM-DD_HHMMMSS. - """ - # check the experiment and method names - assert self.method_name is not None, "Please set method name in config or via the cli" + def set_experiment_name(self) -> None: + """Dynamically set the experiment name""" if self.experiment_name is None: self.experiment_name = str(self.pipeline.datamanager.train_dataparser.data_directory).replace("/", "-") - # set the timestamp of the model logging/writer loggign paths - if timestamp is None: - timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") - self.base_dir = Path(f"outputs/{self.experiment_name}/{self.method_name}/{timestamp}") - self.trainer.model_dir = self.base_dir / self.trainer.relative_model_dir - if self.logging.event_writer == "tb": - self.logging.event_writer_config = TensorboardWriterConfig( - log_dir=self.base_dir / self.logging.relative_log_dir - ) - elif self.logging.event_writer == "wandb": - self.logging.event_writer_config = WandbWriterConfig(log_dir=self.base_dir / self.logging.relative_log_dir) - else: - self.logging.event_writer_config = None - self.viewer.log_filename = self.base_dir / self.viewer.relative_log_filename - # disable test if viewer is enabled (for speed purposes) - if self.viewer.enable: - self.trainer.steps_per_eval_batch = self.trainer.max_num_iterations - self.trainer.steps_per_eval_image = self.trainer.max_num_iterations - self.trainer.steps_per_eval_all_images = self.trainer.max_num_iterations + def get_base_dir(self) -> Path: + """Retrieve the base directory to set relative paths""" + # check the experiment and method names + assert self.method_name is not None, "Please set method name in config or via the cli" + self.set_experiment_name() + return Path(f"outputs/{self.experiment_name}/{self.method_name}/{self.timestamp}") diff --git a/nerfactory/configs/base_configs.py b/nerfactory/configs/base_configs.py index 9e0cc6a99..ea7fc7c27 100644 --- a/nerfactory/configs/base_configs.py +++ b/nerfactory/configs/base_configs.py @@ -191,7 +191,6 @@ def _make_base_config_subcommand_type() -> Type[Config]: base_configs_placeholder_timestamp = {} for name, config in base_configs.items(): base_configs_placeholder_timestamp[name] = copy.deepcopy(config) - base_configs_placeholder_timestamp[name].populate_dynamic_fields(timestamp="{timestamp}") return dcargs.extras.subcommand_type_from_defaults(base_configs_placeholder_timestamp) diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py index 4b1523971..502c5652d 100644 --- a/nerfactory/engine/trainer.py +++ b/nerfactory/engine/trainer.py @@ -95,15 +95,23 @@ class Trainer: self._start_step = 0 # optimizers self.grad_scaler = GradScaler(enabled=self.mixed_precision) - # logging/viewer variables - self.viewer_state, banner_messages = viewer_utils.setup_viewer(config.viewer) - writer.setup_event_writer(config.logging) + + self.base_dir = config.get_base_dir() + # directory to save checkpoints + self.checkpoint_dir = Path(self.base_dir / self.config.trainer.relative_model_dir) + logging.info("Saving checkpoints to: %s", self.checkpoint_dir) + # set up viewer if enabled + viewer_log_path = self.base_dir / config.viewer.relative_log_filename + self.viewer_state, banner_messages = viewer_utils.setup_viewer(config.viewer, log_filename=viewer_log_path) + self._check_viewer_warnings() + # set up writers/profilers if enabled + writer_log_path = self.base_dir / config.logging.relative_log_dir + writer.setup_event_writer(config.logging, log_dir=writer_log_path) writer.setup_local_writer( config.logging, max_iter=config.trainer.max_num_iterations, banner_messages=banner_messages ) - profiler.setup_profiler(config.logging) writer.put_config(name="config", config_dict=dataclasses.asdict(config), step=0) - self._check_viewer_warnings() + profiler.setup_profiler(config.logging) def setup(self, test_mode=False): """Setup the Trainer by calling other setup functions. @@ -197,7 +205,7 @@ class Trainer: writer.put_dict(name="Eval Images Metrics Dict (all images)", scalar_dict=metrics_dict, step=step) if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0: - self._save_checkpoint(self.config.trainer.model_dir, step) + self._save_checkpoint(step) self._write_out_storage(step) @@ -206,12 +214,15 @@ class Trainer: def _check_viewer_warnings(self) -> None: """Helper to print out any warnings regarding the way the viewer/loggers are enabled""" if self.config.viewer.enable: - if self.config.logging.event_writer_config: + if self.config.logging.event_writer != "none": string = ( "[WARNING]: Tensorboard or Wandb enabled with Viewer will slow down Viewer. " "Please set `--logging.event_writer none` for faster rendering" ) CONSOLE.print(f"[bold red]{string}") + self.config.trainer.steps_per_eval_batch = self.config.trainer.max_num_iterations + self.config.trainer.steps_per_eval_image = self.config.trainer.max_num_iterations + self.config.trainer.steps_per_eval_all_images = self.config.trainer.max_num_iterations string = "[WARNING] Disabling eval iterations since viewer is enabled." CONSOLE.print(f"[bold red]{string}") @@ -296,19 +307,16 @@ class Trainer: else: logging.info("No checkpoints to load, training from scratch") - logging.info("saving checkpoints to: %s", self.config.trainer.model_dir) - @check_main_thread - def _save_checkpoint(self, output_dir: Path, step: int) -> None: + def _save_checkpoint(self, step: int) -> None: """Save the model and optimizers Args: - output_dir: directory to save the checkpoint step: number of steps in training for given checkpoint """ - if not output_dir.exists(): - output_dir.mkdir(parents=True, exist_ok=True) - ckpt_path = output_dir / f"step-{step:09d}.ckpt" + if not self.checkpoint_dir.exists(): + self.checkpoint_dir.mkdir(parents=True, exist_ok=True) + ckpt_path = self.checkpoint_dir / f"step-{step:09d}.ckpt" if hasattr(self.pipeline, "module"): pipeline = self.pipeline.module.state_dict() # type: ignore else: diff --git a/nerfactory/utils/writer.py b/nerfactory/utils/writer.py index a32c4761c..a801f65d8 100644 --- a/nerfactory/utils/writer.py +++ b/nerfactory/utils/writer.py @@ -187,7 +187,7 @@ def setup_local_writer(config: cfg.LoggingConfig, max_iter: int, banner_messages GLOBAL_BUFFER["events"] = {} -def setup_event_writer(config: cfg.LoggingConfig) -> None: +def setup_event_writer(config: cfg.LoggingConfig, log_dir: Path) -> None: """Initialization of all event writers specified in config Args: @@ -195,20 +195,20 @@ def setup_event_writer(config: cfg.LoggingConfig) -> None: max_iter: maximum number of train iterations banner_messages: list of messages to always display at bottom of screen """ - if config.event_writer_config: - curr_writer = config.event_writer_config.setup() - EVENT_WRITERS.append(curr_writer) - logging.info("logging info to: %s", config.event_writer_config.log_dir) + if config.event_writer == "tb": + curr_writer = TensorboardWriter(log_dir=log_dir) + elif config.event_writer == "wandb": + curr_writer = WandbWriter(log_dir=log_dir) else: logging.info("disabled tensorboard/wandb event writers") + return + EVENT_WRITERS.append(curr_writer) + logging.info("logging %s events to: %s", config.event_writer, log_dir) class Writer: """Writer class""" - def __init__(self, log_dir: Optional[Path]): - self.log_dir = log_dir - @abstractmethod def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None: """method to write out image @@ -276,9 +276,8 @@ class TimeWriter: class WandbWriter(Writer): """WandDB Writer Class""" - def __init__(self, config: cfg.WandbWriterConfig): - super().__init__(config.log_dir) - wandb.init(project="nerfactory-project", dir=str(config.log_dir), reinit=True) + def __init__(self, log_dir: Path): + wandb.init(project="nerfactory-project", dir=str(log_dir), reinit=True) def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None: image = torch.permute(image, (2, 0, 1)) @@ -302,9 +301,8 @@ class WandbWriter(Writer): class TensorboardWriter(Writer): """Tensorboard Writer Class""" - def __init__(self, config: cfg.TensorboardWriterConfig): - super().__init__(config.log_dir) - self.tb_writer = SummaryWriter(log_dir=config.log_dir) + def __init__(self, log_dir: Path): + self.tb_writer = SummaryWriter(log_dir=log_dir) def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None: image = to8b(image) diff --git a/nerfactory/viewer/server/viewer_utils.py b/nerfactory/viewer/server/viewer_utils.py index fcc69770e..f41b4ed8b 100644 --- a/nerfactory/viewer/server/viewer_utils.py +++ b/nerfactory/viewer/server/viewer_utils.py @@ -44,14 +44,14 @@ from nerfactory.viewer.server.visualizer import Viewer console = Console(width=120) -def setup_viewer(config: cfg.ViewerConfig): +def setup_viewer(config: cfg.ViewerConfig, log_filename: Path): """Sets up the viewer if enabled Args: config: the configuration to instantiate viewer """ if config.enable: - viewer_state = ViewerState(config) + viewer_state = ViewerState(config, log_filename=log_filename) banner_messages = [f"Viewer at: {viewer_state.viewer_url}"] return viewer_state, banner_messages return None, None @@ -202,16 +202,17 @@ class ViewerState: config: viewer setup configuration """ - def __init__(self, config: cfg.ViewerConfig): + def __init__(self, config: cfg.ViewerConfig, log_filename: Path): self.config = config self.vis = None self.viewer_url = None + self.log_filename = log_filename if self.config.launch_bridge_server: # start the viewer bridge server zmq_port = int(self.config.zmq_url.split(":")[-1]) websocket_port = self.config.websocket_port - self.config.log_filename.parent.mkdir(exist_ok=True) - run_viewer_bridge_server_as_subprocess(zmq_port, websocket_port, log_filename=str(self.config.log_filename)) + self.log_filename.parent.mkdir(exist_ok=True) + run_viewer_bridge_server_as_subprocess(zmq_port, websocket_port, log_filename=str(self.log_filename)) # TODO(ethan): move this into the writer such that it's at the bottom # of the logging stack and easy to see and click # TODO(ethan): log the output of the viewer bridge server in a file where the training logs go @@ -255,7 +256,7 @@ class ViewerState: if False, only displays dataset until resume train is toggled """ # set the config base dir - self.vis["renderingState/config_base_dir"].write(str(self.config.log_filename.parents[0])) + self.vis["renderingState/config_base_dir"].write(str(self.log_filename.parents[0])) # clear the current scene self.vis["sceneState/sceneBounds"].delete() diff --git a/scripts/run_train.py b/scripts/run_train.py index f59c6a5cc..a373b9f69 100755 --- a/scripts/run_train.py +++ b/scripts/run_train.py @@ -194,9 +194,10 @@ def main(config: cfg.Config) -> None: console.print(config) console.rule("") - assert config.base_dir is not None - config.base_dir.mkdir(parents=True, exist_ok=True) - config_yaml_path = config.base_dir / "config.yml" + base_dir = config.get_base_dir() + assert base_dir is not None + base_dir.mkdir(parents=True, exist_ok=True) + config_yaml_path = base_dir / "config.yml" logging.info(f"Saving config to: {config_yaml_path}") config_yaml_path.write_text(yaml.dump(config), "utf8") diff --git a/tests/test_run_train.py b/tests/test_run_train.py index 7753ccc9b..a9892bac6 100644 --- a/tests/test_run_train.py +++ b/tests/test_run_train.py @@ -35,8 +35,10 @@ def set_reduced_config(config: cfg.Config): config.logging.relative_log_dir = Path("/tmp/") # reduce model factors - config.pipeline.model.num_coarse_samples = 4 - config.pipeline.model.num_importance_samples = 4 + if hasattr(config.pipeline.model, "num_coarse_samples"): + config.pipeline.model.num_coarse_samples = 4 + if hasattr(config.pipeline.model, "num_importance_samples"): + config.pipeline.model.num_importance_samples = 4 # remove viewer config.viewer.enable = False @@ -44,7 +46,6 @@ def set_reduced_config(config: cfg.Config): if config.method_name == "instant-ngp": config.pipeline.model.field_implementation = "torch" - config.populate_dynamic_fields() return config