update docstrings and random cleanups (#387)

* fixing docstrings and small nits; up to fileds

* finished adding and fixing docstrings
This commit is contained in:
evonneng authored and GitHub committed 2022-09-10 23:09:44 -07:00
1 parent bd5967461d
commit d50c71ac43
52 files changed
+846 -646

No files matched your search

+19 -18
View File
@@ -19,6 +19,7 @@ from __future__ import annotations
import functools
import logging
import os
from pathlib import Path
from typing import Any, Dict, List
@@ -45,12 +46,9 @@ def train_loop(local_rank: int, world_size: int, config: cfg.Config) -> Any:
"""Main training function that sets up and runs the trainer per process
Args:
local_rank (int): current rank of process
world_size (int): total number of gpus available
config (Config): config file specifying training regimen
Returns:
Any: TODO(): determine the return type
local_rank: current rank of process
world_size: total number of gpus available
config: config file specifying training regimen
"""
trainer = Trainer(config, local_rank, world_size)
trainer.setup()
@@ -62,9 +60,9 @@ class Trainer:
"""Trainer class
Args:
config (Config): The configuration object.
local_rank (int, optional): Local rank of the process. Defaults to 0.
world_size (int, optional): World size of the process. Defaults to 1.
config: The configuration object.
local_rank: Local rank of the process. Defaults to 0.
world_size: World size of the process. Defaults to 1.
"""
def __init__(self, config: cfg.Config, local_rank: int = 0, world_size: int = 1):
@@ -98,7 +96,7 @@ class Trainer:
"""Setup the Trainer by calling other setup functions.
Args:
test_mode (bool, optional): Whether to setup for testing. Defaults to False.
test_mode: Whether to setup for testing. Defaults to False.
"""
self.pipeline: Pipeline = self.config.pipeline.setup(
device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank
@@ -117,7 +115,7 @@ class Trainer:
assert self.pipeline.datamanager.train_input_dataset is not None, "Missing DatsetInputs"
self.visualizer_state.init_scene(
dataset=self.pipeline.datamanager.train_input_dataset, start_train=self.config.viewer.train
dataset=self.pipeline.datamanager.train_input_dataset, start_train=self.config.viewer.start_train
)
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
num_iterations = self.config.trainer.max_num_iterations
@@ -172,7 +170,7 @@ class Trainer:
"""Perform writes only during appropriate time steps
Args:
step (int): Current training step.
step: Current training step.
"""
if (
step % self.config.logging.steps_per_log == 0
@@ -185,8 +183,12 @@ class Trainer:
def _load_checkpoint(self) -> None:
"""Helper function to load pipeline and optimizer from prespecified checkpoint"""
load_dir = self.config.trainer.load_dir
load_step = self.config.trainer.load_step
if load_dir is not None and load_step is not None:
if load_dir is not None:
load_step = self.config.trainer.load_step
if load_step is None:
print("Loading latest checkpoint from load_dir")
# NOTE: this is specific to the checkpoint name format
load_step = sorted(int(x[x.find("-") + 1 : x.find(".")]) for x in os.listdir(load_dir))[-1]
load_path = load_dir / f"step-{load_step:09d}.ckpt"
assert load_path.exists(), f"Checkpoint {load_path} does not exist"
loaded_state = torch.load(load_path, map_location="cpu")
@@ -199,6 +201,8 @@ class Trainer:
else:
logging.info("No checkpoints to load, training from scratch")
logging.info("saving checkpoints to: %s", self.config.trainer.model_dir)
@check_main_thread
def _save_checkpoint(self, output_dir: Path, step: int) -> None:
"""Save the model and optimizers
@@ -226,13 +230,10 @@ class Trainer:
@profiler.time_function
def train_iteration(self, step: int) -> Dict[str, torch.Tensor]:
"""Run one iteration with a batch of inputs.
"""Run one iteration with a batch of inputs. Returns dictionary of model losses.
Args:
step: Current training step.
Returns:
Dictionary of model losses.
"""
self.optimizers.zero_grad_all()
cpu_or_cuda_str = self.device.split(":")[0]