From 62d4218421ef68bd7ec840c009f223757aa2f7dd Mon Sep 17 00:00:00 2001 From: evonneng Date: Fri, 15 Apr 2022 21:53:35 +0000 Subject: [PATCH] updated logic within Trainer class to allow for multiprocess training --- mattport/nerf/trainer.py | 19 ++++++++++++------- scripts/run_train_nerf.py | 7 +++++-- tests/nerf/test_trainer.py | 15 --------------- 3 files changed, 17 insertions(+), 24 deletions(-) delete mode 100644 tests/nerf/test_trainer.py diff --git a/mattport/nerf/trainer.py b/mattport/nerf/trainer.py index 549934573..0b900d6c8 100644 --- a/mattport/nerf/trainer.py +++ b/mattport/nerf/trainer.py @@ -1,28 +1,29 @@ """ Code to train model. """ -import os - -import numpy as np -import torch import torch.distributed as dist -import torch.multiprocessing as mp -from omegaconf import DictConfig +from torch.nn.parallel import DistributedDataParallel as DDP from mattport.nerf.dataset.build import build_dataset +from mattport.nerf.graph import Graph from mattport.nerf.optimizer import Optimizer class Trainer: """Training class""" - def __init__(self, config: dict): + def __init__(self, local_rank: int, world_size: int, config: dict): + self.local_rank = local_rank + self.world_size = world_size self.config = config self.train_dataset = None self.test_dataset = None self.graph = None self.optimizer = None + self.device = f"cuda:{local_rank}" + self.is_main_thread = self.local_rank % self.world_size == 0 + def setup_dataset(self): """_summary_""" self.train_dataset = build_dataset(self.config.dataset) @@ -30,6 +31,10 @@ class Trainer: def setup_graph(self): """_summary_""" + self.graph = Graph(self.config).to(self.device) + if self.world_size > 1: + self.graph = DDP(self.graph, device_ids=[self.local_rank]) + dist.barrier(device_ids=[self.local_rank]) raise NotImplementedError def setup_optimizer(self): diff --git a/scripts/run_train_nerf.py b/scripts/run_train_nerf.py index 90a1ab246..54d37f099 100644 --- a/scripts/run_train_nerf.py +++ b/scripts/run_train_nerf.py @@ -112,8 +112,11 @@ def _train(local_rank: int, world_size: int, config: DictConfig) -> Any: Returns: Any: TODO(): determine the return type """ - # trainer = Trainer(local_rank, world_size, config) - # trainer.train() + trainer = Trainer(local_rank, world_size, config) + trainer.setup_graph() + trainer.setup_optimizer() + trainer.setup_dataset() + trainer.train() return 0 diff --git a/tests/nerf/test_trainer.py b/tests/nerf/test_trainer.py deleted file mode 100644 index 0cc9ff4dd..000000000 --- a/tests/nerf/test_trainer.py +++ /dev/null @@ -1,15 +0,0 @@ -from tty import CFLAG -import pytest -import runpy - -from mattport.nerf.trainer import Trainer -from hydra import compose, initialize - -CONFIG_DIR = "configs" - -def test_multiprocess_train(): - """Test for trainer to validate distributed data parallel working.""" - with initialize(config_path=CONFIG_DIR): - cfg = compose(config_name='test_basic_graph') - trainer = Trainer(cfg) - trainer.run_multiprocess_train() \ No newline at end of file