updated logic within Trainer class to allow for multiprocess training

This commit is contained in:
evonneng committed 2022-04-15 21:53:35 +00:00
1 parent ac3e7515ef
commit 62d4218421
3 files changed
+17 -24

No files matched your search

+12 -7
View File
@@ -1,28 +1,29 @@
"""
Code to train model.
"""
import os
import numpy as np
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from omegaconf import DictConfig
from torch.nn.parallel import DistributedDataParallel as DDP
from mattport.nerf.dataset.build import build_dataset
from mattport.nerf.graph import Graph
from mattport.nerf.optimizer import Optimizer
class Trainer:
"""Training class"""
def __init__(self, config: dict):
def __init__(self, local_rank: int, world_size: int, config: dict):
self.local_rank = local_rank
self.world_size = world_size
self.config = config
self.train_dataset = None
self.test_dataset = None
self.graph = None
self.optimizer = None
self.device = f"cuda:{local_rank}"
self.is_main_thread = self.local_rank % self.world_size == 0
def setup_dataset(self):
"""_summary_"""
self.train_dataset = build_dataset(self.config.dataset)
@@ -30,6 +31,10 @@ class Trainer:
def setup_graph(self):
"""_summary_"""
self.graph = Graph(self.config).to(self.device)
if self.world_size > 1:
self.graph = DDP(self.graph, device_ids=[self.local_rank])
dist.barrier(device_ids=[self.local_rank])
raise NotImplementedError
def setup_optimizer(self):
+5 -2
View File
@@ -112,8 +112,11 @@ def _train(local_rank: int, world_size: int, config: DictConfig) -> Any:
Returns:
Any: TODO(): determine the return type
"""
# trainer = Trainer(local_rank, world_size, config)
# trainer.train()
trainer = Trainer(local_rank, world_size, config)
trainer.setup_graph()
trainer.setup_optimizer()
trainer.setup_dataset()
trainer.train()
return 0
-15
View File
@@ -1,15 +0,0 @@
from tty import CFLAG
import pytest
import runpy
from mattport.nerf.trainer import Trainer
from hydra import compose, initialize
CONFIG_DIR = "configs"
def test_multiprocess_train():
"""Test for trainer to validate distributed data parallel working."""
with initialize(config_path=CONFIG_DIR):
cfg = compose(config_name='test_basic_graph')
trainer = Trainer(cfg)
trainer.run_multiprocess_train()