mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 04:43:52 +08:00
updated logic within Trainer class to allow for multiprocess training
This commit is contained in:
1 parent
ac3e7515ef
commit
62d4218421
3 files changed
+17
-24
No files matched your search
@@ -1,28 +1,29 @@
|
||||
"""
|
||||
Code to train model.
|
||||
"""
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
from omegaconf import DictConfig
|
||||
from torch.nn.parallel import DistributedDataParallel as DDP
|
||||
|
||||
from mattport.nerf.dataset.build import build_dataset
|
||||
from mattport.nerf.graph import Graph
|
||||
from mattport.nerf.optimizer import Optimizer
|
||||
|
||||
|
||||
class Trainer:
|
||||
"""Training class"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
def __init__(self, local_rank: int, world_size: int, config: dict):
|
||||
self.local_rank = local_rank
|
||||
self.world_size = world_size
|
||||
self.config = config
|
||||
self.train_dataset = None
|
||||
self.test_dataset = None
|
||||
self.graph = None
|
||||
self.optimizer = None
|
||||
|
||||
self.device = f"cuda:{local_rank}"
|
||||
self.is_main_thread = self.local_rank % self.world_size == 0
|
||||
|
||||
def setup_dataset(self):
|
||||
"""_summary_"""
|
||||
self.train_dataset = build_dataset(self.config.dataset)
|
||||
@@ -30,6 +31,10 @@ class Trainer:
|
||||
|
||||
def setup_graph(self):
|
||||
"""_summary_"""
|
||||
self.graph = Graph(self.config).to(self.device)
|
||||
if self.world_size > 1:
|
||||
self.graph = DDP(self.graph, device_ids=[self.local_rank])
|
||||
dist.barrier(device_ids=[self.local_rank])
|
||||
raise NotImplementedError
|
||||
|
||||
def setup_optimizer(self):
|
||||
|
||||
@@ -112,8 +112,11 @@ def _train(local_rank: int, world_size: int, config: DictConfig) -> Any:
|
||||
Returns:
|
||||
Any: TODO(): determine the return type
|
||||
"""
|
||||
# trainer = Trainer(local_rank, world_size, config)
|
||||
# trainer.train()
|
||||
trainer = Trainer(local_rank, world_size, config)
|
||||
trainer.setup_graph()
|
||||
trainer.setup_optimizer()
|
||||
trainer.setup_dataset()
|
||||
trainer.train()
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
from tty import CFLAG
|
||||
import pytest
|
||||
import runpy
|
||||
|
||||
from mattport.nerf.trainer import Trainer
|
||||
from hydra import compose, initialize
|
||||
|
||||
CONFIG_DIR = "configs"
|
||||
|
||||
def test_multiprocess_train():
|
||||
"""Test for trainer to validate distributed data parallel working."""
|
||||
with initialize(config_path=CONFIG_DIR):
|
||||
cfg = compose(config_name='test_basic_graph')
|
||||
trainer = Trainer(cfg)
|
||||
trainer.run_multiprocess_train()
|
||||
Reference in new issue
Block a user