From 6f4fb506e2155bcf6bffedc157055e40f7931b48 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Thu, 20 Oct 2022 01:11:40 -0700 Subject: [PATCH] Multi-gpu improvements (#799) * initial commit * minor changes * small changes * ddp seems to be working * remove commented out code --- docs/quickstart/first_nerf.md | 20 ++++++ nerfstudio/data/datamanagers.py | 14 ---- nerfstudio/engine/trainer.py | 18 +---- nerfstudio/pipelines/base_pipeline.py | 4 +- nerfstudio/utils/comms.py | 41 +++++------ nerfstudio/utils/writer.py | 1 - scripts/train.py | 100 ++++++++++++++------------ tests/test_train.py | 2 +- 8 files changed, 93 insertions(+), 107 deletions(-) diff --git a/docs/quickstart/first_nerf.md b/docs/quickstart/first_nerf.md index 971e2f633..5e9ffc50d 100644 --- a/docs/quickstart/first_nerf.md +++ b/docs/quickstart/first_nerf.md @@ -152,3 +152,23 @@ ns-eval --load-config={PATH_TO_CONFIG} --output-path=output.json ``` We also provide a train and evaluation script that allows you to do benchmarking on the classical Blender dataset (see our [benchmarking workflow](../developer_guides/debugging_tools/benchmarking.md)). + +## Multi-GPU Training + +Here we explain how to use multi-GPU training. This is the command to train the nerfacto model with 4 GPUs. We are using [PyTorch Distributed Data Parallel (DDP)](https://pytorch.org/tutorials/beginner/dist_overview.html), so we scale the learning rate with the number of GPUs used. Plotting will only be done for the first process. Note that you may want to play around with both learning rate and `_num_rays_per_batch` when using DDP. Below is a simple example for how you'd run the vanilla-nerf method with either 1 or 4 GPUs. We don't see much value at the moment for running the fast methods like nerfacto with more than one GPU. + +```python +# 1 GPU (1x LR) +export CUDA_VISIBLE_DEVICES=0 +ns-train vanilla-nerf \ + --machine.num-gpus 1 \ + --vis wandb \ + --optimizers.fields.optimizer.lr 5e-4 + +# 4 GPUs (4x LR) +export CUDA_VISIBLE_DEVICES=0,1,2,3 +ns-train vanilla-nerf \ + --machine.num-gpus 4 \ + --vis wandb \ + --optimizers.fields.optimizer.lr 20e-4 +``` diff --git a/nerfstudio/data/datamanagers.py b/nerfstudio/data/datamanagers.py index c0ad8c7a6..c3cf367c3 100644 --- a/nerfstudio/data/datamanagers.py +++ b/nerfstudio/data/datamanagers.py @@ -294,19 +294,12 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method def setup_train(self): """Sets up the data loaders for training""" assert self.train_dataset is not None - if self.world_size > 1: - sampler = DistributedSampler( - self.train_dataset, num_replicas=self.world_size, rank=self.local_rank, shuffle=True, seed=42 - ) - else: - sampler = None self.train_image_dataloader = CacheDataloader( self.train_dataset, num_images_to_sample_from=self.config.train_num_images_to_sample_from, device=self.device, num_workers=self.world_size * 4, pin_memory=True, - sampler=sampler, ) self.iter_train_image_dataloader = iter(self.train_image_dataloader) self.train_pixel_sampler = PixelSampler(self.config.train_num_rays_per_batch) @@ -321,19 +314,12 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method def setup_eval(self): """Sets up the data loader for evaluation""" assert self.eval_dataset is not None - if self.world_size > 1: - sampler = DistributedSampler( - self.eval_dataset, num_replicas=self.world_size, rank=self.local_rank, shuffle=True, seed=42 - ) - else: - sampler = None self.eval_image_dataloader = CacheDataloader( self.eval_dataset, num_images_to_sample_from=self.config.eval_num_images_to_sample_from, device=self.device, num_workers=self.world_size * 4, pin_memory=True, - sampler=sampler, ) self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) self.eval_pixel_sampler = PixelSampler(self.config.eval_num_rays_per_batch) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 686cfcca4..5b9b60c73 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -21,7 +21,7 @@ import dataclasses import functools import os import time -from typing import Any, Dict, List, Tuple +from typing import Dict, List, Tuple import torch from rich.console import Console @@ -48,20 +48,6 @@ from nerfstudio.viewer.server import viewer_utils CONSOLE = Console(width=120) -def train_loop(local_rank: int, world_size: int, config: cfg.Config) -> Any: - """Main training function that sets up and runs the trainer per process - - Args: - local_rank: current rank of process - world_size: total number of gpus available - config: config file specifying training regimen - """ - trainer = Trainer(config, local_rank, world_size) - trainer.setup() - trainer.train() - return 0 - - class Trainer: """Trainer class @@ -122,7 +108,7 @@ class Trainer: Args: test_mode: Whether to setup for testing. Defaults to False. """ - self.pipeline: VanillaPipeline = self.config.pipeline.setup( + self.pipeline = self.config.pipeline.setup( device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank ) self.optimizers = setup_optimizers(self.config, self.pipeline.get_param_groups()) diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 2858c881d..7c64e268f 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -227,9 +227,7 @@ class VanillaPipeline(Pipeline): self.world_size = world_size if world_size > 1: - self._model = typing.cast( - Model, typing.cast(Model, DDP(self._model, device_ids=[local_rank], find_unused_parameters=True)) - ) + self._model = typing.cast(Model, DDP(self._model, device_ids=[local_rank], find_unused_parameters=True)) dist.barrier(device_ids=[local_rank]) @property diff --git a/nerfstudio/utils/comms.py b/nerfstudio/utils/comms.py index e009f6e92..9be472531 100644 --- a/nerfstudio/utils/comms.py +++ b/nerfstudio/utils/comms.py @@ -13,40 +13,38 @@ # limitations under the License. """functionality to handle multiprocessing syncing and communicating""" -import torch import torch.distributed as dist -_LOCAL_PROCESS_GROUP = None +LOCAL_PROCESS_GROUP = None + + +def is_dist_avail_and_initialized() -> bool: + """Returns True if distributed is available and initialized.""" + return dist.is_available() and dist.is_initialized() def get_world_size() -> int: """Get total number of available gpus""" - if not dist.is_available(): - return 1 - if not dist.is_initialized(): + if not is_dist_avail_and_initialized(): return 1 return dist.get_world_size() def get_rank() -> int: """Get global rank of current thread""" - if not dist.is_available(): - return 0 - if not dist.is_initialized(): + if not is_dist_avail_and_initialized(): return 0 return dist.get_rank() def get_local_rank() -> int: """The rank of the current process within the local (per-machine) process group.""" - if not dist.is_available(): - return 0 - if not dist.is_initialized(): + if not is_dist_avail_and_initialized(): return 0 assert ( - _LOCAL_PROCESS_GROUP is not None + LOCAL_PROCESS_GROUP is not None ), "Local process group is not created! Please use launch() to spawn processes!" - return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + return dist.get_rank(group=LOCAL_PROCESS_GROUP) def get_local_size() -> int: @@ -54,11 +52,9 @@ def get_local_size() -> int: The size of the per-machine process group, i.e. the number of processes per machine. """ - if not dist.is_available(): + if not is_dist_avail_and_initialized(): return 1 - if not dist.is_initialized(): - return 1 - return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + return dist.get_world_size(group=LOCAL_PROCESS_GROUP) def is_main_process() -> bool: @@ -66,21 +62,16 @@ def is_main_process() -> bool: return get_rank() == 0 -def synchronize(world_size): +def synchronize(): """ Helper function to synchronize (barrier) among all processes when using distributed training """ - if not dist.is_available(): - return - if not dist.is_initialized(): - return - world_size = dist.get_world_size() - if world_size == 1: + if dist.get_world_size() == 1: return if dist.get_backend() == dist.Backend.NCCL: # This argument is needed to avoid warnings. # It's valid only for NCCL backend. - dist.barrier(device_ids=[torch.cuda.current_device()]) + dist.barrier(device_ids=[get_local_rank()]) else: dist.barrier() diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py index bd9741a7d..e1ed07057 100644 --- a/nerfstudio/utils/writer.py +++ b/nerfstudio/utils/writer.py @@ -166,7 +166,6 @@ def write_out_storage(): EVENT_STORAGE.clear() -@check_main_thread def setup_local_writer(config: cfg.LoggingConfig, max_iter: int, banner_messages: Optional[List[str]] = None) -> None: """Initialization of all event writers specified in config diff --git a/scripts/train.py b/scripts/train.py index 86dd77d68..37717a11c 100755 --- a/scripts/train.py +++ b/scripts/train.py @@ -48,7 +48,7 @@ from rich.console import Console from nerfstudio.configs import base_config as cfg from nerfstudio.configs.config_utils import convert_markup_to_ansi from nerfstudio.configs.method_configs import AnnotatedBaseConfigUnion -from nerfstudio.engine.trainer import train_loop +from nerfstudio.engine.trainer import Trainer from nerfstudio.utils import comms, profiler CONSOLE = Console(width=120) @@ -59,7 +59,7 @@ torch.backends.cudnn.benchmark = True # type: ignore def _find_free_port() -> str: - """Find free port on open socket""" + """Finds a free port.""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("", 0)) port = sock.getsockname()[1] @@ -74,6 +74,20 @@ def _set_random_seed(seed) -> None: torch.manual_seed(seed) +def train_loop(local_rank: int, world_size: int, config: cfg.Config, global_rank: int = 0): + """Main training function that sets up and runs the trainer per process + + Args: + local_rank: current rank of process + world_size: total number of gpus available + config: config file specifying training regimen + """ + _set_random_seed(config.machine.seed + global_rank) + trainer = Trainer(config, local_rank, world_size) + trainer.setup() + trainer.train() + + def _distributed_worker( local_rank: int, main_func: Callable, @@ -85,55 +99,47 @@ def _distributed_worker( timeout: timedelta = DEFAULT_TIMEOUT, ) -> Any: """Spawned distributed worker that handles the initialization of process group and handles the - training process on multiple processes + training process on multiple processes. Args: - local_rank (int): current rank of process - main_func (Callable): function that will be called by the distributed workers - world_size (int): total number of gpus available - num_gpus_per_machine (int): number of GPUs per machine - machine_rank (int): rank of this machine - dist_url (str): url to connect to for distributed jobs, including protocol - e.g. "tcp://127.0.0.1:8686". - Can be set to "auto" to automatically select a free port on localhost - config (Config): config file specifying training regimen - timeout (timedelta, optional): timeout of the distributed workers Defaults to DEFAULT_TIMEOUT. + local_rank: Current rank of process. + main_func: Function that will be called by the distributed workers. + world_size: Total number of gpus available. + num_gpus_per_machine: Number of GPUs per machine. + machine_rank: Rank of this machine. + dist_url: URL to connect to for distributed jobs, including protocol + E.g., "tcp://127.0.0.1:8686". + It can be set to "auto" to automatically select a free port on localhost. + config: Config specifying training regimen. + timeout: Timeout of the distributed workers. Raises: e: Exception in initializing the process group Returns: - Any: TODO(): determine the return type + Any: TODO: determine the return type """ assert torch.cuda.is_available(), "cuda is not available. Please check your installation." global_rank = machine_rank * num_gpus_per_machine + local_rank - try: - dist.init_process_group( - backend="NCCL", - init_method=dist_url, - world_size=world_size, - rank=global_rank, - timeout=timeout, - ) - except Exception as e: - CONSOLE.log(f"Process group URL: {dist_url}") - raise e - assert comms._LOCAL_PROCESS_GROUP is None # pylint: disable=protected-access + dist.init_process_group( + backend="nccl", + init_method=dist_url, + world_size=world_size, + rank=global_rank, + timeout=timeout, + ) + assert comms.LOCAL_PROCESS_GROUP is None num_machines = world_size // num_gpus_per_machine for i in range(num_machines): ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)) pg = dist.new_group(ranks_on_i) if i == machine_rank: - comms._LOCAL_PROCESS_GROUP = pg # pylint: disable=protected-access + comms.LOCAL_PROCESS_GROUP = pg assert num_gpus_per_machine <= torch.cuda.device_count() - torch.cuda.set_device(local_rank) - _set_random_seed(config.machine.seed + global_rank) - comms.synchronize(world_size) - - output = main_func(local_rank, world_size, config) - comms.synchronize(world_size) + output = main_func(local_rank, world_size, config, global_rank) + comms.synchronize() dist.destroy_process_group() return output @@ -160,23 +166,20 @@ def launch( """ assert config is not None world_size = num_machines * num_gpus_per_machine - if world_size == 0: - # Using only CPU and one process. - _set_random_seed(config.machine.seed) - main_func(local_rank=0, world_size=0, config=config) - elif world_size == 1: - # Using one gpu and one process. - _set_random_seed(config.machine.seed) + if world_size <= 1: + # world_size=0 uses one CPU in one process. + # world_size=1 uses one GPU in one process. try: - main_func(local_rank=0, world_size=1, config=config) + main_func(local_rank=0, world_size=world_size, config=config) except KeyboardInterrupt: - print(traceback.format_exc()) + # print the stack trace + CONSOLE.print(traceback.format_exc()) finally: profiler.flush_profiler(config.logging) elif world_size > 1: # Using multiple gpus with multiple processes. if dist_url == "auto": - assert num_machines == 1, "dist_url=auto not supported in multi-machine jobs." + assert num_machines == 1, "dist_url=auto is not supported for multi-machine jobs." port = _find_free_port() dist_url = f"tcp://127.0.0.1:{port}" if num_machines > 1 and dist_url.startswith("file://"): @@ -185,6 +188,7 @@ def launch( process_context = mp.spawn( _distributed_worker, nprocs=num_gpus_per_machine, + join=False, args=( main_func, world_size, @@ -195,16 +199,18 @@ def launch( timeout, ), ) + # process_context won't be None because join=False, so it's okay to assert this + # for Pylance reasons assert process_context is not None try: process_context.join() except KeyboardInterrupt: for i, process in enumerate(process_context.processes): if process.is_alive(): - CONSOLE.log("Terminating process %s", str(i)) + CONSOLE.log(f"Terminating process {i}...") process.terminate() process.join() - CONSOLE.log("Process %s finished", str(i)) + CONSOLE.log(f"Process {i} finished.") finally: profiler.flush_profiler(config.logging) @@ -226,8 +232,8 @@ def main(config: cfg.Config) -> None: config.save_config() launch( - train_loop, - config.machine.num_gpus, + main_func=train_loop, + num_gpus_per_machine=config.machine.num_gpus, num_machines=config.machine.num_machines, machine_rank=config.machine.machine_rank, dist_url=config.machine.dist_url, diff --git a/tests/test_train.py b/tests/test_train.py index 0816088c8..6b69590da 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -12,7 +12,7 @@ import pytest from nerfstudio.configs.base_config import Config from nerfstudio.configs.method_configs import method_configs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig -from nerfstudio.engine.trainer import train_loop +from scripts.train import train_loop BLACKLIST = ["base", "semantic-nerfw", "instant-ngp", "nerfacto"]