mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 05:14:15 +08:00
refactor + docstrings + eval dataset in main python process
This commit is contained in:
1 parent
37e763f2de
commit
2e338a23c6
1 file changed
+135
-54
@@ -4,8 +4,6 @@ import concurrent.futures
|
||||
import functools
|
||||
import time
|
||||
import queue
|
||||
from abc import abstractmethod
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Generic, List, Literal, Optional, Tuple, Type, Union, cast
|
||||
@@ -13,28 +11,21 @@ from typing import Any, Callable, Dict, Generic, List, Literal, Optional, Tuple,
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
from rich.progress import track
|
||||
from torch import nn
|
||||
from torch.nn import Parameter
|
||||
from torch.utils.data.distributed import DistributedSampler
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from nerfstudio.cameras.cameras import CameraType
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.cameras.lie_groups import exp_map_SE3
|
||||
from nerfstudio.configs.base_config import InstantiateConfig
|
||||
from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion
|
||||
from nerfstudio.data.datamanagers.base_datamanager import DataManager, DataManagerConfig, TDataset, variable_res_collate
|
||||
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
|
||||
from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PatchPixelSampler, PixelSampler
|
||||
from nerfstudio.data.utils.dataloaders import CacheDataloader, FixedIndicesEvalDataloader, RandIndicesEvalDataloader
|
||||
from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate
|
||||
from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes
|
||||
from nerfstudio.model_components.ray_generators import RayGenerator
|
||||
from nerfstudio.utils.misc import IterableWrapper
|
||||
from nerfstudio.utils.rich_utils import CONSOLE
|
||||
from nerfstudio.utils import poses as pose_utils
|
||||
from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader, CacheDataloader, FixedIndicesEvalDataloader
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -70,11 +61,11 @@ class ParallelDataManagerConfig(DataManagerConfig):
|
||||
patch_size: int = 1
|
||||
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
|
||||
n_procs: int = 1
|
||||
"""Number of processes to use for data loading."""
|
||||
"""Number of processes to use for train data loading."""
|
||||
queue_size: int = 2
|
||||
"""Size of shared data queue containing generated ray bundles and batches.
|
||||
If queue_size <= 0, the queue size is infinite."""
|
||||
max_thread_workers: Optional[int] = 1
|
||||
max_thread_workers: Optional[int] = None
|
||||
"""Maximum number of threads to use in thread pool executor. If None, automatically
|
||||
set to harware cpu_count + 4."""
|
||||
position_noise_std: float = 0.0
|
||||
@@ -82,42 +73,48 @@ class ParallelDataManagerConfig(DataManagerConfig):
|
||||
orientation_noise_std: float = 0.0
|
||||
"""Noise to add to initial camera pose orientations. Useful for debugging."""
|
||||
non_trainable_camera_indices: Optional[Tuple[int, ...]] = None
|
||||
"""List of non trainable camera indices"""
|
||||
"""List of non trainable camera indices."""
|
||||
|
||||
|
||||
class DataProc(mp.Process):
|
||||
class DataProcessor(mp.Process):
|
||||
"""Parallel dataset batch processor.
|
||||
|
||||
This class is responsible for generating ray bundles from an input dataset
|
||||
in parallel python processes.
|
||||
|
||||
Args:
|
||||
out_queue: the output queue for storing the processed data
|
||||
config: configuration object for the parallel data manager
|
||||
dataparser_outputs: outputs from the dataparser
|
||||
dataset: input dataset
|
||||
pixel_sampler: The pixel sampler for sampling rays
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
out_queue: mp.Queue,
|
||||
config: ParallelDataManagerConfig,
|
||||
dataparser_outputs: DataparserOutputs,
|
||||
train_dataset: TDataset,
|
||||
pix_sampler: PixelSampler,
|
||||
dataset: TDataset,
|
||||
pixel_sampler: PixelSampler,
|
||||
):
|
||||
super().__init__()
|
||||
self.daemon = True
|
||||
self.out_queue = out_queue
|
||||
self.config = config
|
||||
self.dataparser_outputs = dataparser_outputs
|
||||
self.train_dataset = train_dataset
|
||||
self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device
|
||||
if self.dataparser_outputs is not None:
|
||||
cameras = self.dataparser_outputs.cameras
|
||||
if len(cameras) > 1:
|
||||
for i in range(1, len(cameras)):
|
||||
if cameras[0].width != cameras[i].width or cameras[0].height != cameras[i].height:
|
||||
CONSOLE.print("Variable resolution, using variable_res_collate")
|
||||
self.config.collate_fn = variable_res_collate
|
||||
break
|
||||
self.train_pixel_sampler = pix_sampler
|
||||
self.train_ray_generator = RayGenerator(self.train_dataset.cameras)
|
||||
self.dataset = dataset
|
||||
self.exclude_batch_keys_from_device = self.dataset.exclude_batch_keys_from_device
|
||||
self.pixel_sampler = pixel_sampler
|
||||
self.ray_generator = RayGenerator(self.dataset.cameras)
|
||||
self.cache_images()
|
||||
|
||||
def run(self):
|
||||
"""Append out queue in parallel with ray bundles and batches."""
|
||||
while True:
|
||||
batch = self.train_pixel_sampler.sample(self.img_data)
|
||||
batch = self.pixel_sampler.sample(self.img_data)
|
||||
ray_indices = batch["indices"]
|
||||
ray_bundle: RayBundle = self.train_ray_generator(ray_indices)
|
||||
ray_bundle: RayBundle = self.ray_generator(ray_indices)
|
||||
ray_bundle = ray_bundle.pin_memory()
|
||||
while True:
|
||||
try:
|
||||
@@ -131,19 +128,25 @@ class DataProc(mp.Process):
|
||||
|
||||
def cache_images(self):
|
||||
# caches all the input images in a NxHxWx3 tensor
|
||||
indices = range(len(self.train_dataset))
|
||||
indices = range(len(self.dataset))
|
||||
batch_list = []
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers) as executor:
|
||||
for idx in indices:
|
||||
res = executor.submit(self.train_dataset.__getitem__, idx)
|
||||
res = executor.submit(self.dataset.__getitem__, idx)
|
||||
results.append(res)
|
||||
for res in track(results, description="Loading data batch", transient=True):
|
||||
for res in track(results, description="Loading data batch", transient=False):
|
||||
batch_list.append(res.result())
|
||||
self.img_data = self.config.collate_fn(batch_list)
|
||||
|
||||
|
||||
class ParallelDataManager(DataManager, Generic[TDataset]):
|
||||
"""Data manager implementation for parallel dataloading.
|
||||
|
||||
Args:
|
||||
config: the DataManagerConfig used to instantiate class
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: ParallelDataManagerConfig,
|
||||
@@ -158,7 +161,6 @@ class ParallelDataManager(DataManager, Generic[TDataset]):
|
||||
self.device = device
|
||||
self.world_size = world_size
|
||||
self.local_rank = local_rank
|
||||
# self.sampler = None
|
||||
self.test_mode = test_mode
|
||||
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
|
||||
self.dataparser_config = self.config.dataparser
|
||||
@@ -171,27 +173,23 @@ class ParallelDataManager(DataManager, Generic[TDataset]):
|
||||
self.dataparser.downscale_factor = 1 # Avoid opening images
|
||||
self.includes_time = self.dataparser.includes_time
|
||||
self.train_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split="train")
|
||||
self.eval_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split=self.test_split)
|
||||
cameras = self.train_dataparser_outputs.cameras
|
||||
if len(cameras) > 1:
|
||||
for i in range(1, len(cameras)):
|
||||
if cameras[0].width != cameras[i].width or cameras[0].height != cameras[i].height:
|
||||
CONSOLE.print("Variable resolution, using variable_res_collate")
|
||||
self.config.collate_fn = variable_res_collate
|
||||
break
|
||||
self.non_trainable_camera_indices = self.config.non_trainable_camera_indices
|
||||
if self.config.position_noise_std != 0.0 or self.config.orientation_noise_std != 0.0:
|
||||
self.apply_pose_noise(self.non_trainable_camera_indices)
|
||||
self.train_dataset = self.create_train_dataset()
|
||||
# self.eval_dataset = self.create_eval_dataset()
|
||||
self.pix_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch)
|
||||
self.eval_dataset = self.create_eval_dataset()
|
||||
self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device
|
||||
# Spawn is critical for not freezing the program (PyTorch compatability issue)
|
||||
mp.set_start_method("spawn")
|
||||
# Manager().queue() can be faster
|
||||
# https://stackoverflow.com/questions/43439194/python-multiprocessing-queue-vs-multiprocessing-manager-queue/45236748#45236748
|
||||
self.data_q = mp.Manager().Queue(maxsize=self.config.queue_size)
|
||||
self.data_procs = [
|
||||
DataProc(self.data_q, self.config, self.train_dataparser_outputs, self.train_dataset, self.pix_sampler)
|
||||
for i in range(self.config.n_procs)
|
||||
]
|
||||
for proc in self.data_procs:
|
||||
proc.start()
|
||||
super().__init__()
|
||||
# Prime the executor with the first batch
|
||||
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers)
|
||||
self.batch_fut = self.executor.submit(self.data_q.get)
|
||||
|
||||
def create_train_dataset(self) -> TDataset:
|
||||
"""Sets up the data loaders for training"""
|
||||
@@ -200,6 +198,13 @@ class ParallelDataManager(DataManager, Generic[TDataset]):
|
||||
scale_factor=self.config.camera_res_scale_factor,
|
||||
)
|
||||
|
||||
def create_eval_dataset(self) -> TDataset:
|
||||
"""Sets up the data loaders for evaluation"""
|
||||
return self.dataset_type(
|
||||
dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split),
|
||||
scale_factor=self.config.camera_res_scale_factor,
|
||||
)
|
||||
|
||||
def _get_pixel_sampler(self, dataset: TDataset, *args: Any, **kwargs: Any) -> PixelSampler:
|
||||
"""Infer pixel sampler to use."""
|
||||
if self.config.patch_size > 1:
|
||||
@@ -214,15 +219,87 @@ class ParallelDataManager(DataManager, Generic[TDataset]):
|
||||
CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.")
|
||||
return PixelSampler(*args, **kwargs)
|
||||
|
||||
def setup_train(self):
|
||||
"""Sets up the data loaders for training"""
|
||||
assert self.train_dataset is not None
|
||||
self.train_pix_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch)
|
||||
# Manager().queue() can be faster
|
||||
# https://stackoverflow.com/questions/43439194/python-multiprocessing-queue-vs-multiprocessing-manager-queue/45236748#45236748
|
||||
self.data_queue = mp.Manager().Queue(maxsize=self.config.queue_size)
|
||||
# self.data_queue = mp.Queue(maxsize=self.config.queue_size)
|
||||
self.data_procs = [
|
||||
DataProcessor(
|
||||
out_queue=self.data_queue,
|
||||
config=self.config,
|
||||
dataparser_outputs=self.train_dataparser_outputs,
|
||||
dataset=self.train_dataset,
|
||||
pixel_sampler=self.train_pix_sampler,
|
||||
)
|
||||
for i in range(self.config.n_procs)
|
||||
]
|
||||
for proc in self.data_procs:
|
||||
proc.start()
|
||||
|
||||
# Prime the executor with the first batch
|
||||
self.train_executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers)
|
||||
self.train_batch_fut = self.train_executor.submit(self.data_queue.get)
|
||||
|
||||
def setup_eval(self):
|
||||
"""Sets up the data loader for evaluation"""
|
||||
assert self.eval_dataset is not None
|
||||
CONSOLE.print("Setting up evaluation dataset...")
|
||||
self.eval_image_dataloader = CacheDataloader(
|
||||
self.eval_dataset,
|
||||
num_images_to_sample_from=self.config.eval_num_images_to_sample_from,
|
||||
num_times_to_repeat_images=self.config.eval_num_times_to_repeat_images,
|
||||
device=self.device,
|
||||
num_workers=self.world_size * 4,
|
||||
pin_memory=True,
|
||||
collate_fn=self.config.collate_fn,
|
||||
exclude_batch_keys_from_device=self.exclude_batch_keys_from_device,
|
||||
)
|
||||
self.iter_eval_image_dataloader = iter(self.eval_image_dataloader)
|
||||
self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch)
|
||||
self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device))
|
||||
# for loading full images
|
||||
self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader(
|
||||
input_dataset=self.eval_dataset,
|
||||
device=self.device,
|
||||
num_workers=self.world_size * 4,
|
||||
)
|
||||
self.eval_dataloader = RandIndicesEvalDataloader(
|
||||
input_dataset=self.eval_dataset,
|
||||
device=self.device,
|
||||
num_workers=self.world_size * 4,
|
||||
)
|
||||
|
||||
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
self.train_count += 1
|
||||
|
||||
# Fetch the next batch in an executor to parallelize the queue get() operation
|
||||
# with the train step
|
||||
bundle, batch = self.batch_fut.result()
|
||||
self.batch_fut = self.executor.submit(self.data_q.get)
|
||||
bundle = bundle.to(self.device)
|
||||
return bundle, batch
|
||||
bundle, batch = self.train_batch_fut.result()
|
||||
self.train_batch_fut = self.train_executor.submit(self.data_queue.get)
|
||||
ray_bundle = bundle.to(self.device)
|
||||
return ray_bundle, batch
|
||||
|
||||
def next_eval(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
"""Returns the next batch of data fnon_trainable_camera_indicesrom the eval dataloader."""
|
||||
self.eval_count += 1
|
||||
image_batch = next(self.iter_eval_image_dataloader)
|
||||
assert self.eval_pixel_sampler is not None
|
||||
assert isinstance(image_batch, dict)
|
||||
batch = self.eval_pixel_sampler.sample(image_batch)
|
||||
ray_indices = batch["indices"]
|
||||
ray_bundle = self.eval_ray_generator(ray_indices)
|
||||
return ray_bundle, batch
|
||||
|
||||
def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]:
|
||||
for camera_ray_bundle, batch in self.eval_dataloader:
|
||||
assert camera_ray_bundle.camera_indices is not None
|
||||
image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0])
|
||||
return image_idx, camera_ray_bundle, batch
|
||||
raise ValueError("No more eval images")
|
||||
|
||||
def get_train_rays_per_batch(self) -> int:
|
||||
return self.config.train_num_rays_per_batch
|
||||
@@ -241,7 +318,11 @@ class ParallelDataManager(DataManager, Generic[TDataset]):
|
||||
return {}
|
||||
|
||||
def apply_pose_noise(self, non_trainable_camera_indices: Optional[List[int]]):
|
||||
"""Apply noise to training camera poses"""
|
||||
"""Apply noise to training camera poses.
|
||||
|
||||
Args:
|
||||
non_trainable_camera_indices: list of non trainable cameras
|
||||
"""
|
||||
assert self.config.position_noise_std >= 0.0 and self.config.orientation_noise_std >= 0.0
|
||||
camera_to_worlds = self.train_dataparser_outputs.cameras.camera_to_worlds.to(self.device)
|
||||
num_cameras = len(self.train_dataparser_outputs.cameras)
|
||||
|
||||
Reference in new issue
Block a user