diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index df8296ea1..7b7648884 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -23,19 +23,16 @@ from dataclasses import dataclass, field from typing import Literal, Optional, Type, Union import torch -import tyro from jaxtyping import Float, Int from torch import Tensor, nn from typing_extensions import assert_never from nerfstudio.cameras.lie_groups import exp_map_SE3, exp_map_SO3xR3 +from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig -from nerfstudio.engine.optimizers import AdamOptimizerConfig, OptimizerConfig -from nerfstudio.engine.schedulers import ( - ExponentialDecaySchedulerConfig, - SchedulerConfig, -) from nerfstudio.utils import poses as pose_utils +from nerfstudio.engine.optimizers import OptimizerConfig +from nerfstudio.engine.schedulers import SchedulerConfig @dataclass @@ -47,21 +44,38 @@ class CameraOptimizerConfig(InstantiateConfig): mode: Literal["off", "SO3xR3", "SE3"] = "off" """Pose optimization strategy to use. If enabled, we recommend SO3xR3.""" - position_noise_std: float = 0.0 - """Noise to add to initial positions. Useful for debugging.""" + trans_l2_penalty: float = 1e-2 + """L2 penalty on translation parameters.""" - orientation_noise_std: float = 0.0 - """Noise to add to initial orientations. Useful for debugging.""" + rot_l2_penalty: float = 1e-3 + """L2 penalty on rotation parameters.""" - optimizer: OptimizerConfig = field(default_factory=lambda: AdamOptimizerConfig(lr=6e-4, eps=1e-15)) - """ADAM parameters for camera optimization.""" + optimizer: Optional[OptimizerConfig] = field(default=None) + """Deprecated, now specified inside the optimizers dict""" - scheduler: SchedulerConfig = field(default_factory=lambda: ExponentialDecaySchedulerConfig(max_steps=10000)) - """Learning rate scheduler for camera optimizer..""" + scheduler: Optional[SchedulerConfig] = field(default=None) + """Deprecated, now specified inside the optimizers dict""" - param_group: tyro.conf.Suppress[str] = "camera_opt" - """Name of the parameter group used for pose optimization. Can be any string that doesn't conflict with other - groups.""" + def __post_init__(self): + if self.optimizer is not None: + import warnings + from nerfstudio.utils.rich_utils import CONSOLE + + CONSOLE.print( + "\noptimizer is no longer specified in the CameraOptimizerConfig, it is now defined with the rest of the param groups inside the config file under the name 'camera_opt'\n", + style="bold yellow", + ) + warnings.warn("above message coming from", FutureWarning, stacklevel=3) + + if self.scheduler is not None: + import warnings + from nerfstudio.utils.rich_utils import CONSOLE + + CONSOLE.print( + "\nscheduler is no longer specified in the CameraOptimizerConfig, it is now defined with the rest of the param groups inside the config file under the name 'camera_opt'\n", + style="bold yellow", + ) + warnings.warn("above message coming from", FutureWarning, stacklevel=3) class CameraOptimizer(nn.Module): @@ -91,16 +105,6 @@ class CameraOptimizer(nn.Module): else: assert_never(self.config.mode) - # Initialize pose noise; useful for debugging. - if config.position_noise_std != 0.0 or config.orientation_noise_std != 0.0: - assert config.position_noise_std >= 0.0 and config.orientation_noise_std >= 0.0 - std_vector = torch.tensor( - [config.position_noise_std] * 3 + [config.orientation_noise_std] * 3, device=device - ) - self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((num_cameras, 6), device=device), std_vector)) - else: - self.pose_noise = None - def forward( self, indices: Int[Tensor, "camera_indices"], @@ -125,13 +129,46 @@ class CameraOptimizer(nn.Module): assert_never(self.config.mode) # Detach non-trainable indices by setting to identity transform if self.non_trainable_camera_indices is not None: - outputs[0][self.non_trainable_camera_indices] = torch.eye(4, device=self.device)[:3, :4] + if self.non_trainable_camera_indices.device != self.pose_adjustment.device: + self.non_trainable_camera_indices = self.non_trainable_camera_indices.to(self.pose_adjustment.device) + outputs[0][self.non_trainable_camera_indices] = torch.eye(4, device=self.pose_adjustment.device)[:3, :4] - # Apply initial pose noise. - if self.pose_noise is not None: - outputs.append(self.pose_noise[indices, :, :]) # Return: identity if no transforms are needed, otherwise multiply transforms together. if len(outputs) == 0: # Note that using repeat() instead of tile() here would result in unnecessary copies. return torch.eye(4, device=self.device)[None, :3, :4].tile(indices.shape[0], 1, 1) return functools.reduce(pose_utils.multiply, outputs) + + def apply_to_raybundle(self, raybundle: RayBundle) -> None: + """Apply the pose correction to the raybundle""" + if self.config.mode != "off": + correction_matrices = self(raybundle.camera_indices.squeeze()) # type: ignore + raybundle.origins = raybundle.origins + correction_matrices[:, :3, 3] + raybundle.directions = torch.bmm(correction_matrices[:, :3, :3], raybundle.directions[..., None]).squeeze() + + def get_loss_dict(self, loss_dict: dict) -> None: + """Add regularization""" + if self.config.mode != "off": + loss_dict["camera_opt_regularizer"] = ( + self.pose_adjustment[:, :3].norm(dim=-1).mean() * self.config.trans_l2_penalty + + self.pose_adjustment[:, 3:].norm(dim=-1).mean() * self.config.rot_l2_penalty + ) + + def get_correction_matrices(self): + """Get optimized pose correction matrices""" + return self(torch.arange(0, self.num_cameras).long()) + + def get_metrics_dict(self, metrics_dict: dict) -> None: + """Get camera optimizer metrics""" + if self.config.mode != "off": + metrics_dict["camera_opt_translation"] = self.pose_adjustment[:, :3].norm() + metrics_dict["camera_opt_rotation"] = self.pose_adjustment[:, 3:].norm() + + def get_param_groups(self, param_groups: dict) -> None: + """Get camera optimizer parameters""" + camera_opt_params = list(self.parameters()) + if self.config.mode != "off": + assert len(camera_opt_params) > 0 + param_groups["camera_opt"] = camera_opt_params + else: + assert len(camera_opt_params) == 0 diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index a92c042bf..74cabbafc 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -27,15 +27,23 @@ from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.configs.external_methods import get_external_methods - -from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig - +from nerfstudio.data.datamanagers.base_datamanager import ( + VanillaDataManager, + VanillaDataManagerConfig, +) +from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManagerConfig +from nerfstudio.data.datamanagers.random_cameras_datamanager import ( + RandomCamerasDataManagerConfig, +) from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig -from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig +from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( + InstantNGPDataParserConfig, +) from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig -from nerfstudio.data.dataparsers.phototourism_dataparser import PhototourismDataParserConfig +from nerfstudio.data.dataparsers.phototourism_dataparser import ( + PhototourismDataParserConfig, +) from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig from nerfstudio.data.datasets.depth_dataset import DepthDataset @@ -88,17 +96,15 @@ method_configs["nerfacto"] = TrainerConfig( max_num_iterations=30000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), - ), ), - model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=NerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), + ), ), optimizers={ "proposal_networks": { @@ -109,10 +115,15 @@ method_configs["nerfacto"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) + method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", steps_per_eval_batch=500, @@ -120,14 +131,10 @@ method_configs["nerfacto-big"] = TrainerConfig( max_num_iterations=100000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=8192, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3), - ), ), model=NerfactoModelConfig( eval_num_rays_per_chunk=1 << 15, @@ -139,6 +146,7 @@ method_configs["nerfacto-big"] = TrainerConfig( max_res=4096, proposal_weights_anneal_max_num_iters=5000, log2_hashmap_size=21, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), ), ), optimizers={ @@ -150,10 +158,15 @@ method_configs["nerfacto-big"] = TrainerConfig( "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=50000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) + method_configs["nerfacto-huge"] = TrainerConfig( method_name="nerfacto", steps_per_eval_batch=500, @@ -161,15 +174,10 @@ method_configs["nerfacto-huge"] = TrainerConfig( max_num_iterations=100000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=16384, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-5, max_steps=50000), - ), ), model=NerfactoModelConfig( eval_num_rays_per_chunk=1 << 15, @@ -185,6 +193,7 @@ method_configs["nerfacto-huge"] = TrainerConfig( max_res=8192, proposal_weights_anneal_max_num_iters=5000, log2_hashmap_size=21, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), ), ), optimizers={ @@ -196,6 +205,10 @@ method_configs["nerfacto-huge"] = TrainerConfig( "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=50000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -214,11 +227,11 @@ method_configs["depth-nerfacto"] = TrainerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), ), - model=DepthNerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=DepthNerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), + ), ), optimizers={ "proposal_networks": { @@ -229,6 +242,10 @@ method_configs["depth-nerfacto"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": None, }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -258,7 +275,6 @@ method_configs["instant-ngp"] = TrainerConfig( vis="viewer", ) - method_configs["instant-ngp-bounded"] = TrainerConfig( method_name="instant-ngp-bounded", steps_per_eval_batch=500, @@ -286,12 +302,12 @@ method_configs["instant-ngp-bounded"] = TrainerConfig( viewer=ViewerConfig(num_rays_per_chunk=1 << 12), vis="viewer", ) - - +# +# method_configs["mipnerf"] = TrainerConfig( method_name="mipnerf", pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024), + datamanager=ParallelDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024), model=VanillaModelConfig( _target=MipNerfModel, loss_coefficients={"rgb_loss_coarse": 0.1, "rgb_loss_fine": 1.0}, @@ -364,13 +380,14 @@ method_configs["tensorf"] = TrainerConfig( max_num_iterations=30000, mixed_precision=False, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, ), model=TensoRFModelConfig( regularization="tv", + camera_optimizer=CameraOptimizerConfig(mode="off"), ), ), optimizers={ @@ -382,6 +399,10 @@ method_configs["tensorf"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=0.02), "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.002, max_steps=30000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-4, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-5, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -420,16 +441,16 @@ method_configs["phototourism"] = TrainerConfig( dataparser=PhototourismDataParserConfig(), # NOTE: one of the only differences with nerfacto train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), # Large dataset, so using prior values from VariableResDataManager. train_num_images_to_sample_from=40, train_num_times_to_repeat_images=100, eval_num_images_to_sample_from=40, eval_num_times_to_repeat_images=100, ), - model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=NerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), + ), ), optimizers={ "proposal_networks": { @@ -440,6 +461,10 @@ method_configs["phototourism"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": None, }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -504,9 +529,6 @@ method_configs["neus"] = TrainerConfig( dataparser=SDFStudioDataParserConfig(), train_num_rays_per_batch=1024, eval_num_rays_per_batch=1024, - camera_optimizer=CameraOptimizerConfig( - mode="off", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), ), model=NeuSModelConfig(eval_num_rays_per_chunk=1024), ), @@ -538,9 +560,6 @@ method_configs["neus-facto"] = TrainerConfig( dataparser=SDFStudioDataParserConfig(), train_num_rays_per_batch=2048, eval_num_rays_per_batch=2048, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), ), model=NeuSFactoModelConfig( # proposal network allows for significantly smaller sdf/color network diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index cb63202df..0b937f64c 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -113,8 +113,6 @@ class DataManagerConfig(InstantiateConfig): """Target class to instantiate.""" data: Optional[Path] = None """Source of data, may not be used by all models.""" - camera_optimizer: Optional[CameraOptimizerConfig] = None - """Specifies the camera pose optimizer used during training. Helpful if poses are noisy.""" masks_on_gpu: bool = False """Process masks on GPU for speed at the expense of memory, if True.""" images_on_gpu: bool = False @@ -335,9 +333,8 @@ class VanillaDataManagerConfig(DataManagerConfig): eval_num_times_to_repeat_images: int = -1 """When not evaluating on all images, number of iterations before picking new images. If -1, never pick new images.""" - camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() - """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from - Record3D.""" + eval_image_indices: Optional[Tuple[int, ...]] = (0,) + """Specifies the image indices to use during eval; if None, uses all.""" collate_fn: Callable[[Any], Any] = cast(Any, staticmethod(nerfstudio_collate)) """Specifies the collate function to use for the train and eval dataloaders.""" camera_res_scale_factor: float = 1.0 @@ -346,9 +343,21 @@ class VanillaDataManagerConfig(DataManagerConfig): """ patch_size: int = 1 """Size of patch to sample from. If >1, patch-based sampling will be used.""" + camera_optimizer: Optional[CameraOptimizerConfig] = field(default=None) + """Deprecated, has been moved to the model config.""" pixel_sampler: PixelSamplerConfig = PixelSamplerConfig() """Specifies the pixel sampler used to sample pixels from images.""" + def __post_init__(self): + """Warn user of camera optimizer change.""" + if self.camera_optimizer is not None: + import warnings + + CONSOLE.print( + "\nCameraOptimizerConfig has been moved from the DataManager to the Model.\n", style="bold yellow" + ) + warnings.warn("above message coming from", FutureWarning, stacklevel=3) + TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset) @@ -486,13 +495,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): ) self.iter_train_image_dataloader = iter(self.train_image_dataloader) self.train_pixel_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) - self.train_camera_optimizer = self.config.camera_optimizer.setup( - num_cameras=self.train_dataset.cameras.size, device=self.device - ) - self.train_ray_generator = RayGenerator( - self.train_dataset.cameras.to(self.device), - self.train_camera_optimizer, - ) + self.train_ray_generator = RayGenerator(self.train_dataset.cameras.to(self.device)) def setup_eval(self): """Sets up the data loader for evaluation""" @@ -510,13 +513,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): ) self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch) - self.eval_camera_optimizer = self.config.camera_optimizer.setup( - num_cameras=self.eval_dataset.cameras.size, device=self.device - ) - self.eval_ray_generator = RayGenerator( - self.eval_dataset.cameras.to(self.device), - self.eval_camera_optimizer, - ) + self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device)) # for loading full images self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader( input_dataset=self.eval_dataset, @@ -572,13 +569,4 @@ class VanillaDataManager(DataManager, Generic[TDataset]): Returns: A list of dictionaries containing the data manager's param groups. """ - param_groups = {} - - camera_opt_params = list(self.train_camera_optimizer.parameters()) - if self.config.camera_optimizer.mode != "off": - assert len(camera_opt_params) > 0 - param_groups[self.config.camera_optimizer.param_group] = camera_opt_params - else: - assert len(camera_opt_params) == 0 - - return param_groups + return {} diff --git a/nerfstudio/data/datamanagers/parallel_datamanager.py b/nerfstudio/data/datamanagers/parallel_datamanager.py new file mode 100644 index 000000000..8d541d79e --- /dev/null +++ b/nerfstudio/data/datamanagers/parallel_datamanager.py @@ -0,0 +1,321 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Parallel data manager that generates training data in multiple python processes. +""" +from __future__ import annotations + +import concurrent.futures +import queue +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import ( + Dict, + Generic, + List, + Literal, + Optional, + Tuple, + Type, + Union, +) + +import torch +import torch.multiprocessing as mp +from rich.progress import track +from torch.nn import Parameter + +from nerfstudio.cameras.cameras import CameraType +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.data.datamanagers.base_datamanager import ( + DataManager, + VanillaDataManagerConfig, + TDataset, + variable_res_collate, +) +from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.data.pixel_samplers import ( + PixelSampler, + PixelSamplerConfig, + PatchPixelSamplerConfig, +) +from nerfstudio.data.utils.dataloaders import ( + CacheDataloader, + FixedIndicesEvalDataloader, + RandIndicesEvalDataloader, +) +from nerfstudio.model_components.ray_generators import RayGenerator +from nerfstudio.utils.rich_utils import CONSOLE + + +@dataclass +class ParallelDataManagerConfig(VanillaDataManagerConfig): + """Config for a `ParallelDataManager` which reads data in multiple processes""" + + _target: Type = field(default_factory=lambda: ParallelDataManager) + """Target class to instantiate.""" + num_processes: int = 1 + """Number of processes to use for train data loading. More than 1 doesn't result in that much better performance""" + queue_size: int = 2 + """Size of shared data queue containing generated ray bundles and batches. + If queue_size <= 0, the queue size is infinite.""" + max_thread_workers: Optional[int] = None + """Maximum number of threads to use in thread pool executor. If None, use ThreadPool default.""" + + +class DataProcessor(mp.Process): + """Parallel dataset batch processor. + + This class is responsible for generating ray bundles from an input dataset + in parallel python processes. + + Args: + out_queue: the output queue for storing the processed data + config: configuration object for the parallel data manager + dataparser_outputs: outputs from the dataparser + dataset: input dataset + pixel_sampler: The pixel sampler for sampling rays + """ + + def __init__( + self, + out_queue: mp.Queue, + config: ParallelDataManagerConfig, + dataparser_outputs: DataparserOutputs, + dataset: TDataset, + pixel_sampler: PixelSampler, + ): + super().__init__() + self.daemon = True + self.out_queue = out_queue + self.config = config + self.dataparser_outputs = dataparser_outputs + self.dataset = dataset + self.exclude_batch_keys_from_device = self.dataset.exclude_batch_keys_from_device + self.pixel_sampler = pixel_sampler + self.ray_generator = RayGenerator(self.dataset.cameras) + self.cache_images() + + def run(self): + """Append out queue in parallel with ray bundles and batches.""" + while True: + batch = self.pixel_sampler.sample(self.img_data) + ray_indices = batch["indices"] + ray_bundle: RayBundle = self.ray_generator(ray_indices) + # check that GPUs are available + if torch.cuda.is_available(): + ray_bundle = ray_bundle.pin_memory() + while True: + try: + self.out_queue.put_nowait((ray_bundle, batch)) + break + except queue.Full: + time.sleep(0.0001) + except Exception: + CONSOLE.print_exception() + CONSOLE.print("[bold red]Error: Error occured in parallel datamanager queue.") + + def cache_images(self): + """Caches all input images into a NxHxWx3 tensor.""" + indices = range(len(self.dataset)) + batch_list = [] + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers) as executor: + for idx in indices: + res = executor.submit(self.dataset.__getitem__, idx) + results.append(res) + for res in track(results, description="Loading data batch", transient=False): + batch_list.append(res.result()) + self.img_data = self.config.collate_fn(batch_list) + + +class ParallelDataManager(DataManager, Generic[TDataset]): + """Data manager implementation for parallel dataloading. + + Args: + config: the DataManagerConfig used to instantiate class + """ + + def __init__( + self, + config: ParallelDataManagerConfig, + device: Union[torch.device, str] = "cpu", + test_mode: Literal["test", "val", "inference"] = "val", + world_size: int = 1, + local_rank: int = 0, + **kwargs, + ): + self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", getattr(TDataset, "__default__")) + self.config = config + self.device = device + self.world_size = world_size + self.local_rank = local_rank + self.test_mode = test_mode + self.test_split = "test" if test_mode in ["test", "inference"] else "val" + self.dataparser_config = self.config.dataparser + if self.config.data is not None: + self.config.dataparser.data = Path(self.config.data) + else: + self.config.data = self.config.dataparser.data + self.dataparser = self.dataparser_config.setup() + if test_mode == "inference": + self.dataparser.downscale_factor = 1 # Avoid opening images + self.includes_time = self.dataparser.includes_time + self.train_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split="train") + self.eval_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split=self.test_split) + cameras = self.train_dataparser_outputs.cameras + if len(cameras) > 1: + for i in range(1, len(cameras)): + if cameras[0].width != cameras[i].width or cameras[0].height != cameras[i].height: + CONSOLE.print("Variable resolution, using variable_res_collate") + self.config.collate_fn = variable_res_collate + break + self.train_dataset = self.create_train_dataset() + self.eval_dataset = self.create_eval_dataset() + self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device + # Spawn is critical for not freezing the program (PyTorch compatability issue) + # check if spawn is already set + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method("spawn") + super().__init__() + + def create_train_dataset(self) -> TDataset: + """Sets up the data loaders for training.""" + return self.dataset_type( + dataparser_outputs=self.train_dataparser_outputs, + scale_factor=self.config.camera_res_scale_factor, + ) + + def create_eval_dataset(self) -> TDataset: + """Sets up the data loaders for evaluation.""" + return self.dataset_type( + dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split), + scale_factor=self.config.camera_res_scale_factor, + ) + + def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> PixelSampler: + """Infer pixel sampler to use.""" + if self.config.patch_size > 1 and type(self.config.pixel_sampler) is PixelSamplerConfig: + return PatchPixelSamplerConfig().setup( + patch_size=self.config.patch_size, num_rays_per_batch=num_rays_per_batch + ) + is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all() + if is_equirectangular.any(): + CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.") + return self.config.pixel_sampler.setup( + is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch + ) + + def setup_train(self): + """Sets up parallel python data processes for training.""" + assert self.train_dataset is not None + self.train_pix_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) # type: ignore + self.data_queue = mp.Manager().Queue(maxsize=self.config.queue_size) + self.data_procs = [ + DataProcessor( + out_queue=self.data_queue, # type: ignore + config=self.config, + dataparser_outputs=self.train_dataparser_outputs, + dataset=self.train_dataset, + pixel_sampler=self.train_pix_sampler, + ) + for i in range(self.config.num_processes) + ] + for proc in self.data_procs: + proc.start() + print("Started threads") + + # Prime the executor with the first batch + self.train_executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers) + self.train_batch_fut = self.train_executor.submit(self.data_queue.get) + + def setup_eval(self): + """Sets up the data loader for evaluation.""" + assert self.eval_dataset is not None + CONSOLE.print("Setting up evaluation dataset...") + self.eval_image_dataloader = CacheDataloader( + self.eval_dataset, + num_images_to_sample_from=self.config.eval_num_images_to_sample_from, + num_times_to_repeat_images=self.config.eval_num_times_to_repeat_images, + device=self.device, + num_workers=self.world_size * 4, + pin_memory=True, + collate_fn=self.config.collate_fn, + exclude_batch_keys_from_device=self.exclude_batch_keys_from_device, + ) + self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) + self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch) # type: ignore + self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device)) + # for loading full images + self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader( + input_dataset=self.eval_dataset, + device=self.device, + num_workers=self.world_size * 4, + ) + self.eval_dataloader = RandIndicesEvalDataloader( + input_dataset=self.eval_dataset, + device=self.device, + num_workers=self.world_size * 4, + ) + + def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the parallel training processes.""" + self.train_count += 1 + + # Fetch the next batch in an executor to parallelize the queue get() operation + # with the train step + bundle, batch = self.train_batch_fut.result() + self.train_batch_fut = self.train_executor.submit(self.data_queue.get) + ray_bundle = bundle.to(self.device) + return ray_bundle, batch + + def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the eval dataloader.""" + self.eval_count += 1 + image_batch = next(self.iter_eval_image_dataloader) + assert self.eval_pixel_sampler is not None + assert isinstance(image_batch, dict) + batch = self.eval_pixel_sampler.sample(image_batch) + ray_indices = batch["indices"] + ray_bundle = self.eval_ray_generator(ray_indices) + return ray_bundle, batch + + def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + """Retrieve the next eval image.""" + for camera_ray_bundle, batch in self.eval_dataloader: + assert camera_ray_bundle.camera_indices is not None + image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0]) + return image_idx, camera_ray_bundle, batch + raise ValueError("No more eval images") + + def get_train_rays_per_batch(self) -> int: + """Returns the number of rays per batch for training.""" + return self.config.train_num_rays_per_batch + + def get_eval_rays_per_batch(self) -> int: + """Returns the number of rays per batch for evaluation.""" + return self.config.eval_num_rays_per_batch + + def get_datapath(self) -> Path: + """Returns the path to the data. This is used to determine where to save camera paths.""" + return self.config.dataparser.data + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Get the param groups for the data manager. + Returns: + A list of dictionaries containing the data manager's param groups. + """ + return {} diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index d7e76806d..9234a5d42 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -98,7 +98,7 @@ class PixelSampler: chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size) indices = nonzero_indices[chosen_indices] else: - indices = torch.floor( + indices = ( torch.rand((batch_size, 3), device=device) * torch.tensor([num_images, image_height, image_width], device=device) ).long() @@ -172,13 +172,11 @@ class PixelSampler: collated_batch = { key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and value is not None } - assert collated_batch["image"].shape[0] == num_rays_per_batch # Needed to correct the random indices to their actual camera idx locations. indices[:, 0] = batch["image_idx"][c] collated_batch["indices"] = indices # with the abs camera indices - if keep_full_image: collated_batch["full_image"] = batch["image"] diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py index b80b77bcb..9b8d48dba 100644 --- a/nerfstudio/engine/optimizers.py +++ b/nerfstudio/engine/optimizers.py @@ -84,6 +84,25 @@ class Optimizers: self.schedulers = {} self.parameters = {} for param_group_name, params in param_groups.items(): + # For deprecation, catch the camera_opt param group and fix it nicely + if param_group_name == "camera_opt" and "camera_opt" not in config: + from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig + from nerfstudio.utils.rich_utils import CONSOLE + + CONSOLE.print( + "\nThe 'camera_opt' param group should be assigned an optimizer in the config. Assigning default optimizers for now. This will be removed in a future release.\n", + style="bold yellow", + ) + + config["camera_opt"] = { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=30000), + } + # Print some nice warning messages if the user forgot to specify an optimizer + if param_group_name not in config: + raise RuntimeError( + f"""Optimizer config for '{param_group_name}' not found in config file. Make sure you specify an optimizer for each parameter group. Provided configs were: {config.keys()}""" + ) lr_init = config[param_group_name]["optimizer"].lr self.optimizers[param_group_name] = config[param_group_name]["optimizer"].setup(params=params) self.parameters[param_group_name] = params diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index b4ccba9ee..fe4aa85ce 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -219,13 +219,6 @@ class Trainer: """ optimizer_config = self.config.optimizers.copy() param_groups = self.pipeline.get_param_groups() - camera_optimizer_config = self.config.pipeline.datamanager.camera_optimizer - if camera_optimizer_config is not None and camera_optimizer_config.mode != "off": - assert camera_optimizer_config.param_group not in optimizer_config - optimizer_config[camera_optimizer_config.param_group] = { - "optimizer": camera_optimizer_config.optimizer, - "scheduler": camera_optimizer_config.scheduler, - } return Optimizers(optimizer_config, param_groups) def train(self) -> None: diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index 34c6346fc..f215a0045 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -116,6 +116,7 @@ class NerfactoField(Field): self.use_pred_normals = use_pred_normals self.pass_semantic_gradients = pass_semantic_gradients self.base_res = base_res + self.step = 0 self.direction_encoding = SHEncoding( levels=4, diff --git a/nerfstudio/model_components/ray_generators.py b/nerfstudio/model_components/ray_generators.py index 49754548d..bad32258d 100644 --- a/nerfstudio/model_components/ray_generators.py +++ b/nerfstudio/model_components/ray_generators.py @@ -18,7 +18,6 @@ Ray generator. from jaxtyping import Int from torch import Tensor, nn -from nerfstudio.cameras.camera_optimizers import CameraOptimizer from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle @@ -34,10 +33,9 @@ class RayGenerator(nn.Module): image_coords: Tensor - def __init__(self, cameras: Cameras, pose_optimizer: CameraOptimizer) -> None: + def __init__(self, cameras: Cameras) -> None: super().__init__() self.cameras = cameras - self.pose_optimizer = pose_optimizer self.register_buffer("image_coords", cameras.get_image_coords(), persistent=False) def forward(self, ray_indices: Int[Tensor, "num_rays 3"]) -> RayBundle: @@ -51,11 +49,8 @@ class RayGenerator(nn.Module): x = ray_indices[:, 2] # col indices coords = self.image_coords[y, x] - camera_opt_to_camera = self.pose_optimizer(c) - ray_bundle = self.cameras.generate_rays( camera_indices=c.unsqueeze(-1), coords=coords, - camera_opt_to_camera=camera_opt_to_camera, ) return ray_bundle diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 1122d5c20..df8eed9b6 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -28,6 +28,7 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image import PeakSignalNoiseRatio from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity +from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig from nerfstudio.cameras.rays import RayBundle, RaySamples from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation from nerfstudio.field_components.field_heads import FieldHeadNames @@ -126,6 +127,8 @@ class NerfactoModelConfig(ModelConfig): """Which implementation to use for the model.""" appearance_embed_dim: int = 32 """Dimension of the appearance embedding.""" + camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="SO3xR3") + """Config of the camera optimizer to use""" class NerfactoModel(Model): @@ -165,6 +168,9 @@ class NerfactoModel(Model): implementation=self.config.implementation, ) + self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup( + num_cameras=self.num_train_data, device="cpu" + ) self.density_fns = [] num_prop_nets = self.config.num_proposal_iterations # Build the proposal network(s) @@ -229,7 +235,7 @@ class NerfactoModel(Model): # losses self.rgb_loss = MSELoss() - + self.step = 0 # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure @@ -240,6 +246,7 @@ class NerfactoModel(Model): param_groups = {} param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) param_groups["fields"] = list(self.field.parameters()) + self.camera_optimizer.get_param_groups(param_groups=param_groups) return param_groups def get_training_callbacks( @@ -252,6 +259,7 @@ class NerfactoModel(Model): def set_anneal(step): # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + self.step = step train_frac = np.clip(step / N, 0, 1) self.step = step @@ -278,6 +286,9 @@ class NerfactoModel(Model): return callbacks def get_outputs(self, ray_bundle: RayBundle): + # apply the camera optimizer pose tweaks + if self.training: + self.camera_optimizer.apply_to_raybundle(ray_bundle) ray_samples: RaySamples ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals) @@ -324,7 +335,6 @@ class NerfactoModel(Model): for i in range(self.config.num_proposal_iterations): outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) - return outputs def get_metrics_dict(self, outputs, batch): @@ -336,6 +346,8 @@ class NerfactoModel(Model): if self.training: metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"]) + + self.camera_optimizer.get_metrics_dict(metrics_dict) return metrics_dict def get_loss_dict(self, outputs, batch, metrics_dict=None): @@ -364,6 +376,8 @@ class NerfactoModel(Model): loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( outputs["rendered_pred_normal_loss"] ) + # Add loss from camera optimizer + self.camera_optimizer.get_loss_dict(loss_dict) return loss_dict def get_image_metrics_and_images( diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index a0380d42e..9cc9c7ccf 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -53,6 +53,7 @@ from nerfstudio.model_components.renderers import ( from nerfstudio.model_components.scene_colliders import AABBBoxCollider from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import colormaps, colors, misc +from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig @dataclass @@ -89,6 +90,8 @@ class TensoRFModelConfig(ModelConfig): tensorf_encoding: Literal["triplane", "vm", "cp"] = "vm" regularization: Literal["none", "l1", "tv"] = "l1" """Regularization method used in tensorf paper""" + camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="SO3xR3") + """Config of the camera optimizer to use""" background_color: Literal["random", "last_sample", "black", "white"] = "white" """Whether to randomize the background color.""" @@ -256,6 +259,11 @@ class TensoRFModel(Model): if self.config.tensorf_encoding == "cp" and self.config.regularization == "tv": raise RuntimeError("TV reg not supported for CP decomposition") + # (optional) camera optimizer + self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup( + num_cameras=self.num_train_data, device="cpu" + ) + def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} @@ -267,11 +275,14 @@ class TensoRFModel(Model): param_groups["encodings"] = list(self.field.color_encoding.parameters()) + list( self.field.density_encoding.parameters() ) + self.camera_optimizer.get_param_groups(param_groups=param_groups) return param_groups def get_outputs(self, ray_bundle: RayBundle): # uniform sampling + if self.training: + self.camera_optimizer.apply_to_raybundle(ray_bundle) ray_samples_uniform = self.sampler_uniform(ray_bundle) dens = self.field.get_density(ray_samples_uniform) weights = ray_samples_uniform.get_weights(dens) @@ -334,6 +345,8 @@ class TensoRFModel(Model): else: raise ValueError(f"Regularization {self.config.regularization} not supported") + self.camera_optimizer.get_loss_dict(loss_dict) + loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) return loss_dict @@ -367,5 +380,7 @@ class TensoRFModel(Model): "ssim": float(ssim.item()), "lpips": float(lpips.item()), } + self.camera_optimizer.get_metrics_dict(metrics_dict) + images_dict = {"img": combined_rgb, "accumulation": acc, "depth": depth} return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 5e202b06c..10aca0e70 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -45,6 +45,7 @@ from nerfstudio.data.datamanagers.base_datamanager import ( DataManagerConfig, VanillaDataManager, ) +from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import profiler @@ -298,18 +299,6 @@ class VanillaPipeline(Pipeline): ray_bundle, batch = self.datamanager.next_train(step) model_outputs = self._model(ray_bundle) # train distributed data parallel model if world_size > 1 metrics_dict = self.model.get_metrics_dict(model_outputs, batch) - - if self.config.datamanager.camera_optimizer is not None: - camera_opt_param_group = self.config.datamanager.camera_optimizer.param_group - if camera_opt_param_group in self.datamanager.get_param_groups(): - # Report the camera optimization metrics - metrics_dict["camera_opt_translation"] = ( - self.datamanager.get_param_groups()[camera_opt_param_group][0].data[:, :3].norm() - ) - metrics_dict["camera_opt_rotation"] = ( - self.datamanager.get_param_groups()[camera_opt_param_group][0].data[:, 3:].norm() - ) - loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) return model_outputs, loss_dict, metrics_dict @@ -372,7 +361,7 @@ class VanillaPipeline(Pipeline): """ self.eval() metrics_dict_list = [] - assert isinstance(self.datamanager, VanillaDataManager) + assert isinstance(self.datamanager, (VanillaDataManager, ParallelDataManager)) num_images = len(self.datamanager.fixed_indices_eval_dataloader) with Progress( TextColumn("[progress.description]{task.description}"), diff --git a/nerfstudio/utils/tensor_dataclass.py b/nerfstudio/utils/tensor_dataclass.py index 0ebe7b5ce..a2b8d1dad 100644 --- a/nerfstudio/utils/tensor_dataclass.py +++ b/nerfstudio/utils/tensor_dataclass.py @@ -261,6 +261,14 @@ class TensorDataclass: """ return self._apply_fn_to_fields(lambda x: x.to(device)) + def pin_memory(self: TensorDataclassT) -> TensorDataclassT: + """Pins the tensor dataclass memory + + Returns: + TensorDataclass: A new TensorDataclass with the same data but pinned. + """ + return self._apply_fn_to_fields(lambda x: x.pin_memory()) + def _apply_fn_to_fields( self: TensorDataclassT, fn: Callable,