diff --git a/docs/developer_guides/config.md b/docs/developer_guides/config.md index 29d093f9d..7c0c7c213 100644 --- a/docs/developer_guides/config.md +++ b/docs/developer_guides/config.md @@ -31,14 +31,8 @@ class NerfactoModelConfig(ModelConfig): _target: Type = field(default_factory=lambda: NerfactoModel) ... -class NerfactoModel(Model): - """Nerfacto model - - Args: - config: Nerfacto configuration to instantiate model - """ - - config: NerfactoModelConfig +class NerfactoModel(Model[NerfactoModelConfig]): + """Nerfacto model""" ... ``` diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md index 6b81803c2..6c645ac51 100644 --- a/docs/developer_guides/pipelines/dataparsers.md +++ b/docs/developer_guides/pipelines/dataparsers.md @@ -73,11 +73,9 @@ class NerfstudioDataParserConfig(DataParserConfig): """The fraction of images to use for training. The remaining images are for eval.""" @dataclass -class Nerfstudio(DataParser): +class Nerfstudio(DataParser[NerfstudioDataParserConfig]): """Nerfstudio DatasetParser""" - config: NerfstudioDataParserConfig - def _generate_dataparser_outputs(self, split="train"): meta = load_from_json(self.config.data / "transforms.json") image_filenames = [] diff --git a/docs/developer_guides/pipelines/models.md b/docs/developer_guides/pipelines/models.md index 8afa76d3a..8110d91b7 100644 --- a/docs/developer_guides/pipelines/models.md +++ b/docs/developer_guides/pipelines/models.md @@ -111,14 +111,8 @@ There are a lot of options! Thankfully, our config system makes this easy to han Furthermore, you have Python autocomplete and static checking working in your favor. At the top of every Model, we specify the config and then can easily pull of values throughout the implementation. Let's take a look at the beginning of the NerfactoModel implementation. ```python -class NerfactoModel(Model): - """Nerfacto model - - Args: - config: Nerfacto configuration to instantiate model - """ - - config: NerfactoModelConfig +class NerfactoModel(Model[NerfactoModelConfig]): + """Nerfacto model""" def populate_modules(self): """Set the fields and modules.""" diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 0c3875c5e..da4d95e2c 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -61,7 +61,7 @@ from nerfstudio.data.utils.dataloaders import ( FixedIndicesEvalDataloader, RandIndicesEvalDataloader, ) -from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate +from nerfstudio.data.utils.nerfstudio_collate import CollateFunction, nerfstudio_collate from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.model_components.ray_generators import RayGenerator from nerfstudio.utils.misc import IterableWrapper @@ -281,7 +281,7 @@ class VanillaDataManagerConfig(InstantiateConfig): camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from Record3D.""" - collate_fn = staticmethod(nerfstudio_collate) + collate_fn: CollateFunction = field(default_factory=lambda: nerfstudio_collate) """Specifies the collate function to use for the train and eval dataloaders.""" camera_res_scale_factor: float = 1.0 """The scale factor for scaling spatial data such as images, mask, semantics diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 89c7c4e21..66df055b4 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -20,7 +20,7 @@ import json from abc import abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, Generic, List, Optional, Type, TypeVar import torch from torchtyping import TensorType @@ -94,23 +94,26 @@ class DataparserOutputs: class DataParserConfig(cfg.InstantiateConfig): """Basic dataset config""" - _target: Type = field(default_factory=lambda: DataParser) + _target: Type = field(default_factory=lambda: DataParser[DataParserConfig]) """_target: target class to instantiate""" data: Path = Path() """Directory specifying location of data.""" +TDataParserConfig = TypeVar("TDataParserConfig", bound=DataParserConfig) + + @dataclass -class DataParser: +class DataParser(Generic[TDataParserConfig]): """A dataset. Args: config: datasetparser config containing all information needed to instantiate dataset """ - config: DataParserConfig + config: TDataParserConfig - def __init__(self, config: DataParserConfig): + def __init__(self, config: TDataParserConfig): super().__init__() self.config = config diff --git a/nerfstudio/data/dataparsers/blender_dataparser.py b/nerfstudio/data/dataparsers/blender_dataparser.py index f1044e826..37ccbe07f 100644 --- a/nerfstudio/data/dataparsers/blender_dataparser.py +++ b/nerfstudio/data/dataparsers/blender_dataparser.py @@ -49,13 +49,11 @@ class BlenderDataParserConfig(DataParserConfig): @dataclass -class Blender(DataParser): +class Blender(DataParser[BlenderDataParserConfig]): """Blender Dataset Some of this code comes from https://github.com/yenchenlin/nerf-pytorch/blob/master/load_blender.py#L37. """ - config: BlenderDataParserConfig - def __init__(self, config: BlenderDataParserConfig): super().__init__(config=config) self.data: Path = config.data diff --git a/nerfstudio/data/dataparsers/dnerf_dataparser.py b/nerfstudio/data/dataparsers/dnerf_dataparser.py index c26621e92..5d8d4db0e 100644 --- a/nerfstudio/data/dataparsers/dnerf_dataparser.py +++ b/nerfstudio/data/dataparsers/dnerf_dataparser.py @@ -49,11 +49,9 @@ class DNeRFDataParserConfig(DataParserConfig): @dataclass -class DNeRF(DataParser): +class DNeRF(DataParser[DNeRFDataParserConfig]): """DNeRF Dataset""" - config: DNeRFDataParserConfig - def __init__(self, config: DNeRFDataParserConfig): super().__init__(config=config) self.data: Path = config.data diff --git a/nerfstudio/data/dataparsers/dycheck_dataparser.py b/nerfstudio/data/dataparsers/dycheck_dataparser.py index 92cca35ff..6239fc780 100644 --- a/nerfstudio/data/dataparsers/dycheck_dataparser.py +++ b/nerfstudio/data/dataparsers/dycheck_dataparser.py @@ -198,11 +198,9 @@ class DycheckDataParserConfig(DataParserConfig): @dataclass -class Dycheck(DataParser): +class Dycheck(DataParser[DycheckDataParserConfig]): """Dycheck (https://arxiv.org/abs/2210.13445) Dataset `iphone` subset""" - config: DycheckDataParserConfig - def __init__(self, config: DycheckDataParserConfig): super().__init__(config=config) self.data: Path = config.data diff --git a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py index 2fdc2df85..a8281c536 100644 --- a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py +++ b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py @@ -51,11 +51,9 @@ class InstantNGPDataParserConfig(DataParserConfig): @dataclass -class InstantNGP(DataParser): +class InstantNGP(DataParser[InstantNGPDataParserConfig]): """Instant NGP Dataset""" - config: InstantNGPDataParserConfig - def _generate_dataparser_outputs(self, split="train"): if self.config.data.suffix == ".json": meta = load_from_json(self.config.data) diff --git a/nerfstudio/data/dataparsers/minimal_dataparser.py b/nerfstudio/data/dataparsers/minimal_dataparser.py index 2fa2218a4..506ccfc43 100644 --- a/nerfstudio/data/dataparsers/minimal_dataparser.py +++ b/nerfstudio/data/dataparsers/minimal_dataparser.py @@ -43,11 +43,9 @@ class MinimalDataParserConfig(DataParserConfig): @dataclass -class MinimalDataParser(DataParser): +class MinimalDataParser(DataParser[MinimalDataParserConfig]): """Minimal DatasetParser""" - config: MinimalDataParserConfig - def _generate_dataparser_outputs(self, split="train"): filepath = self.config.data / f"{split}.npz" data = np.load(filepath, allow_pickle=True) diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index fd0d49830..b6e78a0d8 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -67,10 +67,9 @@ class NerfstudioDataParserConfig(DataParserConfig): @dataclass -class Nerfstudio(DataParser): +class Nerfstudio(DataParser[NerfstudioDataParserConfig]): """Nerfstudio DatasetParser""" - config: NerfstudioDataParserConfig downscale_factor: Optional[int] = None def _generate_dataparser_outputs(self, split="train"): diff --git a/nerfstudio/data/dataparsers/nuscenes_dataparser.py b/nerfstudio/data/dataparsers/nuscenes_dataparser.py index 6b35cf356..da15208d2 100644 --- a/nerfstudio/data/dataparsers/nuscenes_dataparser.py +++ b/nerfstudio/data/dataparsers/nuscenes_dataparser.py @@ -80,15 +80,15 @@ class NuScenesDataParserConfig(DataParserConfig): @dataclass -class NuScenes(DataParser): +class NuScenes(DataParser[NuScenesDataParserConfig]): """NuScenes DatasetParser""" - config: NuScenesDataParserConfig - def _generate_dataparser_outputs(self, split="train"): # pylint: disable=too-many-statements - nusc = NuScenesDatabase(version=self.config.version, dataroot=self.config.data_dir, verbose=self.config.verbose) + nusc = NuScenesDatabase( + version=self.config.version, dataroot=str(self.config.data_dir), verbose=self.config.verbose + ) cameras = ["CAM_" + camera for camera in self.config.cameras] assert ( diff --git a/nerfstudio/data/dataparsers/phototourism_dataparser.py b/nerfstudio/data/dataparsers/phototourism_dataparser.py index 5b5287d3d..68ea89326 100644 --- a/nerfstudio/data/dataparsers/phototourism_dataparser.py +++ b/nerfstudio/data/dataparsers/phototourism_dataparser.py @@ -69,13 +69,11 @@ class PhototourismDataParserConfig(DataParserConfig): @dataclass -class Phototourism(DataParser): +class Phototourism(DataParser[PhototourismDataParserConfig]): """Phototourism dataset. This is based on https://github.com/kwea123/nerf_pl/blob/nerfw/datasets/phototourism.py and uses colmap's utils file to read the poses. """ - config: PhototourismDataParserConfig - def __init__(self, config: PhototourismDataParserConfig): super().__init__(config=config) self.data: Path = config.data diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py index f25c8f3e7..460a4a709 100644 --- a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py @@ -59,11 +59,9 @@ class SDFStudioDataParserConfig(DataParserConfig): @dataclass -class SDFStudio(DataParser): +class SDFStudio(DataParser[SDFStudioDataParserConfig]): """SDFStudio Dataset""" - config: SDFStudioDataParserConfig - def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements # load meta data meta = load_from_json(self.config.data / "meta_data.json") diff --git a/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py b/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py index b6906ce22..426fe6d21 100644 --- a/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py +++ b/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py @@ -59,11 +59,9 @@ class Sitcoms3DDataParserConfig(DataParserConfig): @dataclass -class Sitcoms3D(DataParser): +class Sitcoms3D(DataParser[Sitcoms3DDataParserConfig]): """Sitcoms3D Dataset""" - config: Sitcoms3DDataParserConfig - def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements cameras_json = load_from_json(self.config.data / "cameras.json") diff --git a/nerfstudio/data/utils/data_utils.py b/nerfstudio/data/utils/data_utils.py index f04fb368c..40fe0aa61 100644 --- a/nerfstudio/data/utils/data_utils.py +++ b/nerfstudio/data/utils/data_utils.py @@ -45,7 +45,7 @@ def get_semantics_and_mask_tensors_from_path( If no mask is required - use mask_indices = [] """ if isinstance(mask_indices, List): - mask_indices = torch.tensor(mask_indices, dtype="int64").view(1, 1, -1) + mask_indices = torch.tensor(mask_indices, dtype=torch.int64).view(1, 1, -1) pil_image = Image.open(filepath) if scale_factor != 1.0: width, height = pil_image.size diff --git a/nerfstudio/data/utils/dataloaders.py b/nerfstudio/data/utils/dataloaders.py index c4e155102..7bf58a4bb 100644 --- a/nerfstudio/data/utils/dataloaders.py +++ b/nerfstudio/data/utils/dataloaders.py @@ -16,22 +16,22 @@ Code for sampling images from a dataset of images. """ -# for multithreading import concurrent.futures import multiprocessing import random + +# for multithreading from abc import abstractmethod -from typing import Dict, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import torch from rich.progress import Console, track -from torch.utils.data import Dataset -from torch.utils.data.dataloader import DataLoader +from torch.utils.data.dataloader import DataLoader, Dataset from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datasets.base_dataset import InputDataset -from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate +from nerfstudio.data.utils.nerfstudio_collate import CollateFunction, nerfstudio_collate from nerfstudio.utils.misc import get_dict_to_torch CONSOLE = Console(width=120) @@ -55,7 +55,7 @@ class CacheDataloader(DataLoader): num_images_to_sample_from: int = -1, num_times_to_repeat_images: int = -1, device: Union[torch.device, str] = "cpu", - collate_fn=nerfstudio_collate, + collate_fn: CollateFunction = nerfstudio_collate, **kwargs, ): self.dataset = dataset @@ -91,7 +91,7 @@ class CacheDataloader(DataLoader): def __getitem__(self, idx): return self.dataset.__getitem__(idx) - def _get_batch_list(self): + def _get_batch_list(self) -> List[Dict[str, Any]]: """Returns a list of batches from the dataset attribute.""" indices = random.sample(range(len(self.dataset)), k=self.num_images_to_sample_from) @@ -123,6 +123,7 @@ class CacheDataloader(DataLoader): while True: if self.cache_all_images: collated_batch = self.cached_collated_batch + assert collated_batch is not None elif self.first_time or ( self.num_times_to_repeat_images != -1 and self.num_repeated >= self.num_times_to_repeat_images ): @@ -134,6 +135,7 @@ class CacheDataloader(DataLoader): self.first_time = False else: collated_batch = self.cached_collated_batch + assert collated_batch is not None self.num_repeated += 1 yield collated_batch diff --git a/nerfstudio/data/utils/nerfstudio_collate.py b/nerfstudio/data/utils/nerfstudio_collate.py index ffcd989b4..8e35d72fd 100644 --- a/nerfstudio/data/utils/nerfstudio_collate.py +++ b/nerfstudio/data/utils/nerfstudio_collate.py @@ -19,11 +19,12 @@ Custom collate function that includes cases for nerfstudio types. import collections import collections.abc import re -from typing import Callable, Dict, Union +from typing import Callable, Dict, List, Union import torch import torch.utils.data from torch._six import string_classes +from typing_extensions import Protocol, TypeVar from nerfstudio.cameras.cameras import Cameras @@ -34,9 +35,17 @@ NERFSTUDIO_COLLATE_ERR_MSG_FORMAT = ( np_str_obj_array_pattern = re.compile(r"[SaUO]") -def nerfstudio_collate( - batch, extra_mappings: Union[Dict[type, Callable], None] = None -): # pylint: disable=too-many-return-statements +T = TypeVar("T") + + +class CollateFunction(Protocol): # pylint: disable=too-few-public-methods,missing-class-docstring + def __call__(self, batch: List[T], extra_mappings: Union[Dict[type, Callable], None] = None) -> T: + raise NotImplementedError() + + +def nerfstudio_collate( # pylint: disable=too-many-return-statements + batch: List[T], extra_mappings: Union[Dict[type, Callable], None] = None +) -> T: r""" This is the default pytorch collate function, but with support for nerfstudio types. All documentation below is copied straight over from pytorch's default_collate function, python version 3.8.13, diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 3cc5fa2ec..aa2181b38 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -21,7 +21,7 @@ from __future__ import annotations from abc import abstractmethod from collections import defaultdict from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar import torch from torch import nn @@ -52,7 +52,10 @@ class ModelConfig(InstantiateConfig): """specifies number of rays per chunk during eval""" -class Model(nn.Module): +TConfig = TypeVar("TConfig", bound=ModelConfig) + + +class Model(nn.Module, Generic[TConfig]): """Model class Where everything (Fields, Optimizers, Samplers, Visualization, etc) is linked together. This should be subclassed for custom NeRF model. @@ -62,11 +65,11 @@ class Model(nn.Module): scene_box: dataset scene box """ - config: ModelConfig + config: TConfig def __init__( self, - config: ModelConfig, + config: TConfig, scene_box: SceneBox, num_train_data: int, **kwargs, @@ -101,6 +104,7 @@ class Model(nn.Module): # NOTE: call `super().populate_modules()` in subclasses if self.config.enable_collider: + assert self.config.collider_params is not None self.collider = NearFarCollider( near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"] ) diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index ace4b77c6..21746a683 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -81,19 +81,15 @@ class InstantNGPModelConfig(ModelConfig): """The color that is given to untrained areas.""" -class NGPModel(Model): +class NGPModel(Model[InstantNGPModelConfig]): """Instant NGP model Args: config: instant NGP configuration to instantiate model """ - config: InstantNGPModelConfig field: TCNNInstantNGPField - def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None: - super().__init__(config=config, **kwargs) - def populate_modules(self): """Set the fields and modules.""" super().populate_modules() diff --git a/nerfstudio/models/mipnerf.py b/nerfstudio/models/mipnerf.py index 31a337d02..992390351 100644 --- a/nerfstudio/models/mipnerf.py +++ b/nerfstudio/models/mipnerf.py @@ -36,25 +36,18 @@ from nerfstudio.model_components.renderers import ( DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.models.base_model import Model +from nerfstudio.models.vanilla_nerf import VanillaModelConfig from nerfstudio.utils import colormaps, colors, misc -class MipNerfModel(Model): +class MipNerfModel(Model[VanillaModelConfig]): """mip-NeRF model Args: config: MipNerf configuration to instantiate model """ - def __init__( - self, - config: ModelConfig, - **kwargs, - ) -> None: - self.field = None - super().__init__(config=config, **kwargs) - def populate_modules(self): """Set the fields and modules""" super().populate_modules() diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 53173640a..543b3ad60 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -130,15 +130,13 @@ class NerfactoModelConfig(ModelConfig): """Whether to disable scene contraction or not.""" -class NerfactoModel(Model): +class NerfactoModel(Model[NerfactoModelConfig]): """Nerfacto model Args: config: Nerfacto configuration to instantiate model """ - config: NerfactoModelConfig - def populate_modules(self): """Set the fields and modules.""" super().populate_modules() diff --git a/nerfstudio/models/semantic_nerfw.py b/nerfstudio/models/semantic_nerfw.py index 4324e1eeb..a51bbe7d1 100644 --- a/nerfstudio/models/semantic_nerfw.py +++ b/nerfstudio/models/semantic_nerfw.py @@ -65,15 +65,13 @@ class SemanticNerfWModelConfig(NerfactoModelConfig): pass_semantic_gradients: bool = False -class SemanticNerfWModel(Model): +class SemanticNerfWModel(Model[SemanticNerfWModelConfig]): """Nerfacto model Args: config: Nerfacto configuration to instantiate model """ - config: SemanticNerfWModelConfig - def __init__(self, config: SemanticNerfWModelConfig, metadata: Dict, **kwargs) -> None: assert "semantics" in metadata.keys() and isinstance(metadata["semantics"], Semantics) self.semantics = metadata["semantics"] diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index 3b27354d1..5784a763f 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -81,7 +81,7 @@ class TensoRFModelConfig(ModelConfig): tensorf_encoding: Literal["triplane", "vm", "cp"] = "vm" -class TensoRFModel(Model): +class TensoRFModel(Model[TensoRFModelConfig]): """TensoRF Model Args: diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py index 53e838891..034d465d5 100644 --- a/nerfstudio/models/vanilla_nerf.py +++ b/nerfstudio/models/vanilla_nerf.py @@ -60,27 +60,13 @@ class VanillaModelConfig(ModelConfig): """Parameters to instantiate temporal distortion with""" -class NeRFModel(Model): +class NeRFModel(Model[VanillaModelConfig]): """Vanilla NeRF model Args: config: Basic NeRF configuration to instantiate model """ - def __init__( - self, - config: VanillaModelConfig, - **kwargs, - ) -> None: - self.field_coarse = None - self.field_fine = None - self.temporal_distortion = None - - super().__init__( - config=config, - **kwargs, - ) - def populate_modules(self): """Set the fields and modules""" super().populate_modules() @@ -120,6 +106,7 @@ class NeRFModel(Model): self.ssim = structural_similarity_index_measure self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + self.temporal_distortion = None if getattr(self.config, "enable_temporal_distortion", False): params = self.config.temporal_distortion_params kind = params.pop("kind") diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index c1cac0369..063237abd 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -21,7 +21,7 @@ import typing from abc import abstractmethod from dataclasses import dataclass, field from time import time -from typing import Any, Dict, List, Mapping, Optional, Type, Union, cast +from typing import Any, Dict, Generic, List, Mapping, Optional, Type, Union, cast import torch import torch.distributed as dist @@ -35,7 +35,7 @@ from rich.progress import ( from torch import nn from torch.nn import Parameter from torch.nn.parallel import DistributedDataParallel as DDP -from typing_extensions import Literal +from typing_extensions import Literal, TypeVar from nerfstudio.configs import base_config as cfg from nerfstudio.data.datamanagers.base_datamanager import ( @@ -174,11 +174,13 @@ class Pipeline(nn.Module): step: training step of the loaded checkpoint """ + @abstractmethod def get_training_callbacks( self, training_callback_attributes: TrainingCallbackAttributes ) -> List[TrainingCallback]: """Returns the training callbacks from both the Dataloader and the Model.""" + @abstractmethod def get_param_groups(self) -> Dict[str, List[Parameter]]: """Get the param groups for the pipeline. @@ -199,7 +201,10 @@ class VanillaPipelineConfig(cfg.InstantiateConfig): """specifies the model config""" -class VanillaPipeline(Pipeline): +TPipelineConfig = TypeVar("TPipelineConfig", bound=VanillaPipelineConfig, default=VanillaPipelineConfig) + + +class VanillaPipeline(Pipeline, Generic[TPipelineConfig]): """The pipeline class for the vanilla nerf setup of multiple cameras for one or a few scenes. config: configuration to instantiate pipeline @@ -216,9 +221,11 @@ class VanillaPipeline(Pipeline): model: The model that will be used """ + config: TPipelineConfig + def __init__( self, - config: VanillaPipelineConfig, + config: TPipelineConfig, device: str, test_mode: Literal["test", "val", "inference"] = "val", world_size: int = 1, diff --git a/nerfstudio/pipelines/dynamic_batch.py b/nerfstudio/pipelines/dynamic_batch.py index 293ea1706..0bec74f81 100644 --- a/nerfstudio/pipelines/dynamic_batch.py +++ b/nerfstudio/pipelines/dynamic_batch.py @@ -37,12 +37,11 @@ class DynamicBatchPipelineConfig(VanillaPipelineConfig): """The maximum number of samples to be placed along a ray.""" -class DynamicBatchPipeline(VanillaPipeline): +class DynamicBatchPipeline(VanillaPipeline[DynamicBatchPipelineConfig]): """Pipeline with logic for changing the number of rays per batch.""" # pylint: disable=abstract-method - config: DynamicBatchPipelineConfig datamanager: VanillaDataManager dynamic_num_rays_per_batch: int @@ -90,6 +89,7 @@ class DynamicBatchPipeline(VanillaPipeline): # add the number of rays assert "num_rays_per_batch" not in metrics_dict + assert self.datamanager.train_pixel_sampler is not None metrics_dict["num_rays_per_batch"] = torch.tensor(self.datamanager.train_pixel_sampler.num_rays_per_batch) return model_outputs, loss_dict, metrics_dict @@ -99,6 +99,7 @@ class DynamicBatchPipeline(VanillaPipeline): # add the number of rays assert "num_rays_per_batch" not in metrics_dict + assert self.datamanager.eval_pixel_sampler is not None metrics_dict["num_rays_per_batch"] = torch.tensor(self.datamanager.eval_pixel_sampler.num_rays_per_batch) return model_outputs, loss_dict, metrics_dict diff --git a/nerfstudio/utils/misc.py b/nerfstudio/utils/misc.py index 88a76e8fc..edd71ea65 100644 --- a/nerfstudio/utils/misc.py +++ b/nerfstudio/utils/misc.py @@ -20,9 +20,12 @@ Miscellaneous helper code. from typing import Any, Callable, Dict, List, Optional, Union import torch +from typing_extensions import TypeVar + +T = TypeVar("T") -def get_dict_to_torch(stuff: Any, device: Union[torch.device, str] = "cpu", exclude: Optional[List[str]] = None): +def get_dict_to_torch(stuff: T, device: Union[torch.device, str] = "cpu", exclude: Optional[List[str]] = None) -> T: """Set everything in the dict to the specified torch device. Args: