mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 23:25:39 +08:00
282 lines
13 KiB
Python
282 lines
13 KiB
Python
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
Data manager that outputs cameras / images instead of raybundles
|
|
|
|
Good for things like gaussian splatting which require full cameras instead of the standard ray
|
|
paradigm
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import random
|
|
import sys
|
|
from abc import abstractmethod
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass, field
|
|
from functools import cached_property
|
|
from pathlib import Path
|
|
from typing import (Any, Callable, Dict, ForwardRef, Generic, List, Literal,
|
|
Optional, Tuple, Type, Union, cast, get_args, get_origin)
|
|
|
|
import torch
|
|
from rich.progress import Console
|
|
from torch import Tensor, nn
|
|
from torch.nn import Parameter
|
|
from torch.utils.data.distributed import DistributedSampler
|
|
from typing_extensions import Literal, TypeVar
|
|
|
|
from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig
|
|
from nerfstudio.cameras.cameras import Cameras, CameraType
|
|
from nerfstudio.cameras.rays import RayBundle
|
|
from nerfstudio.configs.base_config import InstantiateConfig
|
|
from nerfstudio.configs.config_utils import to_immutable_dict
|
|
from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion
|
|
from nerfstudio.data.datamanagers.base_datamanager import (DataManager,
|
|
DataManagerConfig)
|
|
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
|
|
from nerfstudio.data.dataparsers.blender_dataparser import \
|
|
BlenderDataParserConfig
|
|
from nerfstudio.data.datasets.base_dataset import InputDataset
|
|
from nerfstudio.data.pixel_samplers import (PatchPixelSamplerConfig,
|
|
PixelSampler, PixelSamplerConfig)
|
|
from nerfstudio.data.scene_box import SceneBox
|
|
from nerfstudio.data.utils.dataloaders import (CacheDataloader,
|
|
FixedIndicesEvalDataloader,
|
|
RandIndicesEvalDataloader)
|
|
from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate
|
|
from nerfstudio.engine.callbacks import (TrainingCallback,
|
|
TrainingCallbackAttributes)
|
|
from nerfstudio.model_components.ray_generators import RayGenerator
|
|
from nerfstudio.utils.misc import IterableWrapper, get_orig_class
|
|
from nerfstudio.utils.rich_utils import CONSOLE
|
|
|
|
CONSOLE = Console(width=120)
|
|
|
|
@dataclass
|
|
class FullImageDatamanagerConfig(DataManagerConfig):
|
|
_target: Type = field(default_factory=lambda: FullImageDatamanager)
|
|
dataparser: AnnotatedDataParserUnion = BlenderDataParserConfig()
|
|
camera_res_scale_factor: float = 1.0
|
|
"""The scale factor for scaling spatial data such as images, mask, semantics
|
|
along with relevant information about camera intrinsics
|
|
"""
|
|
eval_num_images_to_sample_from: int = -1
|
|
"""Number of images to sample during eval iteration."""
|
|
eval_num_times_to_repeat_images: int = -1
|
|
"""When not evaluating on all images, number of iterations before picking
|
|
new images. If -1, never pick new images."""
|
|
eval_image_indices: Optional[Tuple[int, ...]] = (0,)
|
|
"""Specifies the image indices to use during eval; if None, uses all."""
|
|
cache_images: Literal["no-cache", "cpu", "gpu"] = "cpu"
|
|
"""Whether to cache images in memory. If "numpy", caches as numpy arrays, if "torch", caches as torch tensors."""
|
|
|
|
TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset)
|
|
|
|
class FullImageDatamanager(DataManager, Generic[TDataset]):
|
|
"""
|
|
A datamanager that outputs full images and cameras instead of raybundles. This makes the
|
|
datamanager far more lightweight since we dont have to do ray generation for things like
|
|
gaussian splatting that dont require it.
|
|
|
|
TODO (jake-austin): Figure out why the dataparser doesnt show up as a subcommand, preventing us
|
|
from specifying nerfstudio-data instead of blender-data
|
|
"""
|
|
|
|
config: FullImageDatamanagerConfig
|
|
train_dataset: TDataset
|
|
eval_dataset: TDataset
|
|
|
|
def __init__(
|
|
self,
|
|
config: FullImageDatamanagerConfig,
|
|
device: Union[torch.device, str] = "cpu",
|
|
test_mode: Literal["test", "val", "inference"] = "val",
|
|
world_size: int = 1,
|
|
local_rank: int = 0,
|
|
**kwargs,
|
|
):
|
|
self.config = config
|
|
self.device = device
|
|
self.world_size = world_size
|
|
self.local_rank = local_rank
|
|
self.sampler = None
|
|
self.test_mode = test_mode
|
|
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
|
|
self.dataparser_config = self.config.dataparser
|
|
if self.config.data is not None:
|
|
self.config.dataparser.data = Path(self.config.data)
|
|
else:
|
|
self.config.data = self.config.dataparser.data
|
|
self.dataparser = self.dataparser_config.setup()
|
|
if test_mode == "inference":
|
|
self.dataparser.downscale_factor = 1 # Avoid opening images
|
|
self.includes_time = self.dataparser.includes_time
|
|
|
|
self.train_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split="train")
|
|
self.train_dataset = self.create_train_dataset()
|
|
self.eval_dataset = self.create_eval_dataset()
|
|
self.cached_train, self.cached_eval = self.cache_images(self.config.cache_images)
|
|
self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device
|
|
if self.config.masks_on_gpu is True:
|
|
self.exclude_batch_keys_from_device.remove("mask")
|
|
if self.config.images_on_gpu is True:
|
|
self.exclude_batch_keys_from_device.remove("image")
|
|
|
|
# Some logic to make sure we sample every camera in equal amounts
|
|
self.train_unseen_cameras = [i for i in range(len(self.train_dataset))]
|
|
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
|
|
assert len(self.train_unseen_cameras) > 0, "No data found in dataset"
|
|
|
|
super().__init__()
|
|
|
|
|
|
def cache_images(self, cache_images_option):
|
|
if cache_images_option == "no-cache":
|
|
cached_train = {}
|
|
cached_eval = {}
|
|
elif cache_images_option == "cpu":
|
|
cached_train = [self.train_dataset.get_data(i) for i in range(len(self.train_dataset))]
|
|
cached_eval = [self.eval_dataset.get_data(i) for i in range(len(self.eval_dataset))]
|
|
elif cache_images_option == "gpu":
|
|
cached_train = [self.train_dataset.get_data(i) for i in range(len(self.train_dataset))]
|
|
cached_eval = [self.eval_dataset.get_data(i) for i in range(len(self.eval_dataset))]
|
|
for cache in cached_train:
|
|
cache["image"] = cache["image"].to(self.device)
|
|
if "mask" in cache:
|
|
cache["mask"] = cache["mask"].to(self.device)
|
|
for cache in cached_eval:
|
|
cache["image"] = cache["image"].to(self.device)
|
|
if "mask" in cache:
|
|
cache["mask"] = cache["mask"].to(self.device)
|
|
else:
|
|
raise ValueError(f"Unknown cache_images option {cache_images_option}")
|
|
|
|
return cached_train, cached_eval
|
|
|
|
def create_train_dataset(self) -> TDataset:
|
|
"""Sets up the data loaders for training"""
|
|
return self.dataset_type(
|
|
dataparser_outputs=self.train_dataparser_outputs,
|
|
scale_factor=self.config.camera_res_scale_factor,
|
|
)
|
|
|
|
def create_eval_dataset(self) -> TDataset:
|
|
"""Sets up the data loaders for evaluation"""
|
|
return self.dataset_type(
|
|
dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split),
|
|
scale_factor=self.config.camera_res_scale_factor,
|
|
)
|
|
|
|
@cached_property
|
|
def dataset_type(self) -> Type[TDataset]:
|
|
"""Returns the dataset type passed as the generic argument"""
|
|
default: Type[TDataset] = cast(TDataset, TDataset.__default__) # type: ignore
|
|
orig_class: Type[FullImageDatamanager] = get_orig_class(self, default=None) # type: ignore
|
|
if type(self) is FullImageDatamanager and orig_class is None:
|
|
return default
|
|
if orig_class is not None and get_origin(orig_class) is FullImageDatamanager:
|
|
return get_args(orig_class)[0]
|
|
|
|
# For inherited classes, we need to find the correct type to instantiate
|
|
for base in getattr(self, "__orig_bases__", []):
|
|
if get_origin(base) is FullImageDatamanager:
|
|
for value in get_args(base):
|
|
if isinstance(value, ForwardRef):
|
|
if value.__forward_evaluated__:
|
|
value = value.__forward_value__
|
|
elif value.__forward_module__ is None:
|
|
value.__forward_module__ = type(self).__module__
|
|
value = getattr(value, "_evaluate")(None, None, set())
|
|
assert isinstance(value, type)
|
|
if issubclass(value, InputDataset):
|
|
return cast(Type[TDataset], value)
|
|
return default
|
|
|
|
def get_datapath(self) -> Path:
|
|
return self.config.dataparser.data
|
|
|
|
def setup_train(self):
|
|
"""Sets up the data loaders for training"""
|
|
|
|
def setup_eval(self):
|
|
"""Sets up the data loader for evaluation"""
|
|
|
|
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
|
"""Get the param groups for the data manager.
|
|
Returns:
|
|
A list of dictionaries containing the data manager's param groups.
|
|
"""
|
|
return {}
|
|
|
|
def get_train_rays_per_batch(self):
|
|
# TODO (jake-austin): fix this to be the resolution of the last image rendered
|
|
return 800*800
|
|
|
|
def next_train(self, step: int) -> Tuple[Cameras, Dict]:
|
|
"""Returns the next training batch
|
|
|
|
Returns a Camera instead of raybundle"""
|
|
image_idx = self.train_unseen_cameras.pop(random.randint(0, len(self.train_unseen_cameras)-1))
|
|
# Make sure to re-populate the unseen cameras list if we have exhausted it
|
|
if len(self.train_unseen_cameras) == 0:
|
|
self.train_unseen_cameras = [i for i in range(len(self.train_dataset))]
|
|
|
|
if self.config.cache_images == "no-cache":
|
|
data = self.train_dataset.get_data(image_idx)
|
|
data["image"] = data["image"].to(self.device)
|
|
elif self.config.cache_images == "cpu":
|
|
data = self.cached_train[image_idx]
|
|
data["image"] = data["image"].to(self.device)
|
|
elif self.config.cache_images == "gpu":
|
|
data = self.cached_train[image_idx]
|
|
else:
|
|
raise ValueError(f"Unknown cache_images option {self.config.cache_images}")
|
|
|
|
|
|
assert len(self.train_dataset.cameras.shape) == 1, "Assumes single batch dimension"
|
|
camera = self.train_dataset.cameras[image_idx].to(self.device)
|
|
return camera, data
|
|
|
|
def next_eval(self, step: int) -> Tuple[Cameras, Dict]:
|
|
"""Returns the next evaluation batch
|
|
|
|
Returns a Camera instead of raybundle"""
|
|
image_idx = self.eval_unseen_cameras.pop(random.randint(0, len(self.eval_unseen_cameras)-1))
|
|
# Make sure to re-populate the unseen cameras list if we have exhausted it
|
|
if len(self.eval_unseen_cameras) == 0:
|
|
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
|
|
data = self.eval_dataset.get_data(image_idx)
|
|
data["image"] = data["image"].to(self.device)
|
|
assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension"
|
|
camera = self.eval_dataset.cameras[image_idx].to(self.device)
|
|
return camera, data
|
|
|
|
def next_eval_image(self, step: int) -> Tuple[int, Cameras, Dict]:
|
|
"""Returns the next evaluation batch
|
|
|
|
Returns a Camera instead of raybundle
|
|
|
|
TODO (jake-austin): Make sure this logic is consistent with the vanilladatamanager"""
|
|
image_idx = self.eval_unseen_cameras.pop(random.randint(0, len(self.eval_unseen_cameras)-1))
|
|
# Make sure to re-populate the unseen cameras list if we have exhausted it
|
|
if len(self.eval_unseen_cameras) == 0:
|
|
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
|
|
data = self.eval_dataset.get_data(image_idx)
|
|
data["image"] = data["image"].to(self.device)
|
|
assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension"
|
|
camera = self.eval_dataset.cameras[image_idx].to(self.device)
|
|
return image_idx, camera, data
|