Files
nerfstudio/projects-lerf/lerf/data/lerf_datamanager.py
T
2023-04-02 18:35:21 +00:00

134 lines
5.2 KiB
Python

# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Datamanager.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Type, Union, Literal
import os.path as osp
import yaml
import torch
from rich.progress import Console
from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate
from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes
from nerfstudio.model_components.ray_generators import RayGenerator
from nerfstudio.utils.misc import IterableWrapper
from nerfstudio.cameras.rays import RayBundle
CONSOLE = Console(width=120)
from nerfstudio.data.datamanagers.base_datamanager import (
VanillaDataManager,
VanillaDataManagerConfig,
)
from lerf.data.utils.dino_dataloader import DinoDataloader
from lerf.data.utils.patched_dataloader import PatchedDataloader
from lerf.encoders.image_encoder import BaseImageEncoder
@dataclass
class LERFDataManagerConfig(VanillaDataManagerConfig):
_target: Type = field(default_factory=lambda: LERFDataManager)
patch_tile_size_range: Tuple[int, int] = (0.05, 0.5)
patch_tile_size_res: int = 7
patch_stride_scaler: float = 0.5
class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method
"""Basic stored data manager implementation.
This is pretty much a port over from our old dataloading utilities, and is a little jank
under the hood. We may clean this up a little bit under the hood with more standard dataloading
components that can be strung together, but it can be just used as a black box for now since
only the constructor is likely to change in the future, or maybe passing in step number to the
next_train and next_eval functions.
Args:
config: the DataManagerConfig used to instantiate class
"""
config: LERFDataManagerConfig
def __init__(
self,
config: LERFDataManagerConfig,
device: Union[torch.device, str] = "cpu",
test_mode: Literal["test", "val", "inference"] = "val",
world_size: int = 1,
local_rank: int = 0,
**kwargs, # pylint: disable=unused-argument
):
super().__init__(
config=config, device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank, **kwargs
)
self.image_encoder: BaseImageEncoder = kwargs["image_encoder"]
images = [self.train_dataset[i]["image"].permute(2, 0, 1)[None, ...] for i in range(len(self.train_dataset))]
images = torch.cat(images)
cache_dir = f"outputs/{self.config.dataparser.data.name}"
# clip_cache_path, dino_cache_path = self._search_cache(cache_dir)
clip_cache_path = osp.join(cache_dir, "clip")
dino_cache_path = osp.join(cache_dir, "dino.npy")
self.clip_interpolator = PatchedDataloader(
image_list=images,
device=self.device,
cfg=
{
"tile_size_range": (0.05, 0.5),
"tile_size_res": 7,
"stride_scaler": 0.5,
"image_shape": tuple(images.shape[2:4])
},
cache_path=clip_cache_path,
model=self.image_encoder,
)
self.dino_dataloader = DinoDataloader(
image_list=images, device=self.device,
cfg={"image_shape": tuple(images.shape[2:4])},
cache_path=dino_cache_path
)
# def _search_cache(self, cache_path, prop_dict):
# if osp.exists(cache_path):
# # load yaml file
# with open(cache_path, "r") as f:
# cache = yaml.load(f, Loader=yaml.FullLoader)
# for k, v in cache.items():
# if k in self.config:
# self.config[k] = v
# else:
# ...
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
"""Returns the next batch of data from the train dataloader."""
self.train_count += 1
image_batch = next(self.iter_train_image_dataloader)
assert self.train_pixel_sampler is not None
batch = self.train_pixel_sampler.sample(image_batch)
ray_indices = batch["indices"]
ray_bundle = self.train_ray_generator(ray_indices)
batch["clip"], clip_scale = self.clip_interpolator(ray_indices)
batch["dino"] = self.dino_dataloader(ray_indices)
ray_bundle.metadata["clip_scales"] = clip_scale
# assume all cameras have the same focal length and image width
ray_bundle.metadata["fx"] = self.train_dataset.cameras[0].fx.item()
ray_bundle.metadata["width"] = self.train_dataset.cameras[0].width.item()
return ray_bundle, batch