Basic working, still needs to be profiled and tuned, some preliminary TODOs labeled 'jake-austin' need to be addressed

This commit is contained in:
Jake Austin jake-austin@berkeley.edu committed 2023-09-12 03:15:11 +00:00
1 parent 749059d4fb
commit bb90b94fb0
11 files changed
+934 -62

No files matched your search

@@ -33,7 +33,7 @@ from torch.utils.data.distributed import DistributedSampler
from typing_extensions import TypeVar
from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig
from nerfstudio.cameras.cameras import CameraType
from nerfstudio.cameras.cameras import Cameras, CameraType
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.configs.base_config import InstantiateConfig
from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion
@@ -115,7 +115,7 @@ class DataManager(nn.Module):
To get data, use the next_train and next_eval functions.
This data manager's next_train and next_eval methods will return 2 things:
1. A Raybundle: This will contain the rays we are sampling, with latents and
1. A rays: This will contain the rays/camera we are sampling, with latents and
conditionals attached (everything needed at inference)
2. A "batch" of auxiliary information: This will contain the mask, the ground truth
pixels, etc needed to actually train, score, etc the model
@@ -230,7 +230,7 @@ class DataManager(nn.Module):
"""Sets up the data manager for evaluation"""
@abstractmethod
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
def next_train(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]:
"""Returns the next batch of data from the train data manager.
Args:
@@ -242,25 +242,25 @@ class DataManager(nn.Module):
raise NotImplementedError
@abstractmethod
def next_eval(self, step: int) -> Tuple[RayBundle, Dict]:
def next_eval(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]:
"""Returns the next batch of data from the eval data manager.
Args:
step: the step number of the eval image to retrieve
Returns:
A tuple of the ray bundle for the image, and a dictionary of additional batch information
A tuple of the ray/camera for the image, and a dictionary of additional batch information
such as the groundtruth image.
"""
raise NotImplementedError
@abstractmethod
def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]:
def next_eval_image(self, step: int) -> Tuple[int, Union[RayBundle, Cameras], Dict]:
"""Retrieve the next eval image.
Args:
step: the step number of the eval image to retrieve
Returns:
A tuple of the step number, the ray bundle for the image, and a dictionary of
A tuple of the step number, the ray/camera for the image, and a dictionary of
additional batch information such as the groundtruth image.
"""
raise NotImplementedError
@@ -297,7 +297,7 @@ class DataManager(nn.Module):
@dataclass
class VanillaDataManagerConfig(DataManagerConfig):
"""A basic data manager"""
"""A basic data manager for a ray-based model"""
_target: Type = field(default_factory=lambda: VanillaDataManager)
"""Target class to instantiate."""
@@ -21,6 +21,7 @@ paradigm
from __future__ import annotations
import random
import sys
from abc import abstractmethod
from collections import defaultdict
@@ -29,7 +30,7 @@ from functools import cached_property
from pathlib import Path
from typing import (Any, Callable, Dict, ForwardRef, Generic, List, Literal,
Optional, Tuple, Type, Union, cast, get_args, get_origin)
import random
import torch
from rich.progress import Console
from torch import Tensor, nn
@@ -87,6 +88,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
A datamanager that outputs full images and cameras instead of raybundles. This makes the
datamanager far more lightweight since we dont have to do ray generation for things like
gaussian splatting that dont require it.
TODO (jake-austin): Figure out why the dataparser doesnt show up as a subcommand, preventing us
from specifying nerfstudio-data instead of blender-data
"""
config: FullImageDatamanagerConfig
@@ -191,6 +195,10 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
"""
return {}
def get_train_rays_per_batch(self):
# TODO (jake-austin): fix this to be the resolution of the last image rendered
return 800*800
def next_train(self, step: int) -> Tuple[Cameras, Dict]:
"""Returns the next training batch
@@ -200,8 +208,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
if len(self.train_unseen_cameras) == 0:
self.train_unseen_cameras = [i for i in range(len(self.train_dataset))]
data = self.train_dataset.get_data(image_idx)
data["image"] = data["image"].to(self.device)
assert len(self.train_dataset.cameras.shape) == 1, "Assumes single batch dimension"
camera = self.train_dataset.cameras[image_idx]
camera = self.train_dataset.cameras[image_idx].to(self.device)
return camera, data
def next_eval(self, step: int) -> Tuple[Cameras, Dict]:
@@ -213,8 +222,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
if len(self.eval_unseen_cameras) == 0:
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
data = self.eval_dataset.get_data(image_idx)
data["image"] = data["image"].to(self.device)
assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension"
camera = self.eval_dataset.cameras[image_idx]
camera = self.eval_dataset.cameras[image_idx].to(self.device)
return camera, data
def next_eval_image(self, step: int) -> Tuple[int, Cameras, Dict]:
@@ -228,6 +238,7 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
if len(self.eval_unseen_cameras) == 0:
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
data = self.eval_dataset.get_data(image_idx)
data["image"] = data["image"].to(self.device)
assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension"
camera = self.eval_dataset.cameras[image_idx]
camera = self.eval_dataset.cameras[image_idx].to(self.device)
return image_idx, camera, data