mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 05:44:31 +08:00
Basic working, still needs to be profiled and tuned, some preliminary TODOs labeled 'jake-austin' need to be addressed
This commit is contained in:
1 parent
749059d4fb
commit
bb90b94fb0
11 files changed
+934
-62
No files matched your search
@@ -33,7 +33,7 @@ from torch.utils.data.distributed import DistributedSampler
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig
|
||||
from nerfstudio.cameras.cameras import CameraType
|
||||
from nerfstudio.cameras.cameras import Cameras, CameraType
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.configs.base_config import InstantiateConfig
|
||||
from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion
|
||||
@@ -115,7 +115,7 @@ class DataManager(nn.Module):
|
||||
To get data, use the next_train and next_eval functions.
|
||||
This data manager's next_train and next_eval methods will return 2 things:
|
||||
|
||||
1. A Raybundle: This will contain the rays we are sampling, with latents and
|
||||
1. A rays: This will contain the rays/camera we are sampling, with latents and
|
||||
conditionals attached (everything needed at inference)
|
||||
2. A "batch" of auxiliary information: This will contain the mask, the ground truth
|
||||
pixels, etc needed to actually train, score, etc the model
|
||||
@@ -230,7 +230,7 @@ class DataManager(nn.Module):
|
||||
"""Sets up the data manager for evaluation"""
|
||||
|
||||
@abstractmethod
|
||||
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
def next_train(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]:
|
||||
"""Returns the next batch of data from the train data manager.
|
||||
|
||||
Args:
|
||||
@@ -242,25 +242,25 @@ class DataManager(nn.Module):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def next_eval(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
def next_eval(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]:
|
||||
"""Returns the next batch of data from the eval data manager.
|
||||
|
||||
Args:
|
||||
step: the step number of the eval image to retrieve
|
||||
Returns:
|
||||
A tuple of the ray bundle for the image, and a dictionary of additional batch information
|
||||
A tuple of the ray/camera for the image, and a dictionary of additional batch information
|
||||
such as the groundtruth image.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]:
|
||||
def next_eval_image(self, step: int) -> Tuple[int, Union[RayBundle, Cameras], Dict]:
|
||||
"""Retrieve the next eval image.
|
||||
|
||||
Args:
|
||||
step: the step number of the eval image to retrieve
|
||||
Returns:
|
||||
A tuple of the step number, the ray bundle for the image, and a dictionary of
|
||||
A tuple of the step number, the ray/camera for the image, and a dictionary of
|
||||
additional batch information such as the groundtruth image.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -297,7 +297,7 @@ class DataManager(nn.Module):
|
||||
|
||||
@dataclass
|
||||
class VanillaDataManagerConfig(DataManagerConfig):
|
||||
"""A basic data manager"""
|
||||
"""A basic data manager for a ray-based model"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: VanillaDataManager)
|
||||
"""Target class to instantiate."""
|
||||
|
||||
@@ -21,6 +21,7 @@ paradigm
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
from collections import defaultdict
|
||||
@@ -29,7 +30,7 @@ from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import (Any, Callable, Dict, ForwardRef, Generic, List, Literal,
|
||||
Optional, Tuple, Type, Union, cast, get_args, get_origin)
|
||||
import random
|
||||
|
||||
import torch
|
||||
from rich.progress import Console
|
||||
from torch import Tensor, nn
|
||||
@@ -87,6 +88,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
|
||||
A datamanager that outputs full images and cameras instead of raybundles. This makes the
|
||||
datamanager far more lightweight since we dont have to do ray generation for things like
|
||||
gaussian splatting that dont require it.
|
||||
|
||||
TODO (jake-austin): Figure out why the dataparser doesnt show up as a subcommand, preventing us
|
||||
from specifying nerfstudio-data instead of blender-data
|
||||
"""
|
||||
|
||||
config: FullImageDatamanagerConfig
|
||||
@@ -191,6 +195,10 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
|
||||
"""
|
||||
return {}
|
||||
|
||||
def get_train_rays_per_batch(self):
|
||||
# TODO (jake-austin): fix this to be the resolution of the last image rendered
|
||||
return 800*800
|
||||
|
||||
def next_train(self, step: int) -> Tuple[Cameras, Dict]:
|
||||
"""Returns the next training batch
|
||||
|
||||
@@ -200,8 +208,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
|
||||
if len(self.train_unseen_cameras) == 0:
|
||||
self.train_unseen_cameras = [i for i in range(len(self.train_dataset))]
|
||||
data = self.train_dataset.get_data(image_idx)
|
||||
data["image"] = data["image"].to(self.device)
|
||||
assert len(self.train_dataset.cameras.shape) == 1, "Assumes single batch dimension"
|
||||
camera = self.train_dataset.cameras[image_idx]
|
||||
camera = self.train_dataset.cameras[image_idx].to(self.device)
|
||||
return camera, data
|
||||
|
||||
def next_eval(self, step: int) -> Tuple[Cameras, Dict]:
|
||||
@@ -213,8 +222,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
|
||||
if len(self.eval_unseen_cameras) == 0:
|
||||
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
|
||||
data = self.eval_dataset.get_data(image_idx)
|
||||
data["image"] = data["image"].to(self.device)
|
||||
assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension"
|
||||
camera = self.eval_dataset.cameras[image_idx]
|
||||
camera = self.eval_dataset.cameras[image_idx].to(self.device)
|
||||
return camera, data
|
||||
|
||||
def next_eval_image(self, step: int) -> Tuple[int, Cameras, Dict]:
|
||||
@@ -228,6 +238,7 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
|
||||
if len(self.eval_unseen_cameras) == 0:
|
||||
self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))]
|
||||
data = self.eval_dataset.get_data(image_idx)
|
||||
data["image"] = data["image"].to(self.device)
|
||||
assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension"
|
||||
camera = self.eval_dataset.cameras[image_idx]
|
||||
camera = self.eval_dataset.cameras[image_idx].to(self.device)
|
||||
return image_idx, camera, data
|
||||
Reference in new issue
Block a user