Monodepth Regularization (#2284)

* monodepth implemented

* device

* pair pixel sampler

* docstring

* removing unused imports

* fix bugs

* cache in cache dir, no longer edit json file

* moving to depth nerfacto

* fixed bug using median depth by accident

* removing depthdataset from nerfacto

* added pixelsamplerconfig

* added equirectangular warning back

* bug fix

* delete sample_config.py + small fixes

* rename to pixel_sampler

* cleaning

* test

* passing num_rays_per_batch properly

* fixed transforms typo

* renaming in nerfacto

* remove depth files from test data

* removed defaulting to base pixel sampler with mask

* adding patch_size back

---------

Co-authored-by: AdamRashid96 <abrashid@berkeley.edu>
Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
authored and GitHub committed 2023-08-01 18:52:47 -07:00
1 parent 72da747fc2
commit 71c6b9cf3d
8 files changed
+293 -89

No files matched your search

@@ -55,9 +55,9 @@ from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.pixel_samplers import (
EquirectangularPixelSampler,
PatchPixelSampler,
PixelSampler,
PixelSamplerConfig,
PatchPixelSamplerConfig,
)
from nerfstudio.data.utils.dataloaders import (
CacheDataloader,
@@ -348,6 +348,8 @@ class VanillaDataManagerConfig(DataManagerConfig):
"""
patch_size: int = 1
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
pixel_sampler: PixelSamplerConfig = PixelSamplerConfig()
"""Specifies the pixel sampler used to sample pixels from images."""
TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset)
@@ -457,19 +459,18 @@ class VanillaDataManager(DataManager, Generic[TDataset]):
scale_factor=self.config.camera_res_scale_factor,
)
def _get_pixel_sampler(self, dataset: TDataset, *args: Any, **kwargs: Any) -> PixelSampler:
def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> PixelSampler:
"""Infer pixel sampler to use."""
if self.config.patch_size > 1:
return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size)
# If all images are equirectangular, use equirectangular pixel sampler
is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value
if is_equirectangular.all():
return EquirectangularPixelSampler(*args, **kwargs)
# Otherwise, use the default pixel sampler
return PatchPixelSamplerConfig().setup(
patch_size=self.config.patch_size, num_rays_per_batch=num_rays_per_batch
)
is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all()
if is_equirectangular.any():
CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.")
return PixelSampler(*args, **kwargs)
return self.config.pixel_sampler.setup(
is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch
)
def setup_train(self):
"""Sets up the data loaders for training"""
+79 -7
View File
@@ -18,31 +18,95 @@ Depth dataset.
from typing import Dict
import numpy as np
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
from nerfstudio.model_components import losses
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.utils.data_utils import get_depth_image_from_path
from nerfstudio.utils.rich_utils import CONSOLE
from typing import Union
from PIL import Image
import torch
from rich.progress import track
from pathlib import Path
import json
class DepthDataset(InputDataset):
"""Dataset that returns images and depths.
"""Dataset that returns images and depths. If no depths are found, then we generate them with Zoe Depth.
Args:
dataparser_outputs: description of where and how to read input images.
scale_factor: The scaling factor for the dataparser outputs.
"""
exclude_batch_keys_from_device = InputDataset.exclude_batch_keys_from_device + ["depth_image"]
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
super().__init__(dataparser_outputs, scale_factor)
assert (
"depth_filenames" in dataparser_outputs.metadata.keys()
and dataparser_outputs.metadata["depth_filenames"] is not None
)
# if there are no depth images than we want to generate them all with zoe depth
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if len(dataparser_outputs.image_filenames) > 0 and (
"depth_filenames" not in dataparser_outputs.metadata.keys()
or dataparser_outputs.metadata["depth_filenames"] is None
):
losses.DEPTH_METRIC = 0
CONSOLE.print("[bold yellow] No depth data found!")
cache = dataparser_outputs.image_filenames[0].parent / "depths.npy"
if cache.exists():
CONSOLE.print("[bold yellow] Loading cache!")
# load all the depths
self.depths = np.load(cache)
self.depths = torch.from_numpy(self.depths).to(device)
else:
depth_tensors = []
transforms = self._find_transform(dataparser_outputs.image_filenames[0])
data = dataparser_outputs.image_filenames[0].parent
if transforms is not None:
meta = json.load(open(transforms, "r"))
frames = meta["frames"]
filenames = [data / frames[j]["file_path"].split("/")[-1] for j in range(len(frames))]
else:
meta = None
frames = None
filenames = dataparser_outputs.image_filenames
repo = "isl-org/ZoeDepth"
self.zoe = torch.compile(torch.hub.load(repo, "ZoeD_NK", pretrained=True).to(device))
for i in track(range(len(filenames)), description="Generating depth images"):
image_filename = filenames[i]
pil_image = Image.open(image_filename)
image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4)
if len(image.shape) == 2:
image = image[:, :, None].repeat(3, axis=2)
image = torch.from_numpy(image.astype("float32") / 255.0)
with torch.no_grad():
image = torch.permute(image, (2, 0, 1)).unsqueeze(0).to(device)
if image.shape[1] == 4:
image = image[:, :3, :, :]
depth_tensor = self.zoe.infer(image).squeeze().unsqueeze(-1)
depth_tensors.append(depth_tensor)
self.depths = torch.stack(depth_tensors)
np.save(cache, self.depths.cpu().numpy())
dataparser_outputs.metadata["depth_filenames"] = None
dataparser_outputs.metadata["depth_unit_scale_factor"] = 1.0
self.metadata["depth_filenames"] = None
self.metadata["depth_unit_scale_factor"] = 1.0
else:
losses.DEPTH_METRIC = 0
self.depth_filenames = self.metadata["depth_filenames"]
self.depth_unit_scale_factor = self.metadata["depth_unit_scale_factor"]
def get_metadata(self, data: Dict) -> Dict:
if self.depth_filenames is None:
return {"depth_image": self.depths[data["image_idx"]]}
filepath = self.depth_filenames[data["image_idx"]]
height = int(self._dataparser_outputs.cameras.height[data["image_idx"]])
width = int(self._dataparser_outputs.cameras.width[data["image_idx"]])
@@ -54,3 +118,11 @@ class DepthDataset(InputDataset):
)
return {"depth_image": depth_image}
def _find_transform(self, image_path: Path) -> Union[Path, None]:
while image_path.parent != image_path:
transform_path = image_path.parent / "transforms.json"
if transform_path.exists():
return transform_path
image_path = image_path.parent
return None
+165 -66
View File
@@ -17,25 +17,55 @@ Code for sampling pixels.
"""
import random
from typing import Dict, Optional, Union
import torch
from jaxtyping import Int
from torch import Tensor
from dataclasses import dataclass, field
from typing import (
Dict,
Optional,
Type,
Union,
)
from nerfstudio.configs.base_config import (
InstantiateConfig,
)
@dataclass
class PixelSamplerConfig(InstantiateConfig):
"""Configuration for pixel sampler instantiation."""
_target: Type = field(default_factory=lambda: PixelSampler)
"""Target class to instantiate."""
num_rays_per_batch: int = 4096
"""Number of rays to sample per batch."""
keep_full_image: bool = False
"""Whether or not to include a reference to the full image in returned batch."""
is_equirectangular: bool = False
"""List of whether or not camera i is equirectangular."""
class PixelSampler:
"""Samples 'pixel_batch's from 'image_batch's.
Args:
num_rays_per_batch: number of rays to sample per batch
keep_full_image: whether or not to include a reference to the full image in returned batch
config: the DataManagerConfig used to instantiate class
"""
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
config: PixelSamplerConfig
def __init__(self, config: PixelSamplerConfig, **kwargs) -> None:
self.kwargs = kwargs
self.num_rays_per_batch = num_rays_per_batch
self.keep_full_image = keep_full_image
self.config = config
# Possibly override some values if they are present in the kwargs dictionary
self.config.num_rays_per_batch = self.kwargs.get("num_rays_per_batch", self.config.num_rays_per_batch)
self.config.keep_full_image = self.kwargs.get("keep_full_image", self.config.keep_full_image)
self.config.is_equirectangular = self.kwargs.get("is_equirectangular", self.config.is_equirectangular)
self.set_num_rays_per_batch(self.config.num_rays_per_batch)
def set_num_rays_per_batch(self, num_rays_per_batch: int):
"""Set the number of rays to sample per batch.
@@ -74,6 +104,36 @@ class PixelSampler:
return indices
def sample_method_equirectangular(
self,
batch_size: int,
num_images: int,
image_height: int,
image_width: int,
mask: Optional[Tensor] = None,
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
if isinstance(mask, torch.Tensor):
# Note: if there is a mask, sampling reduces back to uniform sampling, which gives more
# sampling weight to the poles of the image than the equators.
# TODO(kevinddchen): implement the correct mask-sampling method.
indices = self.sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
else:
# We sample theta uniformly in [0, 2*pi]
# We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2.
# This is done by inverse transform sampling.
# http://corysimon.github.io/articles/uniformdistn-on-sphere/
num_images_rand = torch.rand(batch_size, device=device)
phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi
theta_rand = torch.rand(batch_size, device=device)
indices = torch.floor(
torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1)
* torch.tensor([num_images, image_height, image_width], device=device)
).long()
return indices
def collate_image_dataset_batch(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False):
"""
Operates on a batch of images and samples pixels to use for generating rays.
@@ -90,11 +150,21 @@ class PixelSampler:
num_images, image_height, image_width, _ = batch["image"].shape
if "mask" in batch:
indices = self.sample_method(
num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device
)
if self.config.is_equirectangular:
indices = self.sample_method_equirectangular(
num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device
)
else:
indices = self.sample_method(
num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device
)
else:
indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device)
if self.config.is_equirectangular:
indices = self.sample_method_equirectangular(
num_rays_per_batch, num_images, image_height, image_width, device=device
)
else:
indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device)
c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1))
c, y, x = c.cpu(), y.cpu(), x.cpu()
@@ -156,7 +226,12 @@ class PixelSampler:
image_height, image_width, _ = batch["image"][i].shape
if i == num_images - 1:
num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch
indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device)
if self.config.is_equirectangular:
indices = self.sample_method_equirectangular(
num_rays_in_batch, 1, image_height, image_width, device=device
)
else:
indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device)
indices[:, 0] = i
all_indices.append(indices)
all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]])
@@ -192,56 +267,26 @@ class PixelSampler:
if isinstance(image_batch["image"], list):
image_batch = dict(image_batch.items()) # copy the dictionary so we don't modify the original
pixel_batch = self.collate_image_dataset_batch_list(
image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image
image_batch, self.num_rays_per_batch, keep_full_image=self.config.keep_full_image
)
elif isinstance(image_batch["image"], torch.Tensor):
pixel_batch = self.collate_image_dataset_batch(
image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image
image_batch, self.num_rays_per_batch, keep_full_image=self.config.keep_full_image
)
else:
raise ValueError("image_batch['image'] must be a list or torch.Tensor")
return pixel_batch
class EquirectangularPixelSampler(PixelSampler):
"""Samples 'pixel_batch's from 'image_batch's. Assumes images are
equirectangular and the sampling is done uniformly on the sphere.
@dataclass
class PatchPixelSamplerConfig(PixelSamplerConfig):
"""Config dataclass for PatchPixelSampler."""
Args:
num_rays_per_batch: number of rays to sample per batch
keep_full_image: whether or not to include a reference to the full image in returned batch
"""
# overrides base method
def sample_method(
self,
batch_size: int,
num_images: int,
image_height: int,
image_width: int,
mask: Optional[Tensor] = None,
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
if isinstance(mask, torch.Tensor):
# Note: if there is a mask, sampling reduces back to uniform sampling, which gives more
# sampling weight to the poles of the image than the equators.
# TODO(kevinddchen): implement the correct mask-sampling method.
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
else:
# We sample theta uniformly in [0, 2*pi]
# We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2.
# This is done by inverse transform sampling.
# http://corysimon.github.io/articles/uniformdistn-on-sphere/
num_images_rand = torch.rand(batch_size, device=device)
phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi
theta_rand = torch.rand(batch_size, device=device)
indices = torch.floor(
torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1)
* torch.tensor([num_images, image_height, image_width], device=device)
).long()
return indices
_target: Type = field(default_factory=lambda: PatchPixelSampler)
"""Target class to instantiate."""
patch_size: int = 2
"""Side length of patch. This must be consistent in the method
config in order for samples to be reshaped into patches correctly."""
class PatchPixelSampler(PixelSampler):
@@ -249,16 +294,10 @@ class PatchPixelSampler(PixelSampler):
from the images randomly. Useful for patch-based losses.
Args:
num_rays_per_batch: number of rays to sample per batch
keep_full_image: whether or not to include a reference to the full image in returned batch
patch_size: side length of patch. This must be consistent in the method
config in order for samples to be reshaped into patches correctly.
config: the PatchPixelSamplerConfig used to instantiate class
"""
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
self.patch_size = kwargs["patch_size"]
num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
super().__init__(num_rays, keep_full_image, **kwargs)
config: PatchPixelSamplerConfig
def set_num_rays_per_batch(self, num_rays_per_batch: int):
"""Set the number of rays to sample per batch. Overridden to deal with patch-based sampling.
@@ -266,7 +305,7 @@ class PatchPixelSampler(PixelSampler):
Args:
num_rays_per_batch: number of rays to sample per batch
"""
self.num_rays_per_batch = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
self.num_rays_per_batch = (num_rays_per_batch // (self.config.patch_size**2)) * (self.config.patch_size**2)
# overrides base method
def sample_method(
@@ -279,19 +318,23 @@ class PatchPixelSampler(PixelSampler):
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
if isinstance(mask, Tensor):
# Note: if there is a mask, sampling reduces back to uniform sampling
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
# Note: if there is a mask, should switch to the base PixelSampler class
raise NotImplementedError()
else:
sub_bs = batch_size // (self.patch_size**2)
sub_bs = batch_size // (self.config.patch_size**2)
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
[num_images, image_height - self.patch_size, image_width - self.patch_size],
[num_images, image_height - self.config.patch_size, image_width - self.config.patch_size],
device=device,
)
indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone()
indices = (
indices.view(sub_bs, 1, 1, 3)
.broadcast_to(sub_bs, self.config.patch_size, self.config.patch_size, 3)
.clone()
)
yys, xxs = torch.meshgrid(
torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device)
torch.arange(self.config.patch_size, device=device), torch.arange(self.config.patch_size, device=device)
)
indices[:, ..., 1] += yys
indices[:, ..., 2] += xxs
@@ -300,3 +343,59 @@ class PatchPixelSampler(PixelSampler):
indices = indices.flatten(0, 2)
return indices
@dataclass
class PairPixelSamplerConfig(PixelSamplerConfig):
"""Config dataclass for PatchPixelSampler."""
_target: Type = field(default_factory=lambda: PairPixelSampler)
"""Target class to instantiate."""
radius: int = 2
"""max distance between pairs of pixels."""
class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
"""Samples pair of pixels from 'image_batch's. Samples pairs of pixels from
from the images randomly within a 'radius' distance apart. Useful for pair-based losses.
Args:
config: the PairPixelSamplerConfig used to instantiate class
"""
def __init__(self, config: PairPixelSamplerConfig, **kwargs) -> None:
self.config = config
self.radius = self.config.radius
self.rays_to_sample = self.config.num_rays_per_batch // 2
super().__init__(self.config, **kwargs)
# overrides base method
def sample_method( # pylint: disable=no-self-use
self,
batch_size: int,
num_images: int,
image_height: int,
image_width: int,
mask: Optional[Tensor] = None,
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
if mask:
# Note: if there is a mask, should switch to the base PixelSampler class
raise NotImplementedError()
else:
s = (self.rays_to_sample, 1)
ns = torch.randint(0, num_images, s, dtype=torch.long, device=device)
hs = torch.randint(self.radius, image_height - self.radius, s, dtype=torch.long, device=device)
ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device)
indices = torch.concat((ns, hs, ws), dim=1)
pair_indices = torch.hstack(
(
torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long),
torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long),
)
)
pair_indices += indices
indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3)
return indices