mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
Monodepth Regularization (#2284)
* monodepth implemented * device * pair pixel sampler * docstring * removing unused imports * fix bugs * cache in cache dir, no longer edit json file * moving to depth nerfacto * fixed bug using median depth by accident * removing depthdataset from nerfacto * added pixelsamplerconfig * added equirectangular warning back * bug fix * delete sample_config.py + small fixes * rename to pixel_sampler * cleaning * test * passing num_rays_per_batch properly * fixed transforms typo * renaming in nerfacto * remove depth files from test data * removed defaulting to base pixel sampler with mask * adding patch_size back --------- Co-authored-by: AdamRashid96 <abrashid@berkeley.edu> Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
8 files changed
+293
-89
No files matched your search
@@ -55,9 +55,9 @@ from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
|
||||
from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.pixel_samplers import (
|
||||
EquirectangularPixelSampler,
|
||||
PatchPixelSampler,
|
||||
PixelSampler,
|
||||
PixelSamplerConfig,
|
||||
PatchPixelSamplerConfig,
|
||||
)
|
||||
from nerfstudio.data.utils.dataloaders import (
|
||||
CacheDataloader,
|
||||
@@ -348,6 +348,8 @@ class VanillaDataManagerConfig(DataManagerConfig):
|
||||
"""
|
||||
patch_size: int = 1
|
||||
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
|
||||
pixel_sampler: PixelSamplerConfig = PixelSamplerConfig()
|
||||
"""Specifies the pixel sampler used to sample pixels from images."""
|
||||
|
||||
|
||||
TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset)
|
||||
@@ -457,19 +459,18 @@ class VanillaDataManager(DataManager, Generic[TDataset]):
|
||||
scale_factor=self.config.camera_res_scale_factor,
|
||||
)
|
||||
|
||||
def _get_pixel_sampler(self, dataset: TDataset, *args: Any, **kwargs: Any) -> PixelSampler:
|
||||
def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> PixelSampler:
|
||||
"""Infer pixel sampler to use."""
|
||||
if self.config.patch_size > 1:
|
||||
return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size)
|
||||
|
||||
# If all images are equirectangular, use equirectangular pixel sampler
|
||||
is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value
|
||||
if is_equirectangular.all():
|
||||
return EquirectangularPixelSampler(*args, **kwargs)
|
||||
# Otherwise, use the default pixel sampler
|
||||
return PatchPixelSamplerConfig().setup(
|
||||
patch_size=self.config.patch_size, num_rays_per_batch=num_rays_per_batch
|
||||
)
|
||||
is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all()
|
||||
if is_equirectangular.any():
|
||||
CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.")
|
||||
return PixelSampler(*args, **kwargs)
|
||||
return self.config.pixel_sampler.setup(
|
||||
is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch
|
||||
)
|
||||
|
||||
def setup_train(self):
|
||||
"""Sets up the data loaders for training"""
|
||||
|
||||
@@ -18,31 +18,95 @@ Depth dataset.
|
||||
|
||||
from typing import Dict
|
||||
|
||||
import numpy as np
|
||||
|
||||
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
|
||||
from nerfstudio.model_components import losses
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.utils.data_utils import get_depth_image_from_path
|
||||
from nerfstudio.utils.rich_utils import CONSOLE
|
||||
|
||||
from typing import Union
|
||||
from PIL import Image
|
||||
import torch
|
||||
from rich.progress import track
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
|
||||
class DepthDataset(InputDataset):
|
||||
"""Dataset that returns images and depths.
|
||||
"""Dataset that returns images and depths. If no depths are found, then we generate them with Zoe Depth.
|
||||
|
||||
Args:
|
||||
dataparser_outputs: description of where and how to read input images.
|
||||
scale_factor: The scaling factor for the dataparser outputs.
|
||||
"""
|
||||
|
||||
exclude_batch_keys_from_device = InputDataset.exclude_batch_keys_from_device + ["depth_image"]
|
||||
|
||||
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
|
||||
super().__init__(dataparser_outputs, scale_factor)
|
||||
assert (
|
||||
"depth_filenames" in dataparser_outputs.metadata.keys()
|
||||
and dataparser_outputs.metadata["depth_filenames"] is not None
|
||||
)
|
||||
# if there are no depth images than we want to generate them all with zoe depth
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
if len(dataparser_outputs.image_filenames) > 0 and (
|
||||
"depth_filenames" not in dataparser_outputs.metadata.keys()
|
||||
or dataparser_outputs.metadata["depth_filenames"] is None
|
||||
):
|
||||
losses.DEPTH_METRIC = 0
|
||||
CONSOLE.print("[bold yellow] No depth data found!")
|
||||
cache = dataparser_outputs.image_filenames[0].parent / "depths.npy"
|
||||
if cache.exists():
|
||||
CONSOLE.print("[bold yellow] Loading cache!")
|
||||
# load all the depths
|
||||
self.depths = np.load(cache)
|
||||
self.depths = torch.from_numpy(self.depths).to(device)
|
||||
else:
|
||||
depth_tensors = []
|
||||
transforms = self._find_transform(dataparser_outputs.image_filenames[0])
|
||||
data = dataparser_outputs.image_filenames[0].parent
|
||||
if transforms is not None:
|
||||
meta = json.load(open(transforms, "r"))
|
||||
frames = meta["frames"]
|
||||
filenames = [data / frames[j]["file_path"].split("/")[-1] for j in range(len(frames))]
|
||||
else:
|
||||
meta = None
|
||||
frames = None
|
||||
filenames = dataparser_outputs.image_filenames
|
||||
|
||||
repo = "isl-org/ZoeDepth"
|
||||
self.zoe = torch.compile(torch.hub.load(repo, "ZoeD_NK", pretrained=True).to(device))
|
||||
|
||||
for i in track(range(len(filenames)), description="Generating depth images"):
|
||||
image_filename = filenames[i]
|
||||
pil_image = Image.open(image_filename)
|
||||
image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4)
|
||||
if len(image.shape) == 2:
|
||||
image = image[:, :, None].repeat(3, axis=2)
|
||||
image = torch.from_numpy(image.astype("float32") / 255.0)
|
||||
|
||||
with torch.no_grad():
|
||||
image = torch.permute(image, (2, 0, 1)).unsqueeze(0).to(device)
|
||||
if image.shape[1] == 4:
|
||||
image = image[:, :3, :, :]
|
||||
depth_tensor = self.zoe.infer(image).squeeze().unsqueeze(-1)
|
||||
|
||||
depth_tensors.append(depth_tensor)
|
||||
|
||||
self.depths = torch.stack(depth_tensors)
|
||||
np.save(cache, self.depths.cpu().numpy())
|
||||
dataparser_outputs.metadata["depth_filenames"] = None
|
||||
dataparser_outputs.metadata["depth_unit_scale_factor"] = 1.0
|
||||
self.metadata["depth_filenames"] = None
|
||||
self.metadata["depth_unit_scale_factor"] = 1.0
|
||||
|
||||
else:
|
||||
losses.DEPTH_METRIC = 0
|
||||
|
||||
self.depth_filenames = self.metadata["depth_filenames"]
|
||||
self.depth_unit_scale_factor = self.metadata["depth_unit_scale_factor"]
|
||||
|
||||
def get_metadata(self, data: Dict) -> Dict:
|
||||
if self.depth_filenames is None:
|
||||
return {"depth_image": self.depths[data["image_idx"]]}
|
||||
|
||||
filepath = self.depth_filenames[data["image_idx"]]
|
||||
height = int(self._dataparser_outputs.cameras.height[data["image_idx"]])
|
||||
width = int(self._dataparser_outputs.cameras.width[data["image_idx"]])
|
||||
@@ -54,3 +118,11 @@ class DepthDataset(InputDataset):
|
||||
)
|
||||
|
||||
return {"depth_image": depth_image}
|
||||
|
||||
def _find_transform(self, image_path: Path) -> Union[Path, None]:
|
||||
while image_path.parent != image_path:
|
||||
transform_path = image_path.parent / "transforms.json"
|
||||
if transform_path.exists():
|
||||
return transform_path
|
||||
image_path = image_path.parent
|
||||
return None
|
||||
@@ -17,25 +17,55 @@ Code for sampling pixels.
|
||||
"""
|
||||
|
||||
import random
|
||||
from typing import Dict, Optional, Union
|
||||
|
||||
import torch
|
||||
from jaxtyping import Int
|
||||
from torch import Tensor
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import (
|
||||
Dict,
|
||||
Optional,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
from nerfstudio.configs.base_config import (
|
||||
InstantiateConfig,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PixelSamplerConfig(InstantiateConfig):
|
||||
"""Configuration for pixel sampler instantiation."""
|
||||
|
||||
_target: Type = field(default_factory=lambda: PixelSampler)
|
||||
"""Target class to instantiate."""
|
||||
num_rays_per_batch: int = 4096
|
||||
"""Number of rays to sample per batch."""
|
||||
keep_full_image: bool = False
|
||||
"""Whether or not to include a reference to the full image in returned batch."""
|
||||
is_equirectangular: bool = False
|
||||
"""List of whether or not camera i is equirectangular."""
|
||||
|
||||
|
||||
class PixelSampler:
|
||||
"""Samples 'pixel_batch's from 'image_batch's.
|
||||
|
||||
Args:
|
||||
num_rays_per_batch: number of rays to sample per batch
|
||||
keep_full_image: whether or not to include a reference to the full image in returned batch
|
||||
config: the DataManagerConfig used to instantiate class
|
||||
"""
|
||||
|
||||
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
|
||||
config: PixelSamplerConfig
|
||||
|
||||
def __init__(self, config: PixelSamplerConfig, **kwargs) -> None:
|
||||
self.kwargs = kwargs
|
||||
self.num_rays_per_batch = num_rays_per_batch
|
||||
self.keep_full_image = keep_full_image
|
||||
self.config = config
|
||||
# Possibly override some values if they are present in the kwargs dictionary
|
||||
self.config.num_rays_per_batch = self.kwargs.get("num_rays_per_batch", self.config.num_rays_per_batch)
|
||||
self.config.keep_full_image = self.kwargs.get("keep_full_image", self.config.keep_full_image)
|
||||
self.config.is_equirectangular = self.kwargs.get("is_equirectangular", self.config.is_equirectangular)
|
||||
self.set_num_rays_per_batch(self.config.num_rays_per_batch)
|
||||
|
||||
def set_num_rays_per_batch(self, num_rays_per_batch: int):
|
||||
"""Set the number of rays to sample per batch.
|
||||
@@ -74,6 +104,36 @@ class PixelSampler:
|
||||
|
||||
return indices
|
||||
|
||||
def sample_method_equirectangular(
|
||||
self,
|
||||
batch_size: int,
|
||||
num_images: int,
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
mask: Optional[Tensor] = None,
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
if isinstance(mask, torch.Tensor):
|
||||
# Note: if there is a mask, sampling reduces back to uniform sampling, which gives more
|
||||
# sampling weight to the poles of the image than the equators.
|
||||
# TODO(kevinddchen): implement the correct mask-sampling method.
|
||||
|
||||
indices = self.sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
|
||||
else:
|
||||
# We sample theta uniformly in [0, 2*pi]
|
||||
# We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2.
|
||||
# This is done by inverse transform sampling.
|
||||
# http://corysimon.github.io/articles/uniformdistn-on-sphere/
|
||||
num_images_rand = torch.rand(batch_size, device=device)
|
||||
phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi
|
||||
theta_rand = torch.rand(batch_size, device=device)
|
||||
indices = torch.floor(
|
||||
torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
|
||||
return indices
|
||||
|
||||
def collate_image_dataset_batch(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False):
|
||||
"""
|
||||
Operates on a batch of images and samples pixels to use for generating rays.
|
||||
@@ -90,11 +150,21 @@ class PixelSampler:
|
||||
num_images, image_height, image_width, _ = batch["image"].shape
|
||||
|
||||
if "mask" in batch:
|
||||
indices = self.sample_method(
|
||||
num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device
|
||||
)
|
||||
if self.config.is_equirectangular:
|
||||
indices = self.sample_method_equirectangular(
|
||||
num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device
|
||||
)
|
||||
else:
|
||||
indices = self.sample_method(
|
||||
num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device
|
||||
)
|
||||
else:
|
||||
indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device)
|
||||
if self.config.is_equirectangular:
|
||||
indices = self.sample_method_equirectangular(
|
||||
num_rays_per_batch, num_images, image_height, image_width, device=device
|
||||
)
|
||||
else:
|
||||
indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device)
|
||||
|
||||
c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1))
|
||||
c, y, x = c.cpu(), y.cpu(), x.cpu()
|
||||
@@ -156,7 +226,12 @@ class PixelSampler:
|
||||
image_height, image_width, _ = batch["image"][i].shape
|
||||
if i == num_images - 1:
|
||||
num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch
|
||||
indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device)
|
||||
if self.config.is_equirectangular:
|
||||
indices = self.sample_method_equirectangular(
|
||||
num_rays_in_batch, 1, image_height, image_width, device=device
|
||||
)
|
||||
else:
|
||||
indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device)
|
||||
indices[:, 0] = i
|
||||
all_indices.append(indices)
|
||||
all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]])
|
||||
@@ -192,56 +267,26 @@ class PixelSampler:
|
||||
if isinstance(image_batch["image"], list):
|
||||
image_batch = dict(image_batch.items()) # copy the dictionary so we don't modify the original
|
||||
pixel_batch = self.collate_image_dataset_batch_list(
|
||||
image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image
|
||||
image_batch, self.num_rays_per_batch, keep_full_image=self.config.keep_full_image
|
||||
)
|
||||
elif isinstance(image_batch["image"], torch.Tensor):
|
||||
pixel_batch = self.collate_image_dataset_batch(
|
||||
image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image
|
||||
image_batch, self.num_rays_per_batch, keep_full_image=self.config.keep_full_image
|
||||
)
|
||||
else:
|
||||
raise ValueError("image_batch['image'] must be a list or torch.Tensor")
|
||||
return pixel_batch
|
||||
|
||||
|
||||
class EquirectangularPixelSampler(PixelSampler):
|
||||
"""Samples 'pixel_batch's from 'image_batch's. Assumes images are
|
||||
equirectangular and the sampling is done uniformly on the sphere.
|
||||
@dataclass
|
||||
class PatchPixelSamplerConfig(PixelSamplerConfig):
|
||||
"""Config dataclass for PatchPixelSampler."""
|
||||
|
||||
Args:
|
||||
num_rays_per_batch: number of rays to sample per batch
|
||||
keep_full_image: whether or not to include a reference to the full image in returned batch
|
||||
"""
|
||||
|
||||
# overrides base method
|
||||
def sample_method(
|
||||
self,
|
||||
batch_size: int,
|
||||
num_images: int,
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
mask: Optional[Tensor] = None,
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
if isinstance(mask, torch.Tensor):
|
||||
# Note: if there is a mask, sampling reduces back to uniform sampling, which gives more
|
||||
# sampling weight to the poles of the image than the equators.
|
||||
# TODO(kevinddchen): implement the correct mask-sampling method.
|
||||
|
||||
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
|
||||
else:
|
||||
# We sample theta uniformly in [0, 2*pi]
|
||||
# We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2.
|
||||
# This is done by inverse transform sampling.
|
||||
# http://corysimon.github.io/articles/uniformdistn-on-sphere/
|
||||
num_images_rand = torch.rand(batch_size, device=device)
|
||||
phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi
|
||||
theta_rand = torch.rand(batch_size, device=device)
|
||||
indices = torch.floor(
|
||||
torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
|
||||
return indices
|
||||
_target: Type = field(default_factory=lambda: PatchPixelSampler)
|
||||
"""Target class to instantiate."""
|
||||
patch_size: int = 2
|
||||
"""Side length of patch. This must be consistent in the method
|
||||
config in order for samples to be reshaped into patches correctly."""
|
||||
|
||||
|
||||
class PatchPixelSampler(PixelSampler):
|
||||
@@ -249,16 +294,10 @@ class PatchPixelSampler(PixelSampler):
|
||||
from the images randomly. Useful for patch-based losses.
|
||||
|
||||
Args:
|
||||
num_rays_per_batch: number of rays to sample per batch
|
||||
keep_full_image: whether or not to include a reference to the full image in returned batch
|
||||
patch_size: side length of patch. This must be consistent in the method
|
||||
config in order for samples to be reshaped into patches correctly.
|
||||
config: the PatchPixelSamplerConfig used to instantiate class
|
||||
"""
|
||||
|
||||
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
|
||||
self.patch_size = kwargs["patch_size"]
|
||||
num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
|
||||
super().__init__(num_rays, keep_full_image, **kwargs)
|
||||
config: PatchPixelSamplerConfig
|
||||
|
||||
def set_num_rays_per_batch(self, num_rays_per_batch: int):
|
||||
"""Set the number of rays to sample per batch. Overridden to deal with patch-based sampling.
|
||||
@@ -266,7 +305,7 @@ class PatchPixelSampler(PixelSampler):
|
||||
Args:
|
||||
num_rays_per_batch: number of rays to sample per batch
|
||||
"""
|
||||
self.num_rays_per_batch = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
|
||||
self.num_rays_per_batch = (num_rays_per_batch // (self.config.patch_size**2)) * (self.config.patch_size**2)
|
||||
|
||||
# overrides base method
|
||||
def sample_method(
|
||||
@@ -279,19 +318,23 @@ class PatchPixelSampler(PixelSampler):
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
if isinstance(mask, Tensor):
|
||||
# Note: if there is a mask, sampling reduces back to uniform sampling
|
||||
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
|
||||
# Note: if there is a mask, should switch to the base PixelSampler class
|
||||
raise NotImplementedError()
|
||||
else:
|
||||
sub_bs = batch_size // (self.patch_size**2)
|
||||
sub_bs = batch_size // (self.config.patch_size**2)
|
||||
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
|
||||
[num_images, image_height - self.patch_size, image_width - self.patch_size],
|
||||
[num_images, image_height - self.config.patch_size, image_width - self.config.patch_size],
|
||||
device=device,
|
||||
)
|
||||
|
||||
indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone()
|
||||
indices = (
|
||||
indices.view(sub_bs, 1, 1, 3)
|
||||
.broadcast_to(sub_bs, self.config.patch_size, self.config.patch_size, 3)
|
||||
.clone()
|
||||
)
|
||||
|
||||
yys, xxs = torch.meshgrid(
|
||||
torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device)
|
||||
torch.arange(self.config.patch_size, device=device), torch.arange(self.config.patch_size, device=device)
|
||||
)
|
||||
indices[:, ..., 1] += yys
|
||||
indices[:, ..., 2] += xxs
|
||||
@@ -300,3 +343,59 @@ class PatchPixelSampler(PixelSampler):
|
||||
indices = indices.flatten(0, 2)
|
||||
|
||||
return indices
|
||||
|
||||
|
||||
@dataclass
|
||||
class PairPixelSamplerConfig(PixelSamplerConfig):
|
||||
"""Config dataclass for PatchPixelSampler."""
|
||||
|
||||
_target: Type = field(default_factory=lambda: PairPixelSampler)
|
||||
"""Target class to instantiate."""
|
||||
radius: int = 2
|
||||
"""max distance between pairs of pixels."""
|
||||
|
||||
|
||||
class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
|
||||
"""Samples pair of pixels from 'image_batch's. Samples pairs of pixels from
|
||||
from the images randomly within a 'radius' distance apart. Useful for pair-based losses.
|
||||
|
||||
Args:
|
||||
config: the PairPixelSamplerConfig used to instantiate class
|
||||
"""
|
||||
|
||||
def __init__(self, config: PairPixelSamplerConfig, **kwargs) -> None:
|
||||
self.config = config
|
||||
self.radius = self.config.radius
|
||||
self.rays_to_sample = self.config.num_rays_per_batch // 2
|
||||
super().__init__(self.config, **kwargs)
|
||||
|
||||
# overrides base method
|
||||
def sample_method( # pylint: disable=no-self-use
|
||||
self,
|
||||
batch_size: int,
|
||||
num_images: int,
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
mask: Optional[Tensor] = None,
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
if mask:
|
||||
# Note: if there is a mask, should switch to the base PixelSampler class
|
||||
|
||||
raise NotImplementedError()
|
||||
else:
|
||||
s = (self.rays_to_sample, 1)
|
||||
ns = torch.randint(0, num_images, s, dtype=torch.long, device=device)
|
||||
hs = torch.randint(self.radius, image_height - self.radius, s, dtype=torch.long, device=device)
|
||||
ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device)
|
||||
indices = torch.concat((ns, hs, ws), dim=1)
|
||||
|
||||
pair_indices = torch.hstack(
|
||||
(
|
||||
torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long),
|
||||
torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long),
|
||||
)
|
||||
)
|
||||
pair_indices += indices
|
||||
indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3)
|
||||
return indices
|
||||
Reference in new issue
Block a user