Adding support for PairPixelSampler and PatchPixelSampler masks (#2305)

* changes

* pair pixel sampling with a mask

* reverting changes

* revert changes

* remove line in erode

* support for patch pixel samplerr masks

* fix

---------

Co-authored-by: machenmusik <machenmusik@users.noreply.github.com>
This commit is contained in:
Ethan Weberandmachenmusik authored and GitHub committed 2023-08-05 10:40:23 -04:00
1 parent 50d8c7ba7d
commit 9b6010ea0d
2 files changed
+98 -13

No files matched your search

+33 -13
View File
@@ -23,6 +23,7 @@ from jaxtyping import Int
from torch import Tensor
from dataclasses import dataclass, field
from nerfstudio.data.utils.pixel_sampling_utils import erode_mask
from typing import (
Dict,
Optional,
@@ -284,7 +285,7 @@ class PatchPixelSamplerConfig(PixelSamplerConfig):
_target: Type = field(default_factory=lambda: PatchPixelSampler)
"""Target class to instantiate."""
patch_size: int = 2
patch_size: int = 32
"""Side length of patch. This must be consistent in the method
config in order for samples to be reshaped into patches correctly."""
@@ -318,9 +319,27 @@ class PatchPixelSampler(PixelSampler):
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
if isinstance(mask, Tensor):
raise NotImplementedError(
"Masked sampling not implemented for PairPixelSampler. Change Config to PixelSamplerConfig instead."
sub_bs = batch_size // (self.config.patch_size**2)
half_patch_size = int(self.config.patch_size / 2)
m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=half_patch_size)
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
chosen_indices = random.sample(range(len(nonzero_indices)), k=sub_bs)
indices = nonzero_indices[chosen_indices]
indices = (
indices.view(sub_bs, 1, 1, 3)
.broadcast_to(sub_bs, self.config.patch_size, self.config.patch_size, 3)
.clone()
)
yys, xxs = torch.meshgrid(
torch.arange(self.config.patch_size, device=device), torch.arange(self.config.patch_size, device=device)
)
indices[:, ..., 1] += yys - half_patch_size
indices[:, ..., 2] += xxs - half_patch_size
indices = torch.floor(indices).long()
indices = indices.flatten(0, 2)
else:
sub_bs = batch_size // (self.config.patch_size**2)
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
@@ -381,9 +400,10 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
device: Union[torch.device, str] = "cpu",
) -> Int[Tensor, "batch_size 3"]:
if isinstance(mask, Tensor):
raise NotImplementedError(
"Masked sampling not implemented for PairPixelSampler. Change Config to PixelSamplerConfig instead."
)
m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius)
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
chosen_indices = random.sample(range(len(nonzero_indices)), k=self.rays_to_sample)
indices = nonzero_indices[chosen_indices]
else:
s = (self.rays_to_sample, 1)
ns = torch.randint(0, num_images, s, dtype=torch.long, device=device)
@@ -391,12 +411,12 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device)
indices = torch.concat((ns, hs, ws), dim=1)
pair_indices = torch.hstack(
(
torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long),
torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long),
)
pair_indices = torch.hstack(
(
torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long),
torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long),
)
pair_indices += indices
indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3)
)
pair_indices += indices
indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3)
return indices