mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 22:56:03 +08:00
Adding support for PairPixelSampler and PatchPixelSampler masks (#2305)
* changes * pair pixel sampling with a mask * reverting changes * revert changes * remove line in erode * support for patch pixel samplerr masks * fix --------- Co-authored-by: machenmusik <machenmusik@users.noreply.github.com>
This commit is contained in:
1 parent
50d8c7ba7d
commit
9b6010ea0d
2 files changed
+98
-13
No files matched your search
@@ -23,6 +23,7 @@ from jaxtyping import Int
|
||||
from torch import Tensor
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from nerfstudio.data.utils.pixel_sampling_utils import erode_mask
|
||||
from typing import (
|
||||
Dict,
|
||||
Optional,
|
||||
@@ -284,7 +285,7 @@ class PatchPixelSamplerConfig(PixelSamplerConfig):
|
||||
|
||||
_target: Type = field(default_factory=lambda: PatchPixelSampler)
|
||||
"""Target class to instantiate."""
|
||||
patch_size: int = 2
|
||||
patch_size: int = 32
|
||||
"""Side length of patch. This must be consistent in the method
|
||||
config in order for samples to be reshaped into patches correctly."""
|
||||
|
||||
@@ -318,9 +319,27 @@ class PatchPixelSampler(PixelSampler):
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
if isinstance(mask, Tensor):
|
||||
raise NotImplementedError(
|
||||
"Masked sampling not implemented for PairPixelSampler. Change Config to PixelSamplerConfig instead."
|
||||
sub_bs = batch_size // (self.config.patch_size**2)
|
||||
half_patch_size = int(self.config.patch_size / 2)
|
||||
m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=half_patch_size)
|
||||
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=sub_bs)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
|
||||
indices = (
|
||||
indices.view(sub_bs, 1, 1, 3)
|
||||
.broadcast_to(sub_bs, self.config.patch_size, self.config.patch_size, 3)
|
||||
.clone()
|
||||
)
|
||||
|
||||
yys, xxs = torch.meshgrid(
|
||||
torch.arange(self.config.patch_size, device=device), torch.arange(self.config.patch_size, device=device)
|
||||
)
|
||||
indices[:, ..., 1] += yys - half_patch_size
|
||||
indices[:, ..., 2] += xxs - half_patch_size
|
||||
|
||||
indices = torch.floor(indices).long()
|
||||
indices = indices.flatten(0, 2)
|
||||
else:
|
||||
sub_bs = batch_size // (self.config.patch_size**2)
|
||||
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
|
||||
@@ -381,9 +400,10 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
if isinstance(mask, Tensor):
|
||||
raise NotImplementedError(
|
||||
"Masked sampling not implemented for PairPixelSampler. Change Config to PixelSamplerConfig instead."
|
||||
)
|
||||
m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius)
|
||||
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=self.rays_to_sample)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
else:
|
||||
s = (self.rays_to_sample, 1)
|
||||
ns = torch.randint(0, num_images, s, dtype=torch.long, device=device)
|
||||
@@ -391,12 +411,12 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
|
||||
ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device)
|
||||
indices = torch.concat((ns, hs, ws), dim=1)
|
||||
|
||||
pair_indices = torch.hstack(
|
||||
(
|
||||
torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long),
|
||||
torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long),
|
||||
)
|
||||
pair_indices = torch.hstack(
|
||||
(
|
||||
torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long),
|
||||
torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long),
|
||||
)
|
||||
pair_indices += indices
|
||||
indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3)
|
||||
)
|
||||
pair_indices += indices
|
||||
indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3)
|
||||
return indices
|
||||
Reference in new issue
Block a user