mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 21:33:56 +08:00
Speed up PairPixelSampler (#3452)
This commit is contained in:
1 parent
6da94443fd
commit
4c7297c75a
2 files changed
+94
-49
No files matched your search
@@ -78,6 +78,61 @@ class PixelSampler:
|
||||
"""
|
||||
self.num_rays_per_batch = num_rays_per_batch
|
||||
|
||||
def rejection_sample_mask(
|
||||
self,
|
||||
mask: Tensor,
|
||||
num_samples: int,
|
||||
num_images: int,
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
device: Union[torch.device, str],
|
||||
) -> Int[Tensor, "batch_size 3"]:
|
||||
"""
|
||||
Samples pixels within a mask using rejection sampling.
|
||||
|
||||
Args:
|
||||
mask: mask of possible pixels in an image to sample from.
|
||||
num_samples: number of samples.
|
||||
num_images: number of images to sample over.
|
||||
image_height: the height of the image.
|
||||
image_width: the width of the image.
|
||||
device: device that the samples should be on.
|
||||
"""
|
||||
indices = (
|
||||
torch.rand((num_samples, 3), device=device)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
|
||||
num_valid = 0
|
||||
for _ in range(self.config.max_num_iterations):
|
||||
c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1))
|
||||
chosen_indices_validity = mask.squeeze()[c, y, x].bool()
|
||||
num_valid = int(torch.sum(chosen_indices_validity).item())
|
||||
if num_valid == num_samples:
|
||||
break
|
||||
else:
|
||||
replacement_indices = (
|
||||
torch.rand((num_samples - num_valid, 3), device=device)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
indices[~chosen_indices_validity] = replacement_indices
|
||||
|
||||
if num_valid != num_samples:
|
||||
warnings.warn(
|
||||
"""
|
||||
Masked sampling failed, mask is either empty or mostly empty.
|
||||
Reverting behavior to non-rejection sampling. Consider setting
|
||||
pipeline.datamanager.pixel-sampler.rejection-sample-mask to False
|
||||
or increasing pipeline.datamanager.pixel-sampler.max-num-iterations
|
||||
"""
|
||||
)
|
||||
self.config.rejection_sample_mask = False
|
||||
nonzero_indices = torch.nonzero(mask.squeeze(), as_tuple=False).to(device)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=num_samples)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
|
||||
return indices
|
||||
|
||||
def sample_method(
|
||||
self,
|
||||
batch_size: int,
|
||||
@@ -95,44 +150,25 @@ class PixelSampler:
|
||||
num_images: number of images to sample over
|
||||
mask: mask of possible pixels in an image to sample from.
|
||||
"""
|
||||
indices = (
|
||||
torch.rand((batch_size, 3), device=device)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
|
||||
if isinstance(mask, torch.Tensor) and not self.config.ignore_mask:
|
||||
if self.config.rejection_sample_mask:
|
||||
num_valid = 0
|
||||
for _ in range(self.config.max_num_iterations):
|
||||
c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1))
|
||||
chosen_indices_validity = mask[..., 0][c, y, x].bool()
|
||||
num_valid = int(torch.sum(chosen_indices_validity).item())
|
||||
if num_valid == batch_size:
|
||||
break
|
||||
else:
|
||||
replacement_indices = (
|
||||
torch.rand((batch_size - num_valid, 3), device=device)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
indices[~chosen_indices_validity] = replacement_indices
|
||||
|
||||
if num_valid != batch_size:
|
||||
warnings.warn(
|
||||
"""
|
||||
Masked sampling failed, mask is either empty or mostly empty.
|
||||
Reverting behavior to non-rejection sampling. Consider setting
|
||||
pipeline.datamanager.pixel-sampler.rejection-sample-mask to False
|
||||
or increasing pipeline.datamanager.pixel-sampler.max-num-iterations
|
||||
"""
|
||||
)
|
||||
self.config.rejection_sample_mask = False
|
||||
nonzero_indices = torch.nonzero(mask[..., 0], as_tuple=False)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
indices = self.rejection_sample_mask(
|
||||
mask=mask,
|
||||
num_samples=batch_size,
|
||||
num_images=num_images,
|
||||
image_height=image_height,
|
||||
image_width=image_width,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
nonzero_indices = torch.nonzero(mask[..., 0], as_tuple=False)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
else:
|
||||
indices = (
|
||||
torch.rand((batch_size, 3), device=device)
|
||||
* torch.tensor([num_images, image_height, image_width], device=device)
|
||||
).long()
|
||||
|
||||
return indices
|
||||
|
||||
@@ -473,6 +509,10 @@ class PairPixelSamplerConfig(PixelSamplerConfig):
|
||||
"""Target class to instantiate."""
|
||||
radius: int = 2
|
||||
"""max distance between pairs of pixels."""
|
||||
rejection_sample_mask: bool = True
|
||||
"""Whether or not to use rejection sampling when sampling images with masks"""
|
||||
max_num_iterations: int = 100
|
||||
"""If rejection sampling masks, the maximum number of times to sample"""
|
||||
|
||||
|
||||
class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
|
||||
@@ -508,9 +548,20 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
|
||||
|
||||
if isinstance(mask, Tensor) and not self.config.ignore_mask:
|
||||
m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius)
|
||||
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=rays_to_sample)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
|
||||
if self.config.rejection_sample_mask:
|
||||
indices = self.rejection_sample_mask(
|
||||
mask=m,
|
||||
num_samples=rays_to_sample,
|
||||
num_images=num_images,
|
||||
image_height=image_height,
|
||||
image_width=image_width,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device)
|
||||
chosen_indices = random.sample(range(len(nonzero_indices)), k=rays_to_sample)
|
||||
indices = nonzero_indices[chosen_indices]
|
||||
else:
|
||||
s = (rays_to_sample, 1)
|
||||
ns = torch.randint(0, num_images, s, dtype=torch.long, device=device)
|
||||
|
||||
@@ -28,9 +28,7 @@ def dilate(tensor: Float[Tensor, "bs 1 H W"], kernel_size=3) -> Float[Tensor, "b
|
||||
Args:
|
||||
kernel_size: Size of the pooling region. Dilates/contracts 1 pixel if kernel_size is 3.
|
||||
"""
|
||||
|
||||
unique_vals = torch.unique(tensor)
|
||||
if any(val not in (0, 1) for val in unique_vals) or tensor.dtype != torch.float32:
|
||||
if not (tensor.dtype == torch.float32 and torch.all((tensor == 0) | (tensor == 1))):
|
||||
raise ValueError("Input tensor should contain only values 0 and 1, and should have dtype torch.float32.")
|
||||
|
||||
return torch.nn.functional.max_pool2d(tensor, kernel_size=kernel_size, stride=1, padding=(kernel_size - 1) // 2)
|
||||
@@ -42,22 +40,18 @@ def erode(tensor: Float[Tensor, "bs 1 H W"], kernel_size=3) -> Float[Tensor, "bs
|
||||
Args:
|
||||
kernel_size: Size of the pooling region. Erodes/expands 1 pixel if kernel_size is 3.
|
||||
"""
|
||||
|
||||
unique_vals = torch.unique(tensor)
|
||||
if any(val not in (0, 1) for val in unique_vals) or tensor.dtype != torch.float32:
|
||||
if not (tensor.dtype == torch.float32 and torch.all((tensor == 0) | (tensor == 1))):
|
||||
raise ValueError("Input tensor should contain only values 0 and 1, and should have dtype torch.float32.")
|
||||
|
||||
x = 1 - dilate(1 - tensor, kernel_size=kernel_size)
|
||||
# set edge pixels to 0
|
||||
# Invert the tensor, dilate, and re-invert
|
||||
eroded = 1 - dilate(1 - tensor, kernel_size=kernel_size)
|
||||
|
||||
# Use padding to control edge effects (set edge pixels to 0)
|
||||
p = (kernel_size - 1) // 2
|
||||
x[:, :, :p, :] *= 0
|
||||
x[:, :, :, :p] *= 0
|
||||
x[:, :, -p:, :] *= 0
|
||||
x[:, :, :, -p:] *= 0
|
||||
return x
|
||||
return torch.nn.functional.pad(eroded[:, :, p:-p, p:-p], pad=(p, p, p, p), mode="constant", value=0)
|
||||
|
||||
|
||||
def erode_mask(tensor: Float[Tensor, "bs 1 H W"], pixel_radius: int = 1):
|
||||
def erode_mask(tensor: Float[Tensor, "bs 1 H W"], pixel_radius: int = 1) -> Float[Tensor, "bs 1 H W"]:
|
||||
"""Erode a mask. Expands 1 values to nearby pixels with a max pooling operation.
|
||||
A pixel radius of 1 will also extend the 1s along the diagonal.
|
||||
|
||||
|
||||
Reference in new issue
Block a user