From 4c7297c75a9459488d672510af246caed35438b8 Mon Sep 17 00:00:00 2001 From: Nicholas Pfaff <53228351+nepfaff@users.noreply.github.com> Date: Mon, 30 Sep 2024 20:31:49 -0400 Subject: [PATCH] Speed up PairPixelSampler (#3452) --- nerfstudio/data/pixel_samplers.py | 121 +++++++++++++----- nerfstudio/data/utils/pixel_sampling_utils.py | 22 ++-- 2 files changed, 94 insertions(+), 49 deletions(-) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 080b4016a..a0f1dd67c 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -78,6 +78,61 @@ class PixelSampler: """ self.num_rays_per_batch = num_rays_per_batch + def rejection_sample_mask( + self, + mask: Tensor, + num_samples: int, + num_images: int, + image_height: int, + image_width: int, + device: Union[torch.device, str], + ) -> Int[Tensor, "batch_size 3"]: + """ + Samples pixels within a mask using rejection sampling. + + Args: + mask: mask of possible pixels in an image to sample from. + num_samples: number of samples. + num_images: number of images to sample over. + image_height: the height of the image. + image_width: the width of the image. + device: device that the samples should be on. + """ + indices = ( + torch.rand((num_samples, 3), device=device) + * torch.tensor([num_images, image_height, image_width], device=device) + ).long() + + num_valid = 0 + for _ in range(self.config.max_num_iterations): + c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) + chosen_indices_validity = mask.squeeze()[c, y, x].bool() + num_valid = int(torch.sum(chosen_indices_validity).item()) + if num_valid == num_samples: + break + else: + replacement_indices = ( + torch.rand((num_samples - num_valid, 3), device=device) + * torch.tensor([num_images, image_height, image_width], device=device) + ).long() + indices[~chosen_indices_validity] = replacement_indices + + if num_valid != num_samples: + warnings.warn( + """ + Masked sampling failed, mask is either empty or mostly empty. + Reverting behavior to non-rejection sampling. Consider setting + pipeline.datamanager.pixel-sampler.rejection-sample-mask to False + or increasing pipeline.datamanager.pixel-sampler.max-num-iterations + """ + ) + self.config.rejection_sample_mask = False + nonzero_indices = torch.nonzero(mask.squeeze(), as_tuple=False).to(device) + chosen_indices = random.sample(range(len(nonzero_indices)), k=num_samples) + indices = nonzero_indices[chosen_indices] + + return indices + def sample_method( self, batch_size: int, @@ -95,44 +150,25 @@ class PixelSampler: num_images: number of images to sample over mask: mask of possible pixels in an image to sample from. """ - indices = ( - torch.rand((batch_size, 3), device=device) - * torch.tensor([num_images, image_height, image_width], device=device) - ).long() - if isinstance(mask, torch.Tensor) and not self.config.ignore_mask: if self.config.rejection_sample_mask: - num_valid = 0 - for _ in range(self.config.max_num_iterations): - c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) - chosen_indices_validity = mask[..., 0][c, y, x].bool() - num_valid = int(torch.sum(chosen_indices_validity).item()) - if num_valid == batch_size: - break - else: - replacement_indices = ( - torch.rand((batch_size - num_valid, 3), device=device) - * torch.tensor([num_images, image_height, image_width], device=device) - ).long() - indices[~chosen_indices_validity] = replacement_indices - - if num_valid != batch_size: - warnings.warn( - """ - Masked sampling failed, mask is either empty or mostly empty. - Reverting behavior to non-rejection sampling. Consider setting - pipeline.datamanager.pixel-sampler.rejection-sample-mask to False - or increasing pipeline.datamanager.pixel-sampler.max-num-iterations - """ - ) - self.config.rejection_sample_mask = False - nonzero_indices = torch.nonzero(mask[..., 0], as_tuple=False) - chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size) - indices = nonzero_indices[chosen_indices] + indices = self.rejection_sample_mask( + mask=mask, + num_samples=batch_size, + num_images=num_images, + image_height=image_height, + image_width=image_width, + device=device, + ) else: nonzero_indices = torch.nonzero(mask[..., 0], as_tuple=False) chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size) indices = nonzero_indices[chosen_indices] + else: + indices = ( + torch.rand((batch_size, 3), device=device) + * torch.tensor([num_images, image_height, image_width], device=device) + ).long() return indices @@ -473,6 +509,10 @@ class PairPixelSamplerConfig(PixelSamplerConfig): """Target class to instantiate.""" radius: int = 2 """max distance between pairs of pixels.""" + rejection_sample_mask: bool = True + """Whether or not to use rejection sampling when sampling images with masks""" + max_num_iterations: int = 100 + """If rejection sampling masks, the maximum number of times to sample""" class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods @@ -508,9 +548,20 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods if isinstance(mask, Tensor) and not self.config.ignore_mask: m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius) - nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device) - chosen_indices = random.sample(range(len(nonzero_indices)), k=rays_to_sample) - indices = nonzero_indices[chosen_indices] + + if self.config.rejection_sample_mask: + indices = self.rejection_sample_mask( + mask=m, + num_samples=rays_to_sample, + num_images=num_images, + image_height=image_height, + image_width=image_width, + device=device, + ) + else: + nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device) + chosen_indices = random.sample(range(len(nonzero_indices)), k=rays_to_sample) + indices = nonzero_indices[chosen_indices] else: s = (rays_to_sample, 1) ns = torch.randint(0, num_images, s, dtype=torch.long, device=device) diff --git a/nerfstudio/data/utils/pixel_sampling_utils.py b/nerfstudio/data/utils/pixel_sampling_utils.py index d49aa22f1..ae77b3287 100644 --- a/nerfstudio/data/utils/pixel_sampling_utils.py +++ b/nerfstudio/data/utils/pixel_sampling_utils.py @@ -28,9 +28,7 @@ def dilate(tensor: Float[Tensor, "bs 1 H W"], kernel_size=3) -> Float[Tensor, "b Args: kernel_size: Size of the pooling region. Dilates/contracts 1 pixel if kernel_size is 3. """ - - unique_vals = torch.unique(tensor) - if any(val not in (0, 1) for val in unique_vals) or tensor.dtype != torch.float32: + if not (tensor.dtype == torch.float32 and torch.all((tensor == 0) | (tensor == 1))): raise ValueError("Input tensor should contain only values 0 and 1, and should have dtype torch.float32.") return torch.nn.functional.max_pool2d(tensor, kernel_size=kernel_size, stride=1, padding=(kernel_size - 1) // 2) @@ -42,22 +40,18 @@ def erode(tensor: Float[Tensor, "bs 1 H W"], kernel_size=3) -> Float[Tensor, "bs Args: kernel_size: Size of the pooling region. Erodes/expands 1 pixel if kernel_size is 3. """ - - unique_vals = torch.unique(tensor) - if any(val not in (0, 1) for val in unique_vals) or tensor.dtype != torch.float32: + if not (tensor.dtype == torch.float32 and torch.all((tensor == 0) | (tensor == 1))): raise ValueError("Input tensor should contain only values 0 and 1, and should have dtype torch.float32.") - x = 1 - dilate(1 - tensor, kernel_size=kernel_size) - # set edge pixels to 0 + # Invert the tensor, dilate, and re-invert + eroded = 1 - dilate(1 - tensor, kernel_size=kernel_size) + + # Use padding to control edge effects (set edge pixels to 0) p = (kernel_size - 1) // 2 - x[:, :, :p, :] *= 0 - x[:, :, :, :p] *= 0 - x[:, :, -p:, :] *= 0 - x[:, :, :, -p:] *= 0 - return x + return torch.nn.functional.pad(eroded[:, :, p:-p, p:-p], pad=(p, p, p, p), mode="constant", value=0) -def erode_mask(tensor: Float[Tensor, "bs 1 H W"], pixel_radius: int = 1): +def erode_mask(tensor: Float[Tensor, "bs 1 H W"], pixel_radius: int = 1) -> Float[Tensor, "bs 1 H W"]: """Erode a mask. Expands 1 values to nearby pixels with a max pooling operation. A pixel radius of 1 will also extend the 1s along the diagonal.