From 076c5bf0b7bdfe84d2cadff2fd6e08cde85cdce6 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Fri, 29 Apr 2022 15:37:33 -0700 Subject: [PATCH 1/2] Add stratified sampling and enable pdf sampling --- mattport/nerf/graph/vanilla_nerf.py | 9 ++----- mattport/nerf/sampler.py | 40 ++++++++++++++++++++--------- mattport/nerf/trainer.py | 3 +++ mattport/structures/rays.py | 15 ++++++----- 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/mattport/nerf/graph/vanilla_nerf.py b/mattport/nerf/graph/vanilla_nerf.py index c3af7c9dc..1f7a0a8f9 100644 --- a/mattport/nerf/graph/vanilla_nerf.py +++ b/mattport/nerf/graph/vanilla_nerf.py @@ -103,11 +103,7 @@ class NeRFGraph(Graph): self.sampler_uniform = UniformSampler( near_plane=self.near_plane, far_plane=self.far_plane, num_samples=self.num_coarse_samples ) - # TODO: this needs to be corrected - # self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples) - self.sampler_pdf = UniformSampler( - near_plane=self.near_plane, far_plane=self.far_plane, num_samples=self.num_importance_samples - ) + self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples) # field self.field_coarse = NeRFField() @@ -147,8 +143,7 @@ class NeRFGraph(Graph): deltas=uniform_ray_samples.deltas, ) # RendererOutputs # fine network: - # pdf_ray_samples = self.sampler_pdf(ray_bundle, uniform_ray_samples, coarse_field_outputs) # RaySamples - pdf_ray_samples = self.sampler_pdf(ray_bundle) # RaySamples + pdf_ray_samples = self.sampler_pdf(ray_bundle, uniform_ray_samples, coarse_field_outputs) # RaySamples fine_field_outputs = self.field_fine(pdf_ray_samples) # FieldOutputs fine_renderer_outputs = self.renderer_rgb( diff --git a/mattport/nerf/sampler.py b/mattport/nerf/sampler.py index 440af251f..7915209da 100644 --- a/mattport/nerf/sampler.py +++ b/mattport/nerf/sampler.py @@ -14,18 +14,20 @@ from mattport.structures.rays import RayBundle, RaySamples class UniformSampler(nn.Module): """Sample uniformly along a ray""" - def __init__(self, near_plane: float, far_plane: float, num_samples: int) -> None: + def __init__(self, near_plane: float, far_plane: float, num_samples: int, train_stratified=True) -> None: """ Args: near_plane (float): Minimum distance along ray to sample far_plane (float): Maximum distance along ray to sample num_samples (int): Number of samples per ray + train_stratified (bool): Use stratified sampling during training. Defults to True """ super().__init__() self.near_plane = near_plane self.far_plane = far_plane self.num_samples = num_samples + self.train_stratified = train_stratified def forward( self, @@ -52,11 +54,17 @@ class UniformSampler(nn.Module): num_rays = ray_bundle.origins.shape[0] - bins = torch.linspace(near_plane, far_plane, num_samples + 1).to(ray_bundle.origins.device) # (num_samples+1,) - bins = bins.unsqueeze(0).repeat(num_rays, 1) # (num_rays, num_samples+1) + bins = torch.linspace(near_plane, far_plane, num_samples + 1).to(ray_bundle.origins.device) # (num_samples+1) + + if self.train_stratified and self.training: + t_rand = torch.rand((num_rays, num_samples), dtype=bins.dtype, device=bins.device) + ts = bins[None, :-1] + t_rand * (bins[None, 1:] - bins[None, :-1]) # (num_rays, num_samples) + else: + ts = (bins[1:] + bins[:-1]) / 2 + ts = ts.unsqueeze(0).repeat(num_rays, 1) # (num_rays, num_samples) ray_samples = RaySamples( - bins=bins, + ts=ts, ray_bundle=ray_bundle, ) @@ -66,14 +74,18 @@ class UniformSampler(nn.Module): class PDFSampler(nn.Module): """Sample based on probability distribution""" - def __init__(self, num_samples: int) -> None: + def __init__(self, num_samples: int, train_stratified: bool = True, include_original: bool = True) -> None: """ Args: num_samples (int): Number of samples per ray + train_stratified: boolean: Randomize location within each bin during training. Defaults to True + include_original: Add original samples to ray. Defaults to True """ super().__init__() self.num_samples = num_samples + self.include_original = include_original + self.train_stratified = train_stratified def forward( self, @@ -81,7 +93,6 @@ class PDFSampler(nn.Module): coarse_ray_samples: RaySamples, field_outputs: Dict[FieldHeadNames, TensorType], num_samples: Optional[int] = None, - randomized: bool = True, eps: float = 1e-5, ) -> RaySamples: """Generates position samples given a distribution. @@ -92,7 +103,6 @@ class PDFSampler(nn.Module): bins (TensorType[..., "num_samples"]): Ray bins weights: (TensorType[..., "num_samples"]): Weights for each bin num_samples (Optional[int]): Number of samples per ray - randomized: boolean: Randomize location within each bin. Defaults to True eps: float: Small value to prevent numerical issues. Defaults to 1e-5 Returns: @@ -113,6 +123,7 @@ class PDFSampler(nn.Module): ) transmittance = torch.exp(-transmittance) # [..., "num_samples"] weights = alphas * transmittance # [..., "num_samples"] + weights = weights[..., 1:] # Add small offset to rays with zero weight to prevent NaNs weights_sum = torch.sum(weights, dim=-1, keepdim=True) @@ -124,13 +135,13 @@ class PDFSampler(nn.Module): cdf = torch.min(torch.ones_like(pdf), torch.cumsum(pdf, dim=-1)) cdf = torch.cat([torch.zeros_like(cdf[..., :1]), cdf], dim=-1) - if randomized: + if self.train_stratified and self.training: u = torch.rand(size=(*cdf.shape[:-1], num_samples), device=cdf.device) else: u = torch.linspace(0.0, 1.0, steps=num_samples, device=cdf.device) u = torch.expand(size=(*cdf.shape[:-1], num_samples)) - # u = u.contiguous() + u = u.contiguous() indicies = torch.searchsorted(cdf, u, right=True) below = torch.max(torch.zeros_like(indicies), indicies - 1) above = torch.min(cdf.shape[-1] - torch.ones_like(indicies), indicies) @@ -138,15 +149,20 @@ class PDFSampler(nn.Module): indicies_g = torch.stack([below, above], -1) matched_shape = (indicies_g.shape[0], indicies_g.shape[1], cdf.shape[-1]) cdf_g = torch.gather(cdf.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g) - bins_g = torch.gather(coarse_ray_samples.bins.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g) + bins_g = torch.gather(coarse_ray_samples.ts.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g) denom = cdf_g[..., 1] - cdf_g[..., 0] denom = torch.where(denom < eps, torch.ones_like(denom), denom) t = (u - cdf_g[..., 0]) / denom - samples = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0]) + ts = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0]) + + if self.include_original: + ts, _ = torch.sort(torch.cat([coarse_ray_samples.ts, ts], -1), -1) + + ts = ts.detach() ray_samples = RaySamples( - bins=samples, # TODO(matt) These are not bins! + ts=ts, ray_bundle=ray_bundle, ) diff --git a/mattport/nerf/trainer.py b/mattport/nerf/trainer.py index ecc44dc63..766028057 100644 --- a/mattport/nerf/trainer.py +++ b/mattport/nerf/trainer.py @@ -233,6 +233,9 @@ class Trainer: combined_image = torch.cat([image, rgb_coarse, rgb_fine], dim=1) self.writer.write_image(f"image_idx_{image_idx}-rgb_coarse_fine", combined_image, step, group="val") + coarse_psnr = get_psnr(image, rgb_coarse) + self.writer.write_scalar(f"image_idx_{image_idx}-coarse_psnr", float(coarse_psnr), step, group="val") + fine_psnr = get_psnr(image, rgb_fine) self.stats.update_value(Stats.CURR_TEST_PSNR, float(fine_psnr), step) self.writer.write_scalar(f"image_idx_{image_idx}-fine_psnr", float(fine_psnr), step, group="val") diff --git a/mattport/structures/rays.py b/mattport/structures/rays.py index 3ab16f782..b54af6488 100644 --- a/mattport/structures/rays.py +++ b/mattport/structures/rays.py @@ -4,6 +4,7 @@ Some ray datastructures. import random from dataclasses import dataclass from typing import Optional +import torch from torchtyping import TensorType @@ -72,22 +73,22 @@ class RaySamples: def __init__( self, - bins: TensorType["num_rays", "num_samples+1"], + ts: TensorType["num_rays", "num_samples"], ray_bundle: RayBundle, ) -> None: - self.bins = bins + self.ray_bundle = ray_bundle + self.ts = ts self.positions = self.get_positions(ray_bundle) self.directions = ray_bundle.directions.unsqueeze(1).repeat(1, self.positions.shape[1], 1) self.deltas = self.get_deltas() def get_positions(self, ray_bundle: RayBundle) -> TensorType["num_rays", "num_samples", 3]: """Returns positions.""" - # TODO(ethan): check with Matt about this - t_mid = (self.bins[:, 1:] + self.bins[:, :-1]) / 2 # (num_rays, num_samples) - return ray_bundle.origins[:, None] + t_mid[:, :, None] * ray_bundle.directions[:, None] + return ray_bundle.origins[:, None] + self.ts[:, :, None] * ray_bundle.directions[:, None] def get_deltas(self) -> TensorType[..., "num_samples"]: """Returns deltas.""" - deltas = self.bins[..., 1:] - self.bins[..., :-1] - # TODO(ethan): check this with Matt + dists = self.ts[..., 1:] - self.ts[..., :-1] + dists = torch.cat([dists, dists[..., -1:]], -1) # [N_rays, N_samples] + deltas = dists * torch.norm(self.ray_bundle.directions[..., None, :], dim=-1) return deltas From cbc85ef450721a5862357ec0a4954f3addee07e2 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Fri, 29 Apr 2022 15:44:19 -0700 Subject: [PATCH 2/2] Fix test --- tests/nerf/test_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nerf/test_sampler.py b/tests/nerf/test_sampler.py index 8e884a4be..167bda16b 100644 --- a/tests/nerf/test_sampler.py +++ b/tests/nerf/test_sampler.py @@ -21,7 +21,7 @@ def test_uniform_sampler(): ray_samples = sampler(ray_bundle) - assert ray_samples.bins.shape[-1] == num_samples + 1 + assert ray_samples.positions.shape[-2] == num_samples # TODO Tancik: Add more precise tests