Merge pull request #48 from ethanweber/tancik/stratified_and_pdf_sampling

Tancik/stratified and pdf sampling
This commit is contained in:
Matthew Tancik authored and GitHub committed 2022-04-29 15:46:31 -07:00
commit d79c112248
5 files changed
+42 -27

No files matched your search

+2 -7
View File
@@ -103,11 +103,7 @@ class NeRFGraph(Graph):
self.sampler_uniform = UniformSampler(
near_plane=self.near_plane, far_plane=self.far_plane, num_samples=self.num_coarse_samples
)
# TODO: this needs to be corrected
# self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples)
self.sampler_pdf = UniformSampler(
near_plane=self.near_plane, far_plane=self.far_plane, num_samples=self.num_importance_samples
)
self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples)
# field
self.field_coarse = NeRFField()
@@ -147,8 +143,7 @@ class NeRFGraph(Graph):
deltas=uniform_ray_samples.deltas,
) # RendererOutputs
# fine network:
# pdf_ray_samples = self.sampler_pdf(ray_bundle, uniform_ray_samples, coarse_field_outputs) # RaySamples
pdf_ray_samples = self.sampler_pdf(ray_bundle) # RaySamples
pdf_ray_samples = self.sampler_pdf(ray_bundle, uniform_ray_samples, coarse_field_outputs) # RaySamples
fine_field_outputs = self.field_fine(pdf_ray_samples) # FieldOutputs
fine_renderer_outputs = self.renderer_rgb(
+28 -12
View File
@@ -14,18 +14,20 @@ from mattport.structures.rays import RayBundle, RaySamples
class UniformSampler(nn.Module):
"""Sample uniformly along a ray"""
def __init__(self, near_plane: float, far_plane: float, num_samples: int) -> None:
def __init__(self, near_plane: float, far_plane: float, num_samples: int, train_stratified=True) -> None:
"""
Args:
near_plane (float): Minimum distance along ray to sample
far_plane (float): Maximum distance along ray to sample
num_samples (int): Number of samples per ray
train_stratified (bool): Use stratified sampling during training. Defults to True
"""
super().__init__()
self.near_plane = near_plane
self.far_plane = far_plane
self.num_samples = num_samples
self.train_stratified = train_stratified
def forward(
self,
@@ -52,11 +54,17 @@ class UniformSampler(nn.Module):
num_rays = ray_bundle.origins.shape[0]
bins = torch.linspace(near_plane, far_plane, num_samples + 1).to(ray_bundle.origins.device) # (num_samples+1,)
bins = bins.unsqueeze(0).repeat(num_rays, 1) # (num_rays, num_samples+1)
bins = torch.linspace(near_plane, far_plane, num_samples + 1).to(ray_bundle.origins.device) # (num_samples+1)
if self.train_stratified and self.training:
t_rand = torch.rand((num_rays, num_samples), dtype=bins.dtype, device=bins.device)
ts = bins[None, :-1] + t_rand * (bins[None, 1:] - bins[None, :-1]) # (num_rays, num_samples)
else:
ts = (bins[1:] + bins[:-1]) / 2
ts = ts.unsqueeze(0).repeat(num_rays, 1) # (num_rays, num_samples)
ray_samples = RaySamples(
bins=bins,
ts=ts,
ray_bundle=ray_bundle,
)
@@ -66,14 +74,18 @@ class UniformSampler(nn.Module):
class PDFSampler(nn.Module):
"""Sample based on probability distribution"""
def __init__(self, num_samples: int) -> None:
def __init__(self, num_samples: int, train_stratified: bool = True, include_original: bool = True) -> None:
"""
Args:
num_samples (int): Number of samples per ray
train_stratified: boolean: Randomize location within each bin during training. Defaults to True
include_original: Add original samples to ray. Defaults to True
"""
super().__init__()
self.num_samples = num_samples
self.include_original = include_original
self.train_stratified = train_stratified
def forward(
self,
@@ -81,7 +93,6 @@ class PDFSampler(nn.Module):
coarse_ray_samples: RaySamples,
field_outputs: Dict[FieldHeadNames, TensorType],
num_samples: Optional[int] = None,
randomized: bool = True,
eps: float = 1e-5,
) -> RaySamples:
"""Generates position samples given a distribution.
@@ -92,7 +103,6 @@ class PDFSampler(nn.Module):
bins (TensorType[..., "num_samples"]): Ray bins
weights: (TensorType[..., "num_samples"]): Weights for each bin
num_samples (Optional[int]): Number of samples per ray
randomized: boolean: Randomize location within each bin. Defaults to True
eps: float: Small value to prevent numerical issues. Defaults to 1e-5
Returns:
@@ -113,6 +123,7 @@ class PDFSampler(nn.Module):
)
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
weights = alphas * transmittance # [..., "num_samples"]
weights = weights[..., 1:]
# Add small offset to rays with zero weight to prevent NaNs
weights_sum = torch.sum(weights, dim=-1, keepdim=True)
@@ -124,13 +135,13 @@ class PDFSampler(nn.Module):
cdf = torch.min(torch.ones_like(pdf), torch.cumsum(pdf, dim=-1))
cdf = torch.cat([torch.zeros_like(cdf[..., :1]), cdf], dim=-1)
if randomized:
if self.train_stratified and self.training:
u = torch.rand(size=(*cdf.shape[:-1], num_samples), device=cdf.device)
else:
u = torch.linspace(0.0, 1.0, steps=num_samples, device=cdf.device)
u = torch.expand(size=(*cdf.shape[:-1], num_samples))
# u = u.contiguous()
u = u.contiguous()
indicies = torch.searchsorted(cdf, u, right=True)
below = torch.max(torch.zeros_like(indicies), indicies - 1)
above = torch.min(cdf.shape[-1] - torch.ones_like(indicies), indicies)
@@ -138,15 +149,20 @@ class PDFSampler(nn.Module):
indicies_g = torch.stack([below, above], -1)
matched_shape = (indicies_g.shape[0], indicies_g.shape[1], cdf.shape[-1])
cdf_g = torch.gather(cdf.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g)
bins_g = torch.gather(coarse_ray_samples.bins.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g)
bins_g = torch.gather(coarse_ray_samples.ts.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g)
denom = cdf_g[..., 1] - cdf_g[..., 0]
denom = torch.where(denom < eps, torch.ones_like(denom), denom)
t = (u - cdf_g[..., 0]) / denom
samples = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0])
ts = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0])
if self.include_original:
ts, _ = torch.sort(torch.cat([coarse_ray_samples.ts, ts], -1), -1)
ts = ts.detach()
ray_samples = RaySamples(
bins=samples, # TODO(matt) These are not bins!
ts=ts,
ray_bundle=ray_bundle,
)
+3
View File
@@ -252,6 +252,9 @@ class Trainer:
combined_image = torch.cat([image, rgb_coarse, rgb_fine], dim=1)
self.writer.write_image(f"image_idx_{image_idx}-rgb_coarse_fine", combined_image, step, group="val")
coarse_psnr = get_psnr(image, rgb_coarse)
self.writer.write_scalar(f"image_idx_{image_idx}-coarse_psnr", float(coarse_psnr), step, group="val")
fine_psnr = get_psnr(image, rgb_fine)
self.stats.update_value(Stats.CURR_TEST_PSNR, float(fine_psnr), step)
self.writer.write_scalar(f"image_idx_{image_idx}-fine_psnr", float(fine_psnr), step, group="val")
+8 -7
View File
@@ -4,6 +4,7 @@ Some ray datastructures.
import random
from dataclasses import dataclass
from typing import Optional
import torch
from torchtyping import TensorType
@@ -72,22 +73,22 @@ class RaySamples:
def __init__(
self,
bins: TensorType["num_rays", "num_samples+1"],
ts: TensorType["num_rays", "num_samples"],
ray_bundle: RayBundle,
) -> None:
self.bins = bins
self.ray_bundle = ray_bundle
self.ts = ts
self.positions = self.get_positions(ray_bundle)
self.directions = ray_bundle.directions.unsqueeze(1).repeat(1, self.positions.shape[1], 1)
self.deltas = self.get_deltas()
def get_positions(self, ray_bundle: RayBundle) -> TensorType["num_rays", "num_samples", 3]:
"""Returns positions."""
# TODO(ethan): check with Matt about this
t_mid = (self.bins[:, 1:] + self.bins[:, :-1]) / 2 # (num_rays, num_samples)
return ray_bundle.origins[:, None] + t_mid[:, :, None] * ray_bundle.directions[:, None]
return ray_bundle.origins[:, None] + self.ts[:, :, None] * ray_bundle.directions[:, None]
def get_deltas(self) -> TensorType[..., "num_samples"]:
"""Returns deltas."""
deltas = self.bins[..., 1:] - self.bins[..., :-1]
# TODO(ethan): check this with Matt
dists = self.ts[..., 1:] - self.ts[..., :-1]
dists = torch.cat([dists, dists[..., -1:]], -1) # [N_rays, N_samples]
deltas = dists * torch.norm(self.ray_bundle.directions[..., None, :], dim=-1)
return deltas
+1 -1
View File
@@ -21,7 +21,7 @@ def test_uniform_sampler():
ray_samples = sampler(ray_bundle)
assert ray_samples.bins.shape[-1] == num_samples + 1
assert ray_samples.positions.shape[-2] == num_samples
# TODO Tancik: Add more precise tests