mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 04:43:52 +08:00
Merge pull request #48 from ethanweber/tancik/stratified_and_pdf_sampling
Tancik/stratified and pdf sampling
This commit is contained in:
5 files changed
+42
-27
No files matched your search
@@ -103,11 +103,7 @@ class NeRFGraph(Graph):
|
||||
self.sampler_uniform = UniformSampler(
|
||||
near_plane=self.near_plane, far_plane=self.far_plane, num_samples=self.num_coarse_samples
|
||||
)
|
||||
# TODO: this needs to be corrected
|
||||
# self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples)
|
||||
self.sampler_pdf = UniformSampler(
|
||||
near_plane=self.near_plane, far_plane=self.far_plane, num_samples=self.num_importance_samples
|
||||
)
|
||||
self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples)
|
||||
|
||||
# field
|
||||
self.field_coarse = NeRFField()
|
||||
@@ -147,8 +143,7 @@ class NeRFGraph(Graph):
|
||||
deltas=uniform_ray_samples.deltas,
|
||||
) # RendererOutputs
|
||||
# fine network:
|
||||
# pdf_ray_samples = self.sampler_pdf(ray_bundle, uniform_ray_samples, coarse_field_outputs) # RaySamples
|
||||
pdf_ray_samples = self.sampler_pdf(ray_bundle) # RaySamples
|
||||
pdf_ray_samples = self.sampler_pdf(ray_bundle, uniform_ray_samples, coarse_field_outputs) # RaySamples
|
||||
fine_field_outputs = self.field_fine(pdf_ray_samples) # FieldOutputs
|
||||
|
||||
fine_renderer_outputs = self.renderer_rgb(
|
||||
|
||||
+28
-12
@@ -14,18 +14,20 @@ from mattport.structures.rays import RayBundle, RaySamples
|
||||
class UniformSampler(nn.Module):
|
||||
"""Sample uniformly along a ray"""
|
||||
|
||||
def __init__(self, near_plane: float, far_plane: float, num_samples: int) -> None:
|
||||
def __init__(self, near_plane: float, far_plane: float, num_samples: int, train_stratified=True) -> None:
|
||||
"""
|
||||
Args:
|
||||
near_plane (float): Minimum distance along ray to sample
|
||||
far_plane (float): Maximum distance along ray to sample
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified (bool): Use stratified sampling during training. Defults to True
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.near_plane = near_plane
|
||||
self.far_plane = far_plane
|
||||
self.num_samples = num_samples
|
||||
self.train_stratified = train_stratified
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -52,11 +54,17 @@ class UniformSampler(nn.Module):
|
||||
|
||||
num_rays = ray_bundle.origins.shape[0]
|
||||
|
||||
bins = torch.linspace(near_plane, far_plane, num_samples + 1).to(ray_bundle.origins.device) # (num_samples+1,)
|
||||
bins = bins.unsqueeze(0).repeat(num_rays, 1) # (num_rays, num_samples+1)
|
||||
bins = torch.linspace(near_plane, far_plane, num_samples + 1).to(ray_bundle.origins.device) # (num_samples+1)
|
||||
|
||||
if self.train_stratified and self.training:
|
||||
t_rand = torch.rand((num_rays, num_samples), dtype=bins.dtype, device=bins.device)
|
||||
ts = bins[None, :-1] + t_rand * (bins[None, 1:] - bins[None, :-1]) # (num_rays, num_samples)
|
||||
else:
|
||||
ts = (bins[1:] + bins[:-1]) / 2
|
||||
ts = ts.unsqueeze(0).repeat(num_rays, 1) # (num_rays, num_samples)
|
||||
|
||||
ray_samples = RaySamples(
|
||||
bins=bins,
|
||||
ts=ts,
|
||||
ray_bundle=ray_bundle,
|
||||
)
|
||||
|
||||
@@ -66,14 +74,18 @@ class UniformSampler(nn.Module):
|
||||
class PDFSampler(nn.Module):
|
||||
"""Sample based on probability distribution"""
|
||||
|
||||
def __init__(self, num_samples: int) -> None:
|
||||
def __init__(self, num_samples: int, train_stratified: bool = True, include_original: bool = True) -> None:
|
||||
"""
|
||||
Args:
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified: boolean: Randomize location within each bin during training. Defaults to True
|
||||
include_original: Add original samples to ray. Defaults to True
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.num_samples = num_samples
|
||||
self.include_original = include_original
|
||||
self.train_stratified = train_stratified
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -81,7 +93,6 @@ class PDFSampler(nn.Module):
|
||||
coarse_ray_samples: RaySamples,
|
||||
field_outputs: Dict[FieldHeadNames, TensorType],
|
||||
num_samples: Optional[int] = None,
|
||||
randomized: bool = True,
|
||||
eps: float = 1e-5,
|
||||
) -> RaySamples:
|
||||
"""Generates position samples given a distribution.
|
||||
@@ -92,7 +103,6 @@ class PDFSampler(nn.Module):
|
||||
bins (TensorType[..., "num_samples"]): Ray bins
|
||||
weights: (TensorType[..., "num_samples"]): Weights for each bin
|
||||
num_samples (Optional[int]): Number of samples per ray
|
||||
randomized: boolean: Randomize location within each bin. Defaults to True
|
||||
eps: float: Small value to prevent numerical issues. Defaults to 1e-5
|
||||
|
||||
Returns:
|
||||
@@ -113,6 +123,7 @@ class PDFSampler(nn.Module):
|
||||
)
|
||||
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
|
||||
weights = alphas * transmittance # [..., "num_samples"]
|
||||
weights = weights[..., 1:]
|
||||
|
||||
# Add small offset to rays with zero weight to prevent NaNs
|
||||
weights_sum = torch.sum(weights, dim=-1, keepdim=True)
|
||||
@@ -124,13 +135,13 @@ class PDFSampler(nn.Module):
|
||||
cdf = torch.min(torch.ones_like(pdf), torch.cumsum(pdf, dim=-1))
|
||||
cdf = torch.cat([torch.zeros_like(cdf[..., :1]), cdf], dim=-1)
|
||||
|
||||
if randomized:
|
||||
if self.train_stratified and self.training:
|
||||
u = torch.rand(size=(*cdf.shape[:-1], num_samples), device=cdf.device)
|
||||
else:
|
||||
u = torch.linspace(0.0, 1.0, steps=num_samples, device=cdf.device)
|
||||
u = torch.expand(size=(*cdf.shape[:-1], num_samples))
|
||||
|
||||
# u = u.contiguous()
|
||||
u = u.contiguous()
|
||||
indicies = torch.searchsorted(cdf, u, right=True)
|
||||
below = torch.max(torch.zeros_like(indicies), indicies - 1)
|
||||
above = torch.min(cdf.shape[-1] - torch.ones_like(indicies), indicies)
|
||||
@@ -138,15 +149,20 @@ class PDFSampler(nn.Module):
|
||||
indicies_g = torch.stack([below, above], -1)
|
||||
matched_shape = (indicies_g.shape[0], indicies_g.shape[1], cdf.shape[-1])
|
||||
cdf_g = torch.gather(cdf.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g)
|
||||
bins_g = torch.gather(coarse_ray_samples.bins.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g)
|
||||
bins_g = torch.gather(coarse_ray_samples.ts.unsqueeze(1).expand(matched_shape), dim=2, index=indicies_g)
|
||||
|
||||
denom = cdf_g[..., 1] - cdf_g[..., 0]
|
||||
denom = torch.where(denom < eps, torch.ones_like(denom), denom)
|
||||
t = (u - cdf_g[..., 0]) / denom
|
||||
samples = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0])
|
||||
ts = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0])
|
||||
|
||||
if self.include_original:
|
||||
ts, _ = torch.sort(torch.cat([coarse_ray_samples.ts, ts], -1), -1)
|
||||
|
||||
ts = ts.detach()
|
||||
|
||||
ray_samples = RaySamples(
|
||||
bins=samples, # TODO(matt) These are not bins!
|
||||
ts=ts,
|
||||
ray_bundle=ray_bundle,
|
||||
)
|
||||
|
||||
|
||||
@@ -252,6 +252,9 @@ class Trainer:
|
||||
combined_image = torch.cat([image, rgb_coarse, rgb_fine], dim=1)
|
||||
self.writer.write_image(f"image_idx_{image_idx}-rgb_coarse_fine", combined_image, step, group="val")
|
||||
|
||||
coarse_psnr = get_psnr(image, rgb_coarse)
|
||||
self.writer.write_scalar(f"image_idx_{image_idx}-coarse_psnr", float(coarse_psnr), step, group="val")
|
||||
|
||||
fine_psnr = get_psnr(image, rgb_fine)
|
||||
self.stats.update_value(Stats.CURR_TEST_PSNR, float(fine_psnr), step)
|
||||
self.writer.write_scalar(f"image_idx_{image_idx}-fine_psnr", float(fine_psnr), step, group="val")
|
||||
@@ -4,6 +4,7 @@ Some ray datastructures.
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
import torch
|
||||
|
||||
from torchtyping import TensorType
|
||||
|
||||
@@ -72,22 +73,22 @@ class RaySamples:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bins: TensorType["num_rays", "num_samples+1"],
|
||||
ts: TensorType["num_rays", "num_samples"],
|
||||
ray_bundle: RayBundle,
|
||||
) -> None:
|
||||
self.bins = bins
|
||||
self.ray_bundle = ray_bundle
|
||||
self.ts = ts
|
||||
self.positions = self.get_positions(ray_bundle)
|
||||
self.directions = ray_bundle.directions.unsqueeze(1).repeat(1, self.positions.shape[1], 1)
|
||||
self.deltas = self.get_deltas()
|
||||
|
||||
def get_positions(self, ray_bundle: RayBundle) -> TensorType["num_rays", "num_samples", 3]:
|
||||
"""Returns positions."""
|
||||
# TODO(ethan): check with Matt about this
|
||||
t_mid = (self.bins[:, 1:] + self.bins[:, :-1]) / 2 # (num_rays, num_samples)
|
||||
return ray_bundle.origins[:, None] + t_mid[:, :, None] * ray_bundle.directions[:, None]
|
||||
return ray_bundle.origins[:, None] + self.ts[:, :, None] * ray_bundle.directions[:, None]
|
||||
|
||||
def get_deltas(self) -> TensorType[..., "num_samples"]:
|
||||
"""Returns deltas."""
|
||||
deltas = self.bins[..., 1:] - self.bins[..., :-1]
|
||||
# TODO(ethan): check this with Matt
|
||||
dists = self.ts[..., 1:] - self.ts[..., :-1]
|
||||
dists = torch.cat([dists, dists[..., -1:]], -1) # [N_rays, N_samples]
|
||||
deltas = dists * torch.norm(self.ray_bundle.directions[..., None, :], dim=-1)
|
||||
return deltas
|
||||
@@ -21,7 +21,7 @@ def test_uniform_sampler():
|
||||
|
||||
ray_samples = sampler(ray_bundle)
|
||||
|
||||
assert ray_samples.bins.shape[-1] == num_samples + 1
|
||||
assert ray_samples.positions.shape[-2] == num_samples
|
||||
|
||||
# TODO Tancik: Add more precise tests
|
||||
|
||||
|
||||
Reference in new issue
Block a user