misc proposal network modifications (#578)

* misc improvements

* cleanup

* remove distortion

* dist loss

* only dist changed now

* changes

* remove comment
This commit is contained in:
Ethan Weber authored and GitHub committed 2022-09-29 16:31:22 -07:00
1 parent 5bc011e5fa
commit 81fa5f9ba5
7 files changed
+100 -28

No files matched your search

+2 -4
View File
@@ -24,7 +24,6 @@ def distortion_loss(
ray_samples: RaySamples,
densities: TensorType["bs":..., "num_samples", 1] = None,
weights: TensorType["bs":..., "num_samples", 1] = None,
scale_factor: float = 1.0,
) -> TensorType["bs":..., 1]:
"""Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss.
@@ -40,7 +39,6 @@ def distortion_loss(
ray_samples: Ray samples to compute loss over
densities: Predicted sample densities
weights: Predicted weights from densities and sample locations
scale_factor: Scale the starts and ends by this factor
"""
if torch.is_tensor(densities):
assert not torch.is_tensor(weights), "Cannot use both densities and weights"
@@ -49,8 +47,8 @@ def distortion_loss(
if torch.is_tensor(weights):
assert not torch.is_tensor(densities), "Cannot use both densities and weights"
starts = ray_samples.frustums.starts * scale_factor
ends = ray_samples.frustums.ends * scale_factor
starts = ray_samples.spacing_starts
ends = ray_samples.spacing_ends
midpoints = (starts + ends) / 2.0 # (..., num_samples, 1)
+20 -4
View File
@@ -82,11 +82,13 @@ class SpacedSampler(Sampler):
spacing_fn_inv: Callable,
num_samples: Optional[int] = None,
train_stratified=True,
single_jitter=False,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
self.train_stratified = train_stratified
self.single_jitter = single_jitter
self.spacing_fn = spacing_fn
self.spacing_fn_inv = spacing_fn_inv
@@ -117,7 +119,10 @@ class SpacedSampler(Sampler):
# TODO More complicated than it needs to be.
if self.train_stratified and self.training:
t_rand = torch.rand((num_rays, num_samples + 1), dtype=bins.dtype, device=bins.device)
if self.single_jitter:
t_rand = torch.rand((num_rays, 1), dtype=bins.dtype, device=bins.device)
else:
t_rand = torch.rand((num_rays, num_samples + 1), dtype=bins.dtype, device=bins.device)
bin_centers = (bins[..., 1:] + bins[..., :-1]) / 2.0
bin_upper = torch.cat([bin_centers, bins[..., -1:]], -1)
bin_lower = torch.cat([bins[..., :1], bin_centers], -1)
@@ -152,6 +157,7 @@ class UniformSampler(SpacedSampler):
self,
num_samples: Optional[int] = None,
train_stratified=True,
single_jitter=False,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
@@ -160,6 +166,7 @@ class UniformSampler(SpacedSampler):
spacing_fn=lambda x: x,
spacing_fn_inv=lambda x: x,
train_stratified=train_stratified,
single_jitter=single_jitter,
density_field=density_field,
weight_threshold=weight_threshold,
)
@@ -179,6 +186,7 @@ class LinearDisparitySampler(SpacedSampler):
self,
num_samples: Optional[int] = None,
train_stratified=True,
single_jitter=False,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
@@ -187,6 +195,7 @@ class LinearDisparitySampler(SpacedSampler):
spacing_fn=lambda x: 1 / x,
spacing_fn_inv=lambda x: 1 / x,
train_stratified=train_stratified,
single_jitter=single_jitter,
density_field=density_field,
weight_threshold=weight_threshold,
)
@@ -206,6 +215,7 @@ class SqrtSampler(SpacedSampler):
self,
num_samples: Optional[int] = None,
train_stratified=True,
single_jitter=False,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
@@ -214,6 +224,7 @@ class SqrtSampler(SpacedSampler):
spacing_fn=torch.sqrt,
spacing_fn_inv=lambda x: x**2,
train_stratified=train_stratified,
single_jitter=single_jitter,
density_field=density_field,
weight_threshold=weight_threshold,
)
@@ -233,6 +244,7 @@ class LogSampler(SpacedSampler):
self,
num_samples: Optional[int] = None,
train_stratified=True,
single_jitter=False,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
@@ -241,6 +253,7 @@ class LogSampler(SpacedSampler):
spacing_fn=torch.log,
spacing_fn_inv=torch.exp,
train_stratified=train_stratified,
single_jitter=single_jitter,
density_field=density_field,
weight_threshold=weight_threshold,
)
@@ -262,6 +275,7 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
self,
num_samples: Optional[int] = None,
train_stratified=True,
single_jitter=False,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
@@ -270,6 +284,7 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
spacing_fn=lambda x: torch.where(x < 1, x / 2, 1 - 1 / (2 * x)),
spacing_fn_inv=lambda x: torch.where(x < 0.5, 2 * x, 1 / (2 - 2 * x)),
train_stratified=train_stratified,
single_jitter=single_jitter,
density_field=density_field,
weight_threshold=weight_threshold,
)
@@ -531,9 +546,10 @@ class ProposalNetworkSampler(Sampler):
def __init__(
self,
num_proposal_samples_per_ray: int = 64,
num_proposal_samples_per_ray: Tuple[int] = (64,),
num_nerf_samples_per_ray: int = 32,
num_proposal_network_iterations: int = 2,
single_jitter: bool = True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
@@ -543,7 +559,7 @@ class ProposalNetworkSampler(Sampler):
self.num_proposal_network_iterations = num_proposal_network_iterations
# samplers
self.initial_sampler = UniformLinDispPiecewiseSampler()
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
self.pdf_sampler = PDFSampler(include_original=False)
self._anneal = 1.0
@@ -566,7 +582,7 @@ class ProposalNetworkSampler(Sampler):
n = self.num_proposal_network_iterations
for i_level in range(n + 1):
is_prop = i_level < n
num_samples = self.num_proposal_samples_per_ray if is_prop else self.num_nerf_samples_per_ray
num_samples = self.num_proposal_samples_per_ray[i_level] if is_prop else self.num_nerf_samples_per_ray
if i_level == 0:
# Uniform sampling because we need to start with some samples
ray_samples = self.initial_sampler(ray_bundle, num_samples=num_samples)