NeuS Implementation (#1444)

* add initial method configs and data manager

* add neus scheduler

* add initial surface model and neus

* add Neus ray sampler

* add sdf field

* move scale/shift calc to math utils

* add debug and fix sdf_dataset typo

* update vscode neus debug

* fix dataset and get initial working version

* add magic number from original repo

* black fixes

* clean up get weights/transmittance from alphas

* clean up typos in sdf datamanager

* type annotations on neus scheduler and remove unused params

* clean up and annotate sdf field along w/ other fixes

* more clean up

* remove unnessary directions norm

* resolve comments

* update neus scheduler to follow refactor

* fix linter errors

* fix some typos

* remove get_weights_from_alpha

* more comment fixes

* more linter error fixes

* clean up and add comments

* more little fixes for types/comments

* more comments

* rename NeuS Scheduler to DelayedScheduler

* rename DelayedScheduler to CosineDecayScheduler
This commit is contained in:
Pablo Vela authored and GitHub committed 2023-03-12 16:01:18 -04:00
1 parent 1b18b44d81
commit ddfa463f4d
16 files changed
+1376 -53

No files matched your search

+4 -44
View File
@@ -23,7 +23,7 @@ from torchtyping import TensorType
from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils.math import masked_reduction
from nerfstudio.utils.math import masked_reduction, normalized_depth_scale_and_shift
L1Loss = nn.L1Loss
MSELoss = nn.MSELoss
@@ -355,7 +355,8 @@ class MiDaSMSELoss(nn.Module):
"""
summed_mask = torch.sum(mask, (1, 2))
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
# multiply by 2 magic number?
image_loss = masked_reduction(image_loss, 2 * summed_mask, self.reduction_type)
return image_loss
@@ -456,47 +457,6 @@ class ScaleAndShiftInvariantLoss(nn.Module):
self.__prediction_ssi = None
@classmethod
def compute_scale_and_shift(
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
):
"""
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
This function computes scale/shift required to normalizes predicted depth map,
to allow for using normalized depth maps as input from monocular depth estimation networks.
These networks are trained such that they predict normalized depth maps.
Solves for scale/shift using a least squares approach with a closed form solution:
Based on:
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
scale and shift for depth prediction
"""
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
a_01 = torch.sum(mask * prediction, (1, 2))
a_11 = torch.sum(mask, (1, 2))
# right hand side: b = [b_0, b_1]
b_0 = torch.sum(mask * prediction * target, (1, 2))
b_1 = torch.sum(mask * target, (1, 2))
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
scale = torch.zeros_like(b_0)
shift = torch.zeros_like(b_1)
det = a_00 * a_11 - a_01 * a_01
valid = det.nonzero()
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
return scale, shift
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
@@ -508,7 +468,7 @@ class ScaleAndShiftInvariantLoss(nn.Module):
Returns:
scale and shift invariant loss
"""
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
scale, shift = normalized_depth_scale_and_shift(prediction, target, mask)
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
total = self.__data_loss(self.__prediction_ssi, target, mask)
+164 -1
View File
@@ -17,7 +17,7 @@ Collection of sampling strategies
"""
from abc import abstractmethod
from typing import Callable, List, Optional, Tuple
from typing import Callable, List, Optional, Tuple, Union
import nerfacc
import torch
@@ -598,3 +598,166 @@ class ProposalNetworkSampler(Sampler):
assert ray_samples is not None
return ray_samples, weights_list, ray_samples_list
class NeuSSampler(Sampler):
"""NeuS sampler that uses a sdf network to generate samples with fixed variance value in each iterations."""
def __init__(
self,
num_samples: int = 64,
num_samples_importance: int = 64,
num_samples_outside: int = 32,
num_upsample_steps: int = 4,
base_variance: float = 64,
single_jitter: bool = True,
) -> None:
super().__init__()
self.num_samples = num_samples
self.num_samples_importance = num_samples_importance
self.num_samples_outside = num_samples_outside
self.num_upsample_steps = num_upsample_steps
self.base_variance = base_variance
self.single_jitter = single_jitter
# samplers
self.uniform_sampler = UniformSampler(single_jitter=single_jitter)
self.pdf_sampler = PDFSampler(
include_original=False,
single_jitter=single_jitter,
histogram_padding=1e-5,
)
self.outside_sampler = LinearDisparitySampler()
def generate_ray_samples(
self,
ray_bundle: Optional[RayBundle] = None,
sdf_fn: Optional[Callable] = None,
ray_samples: Optional[RaySamples] = None,
) -> Union[Tuple[RaySamples, torch.Tensor], RaySamples]:
assert ray_bundle is not None
assert sdf_fn is not None
# Start with uniform sampling
if ray_samples is None:
ray_samples = self.uniform_sampler(ray_bundle, num_samples=self.num_samples)
total_iters = 0
sorted_index = None
new_samples = ray_samples
base_variance = self.base_variance
while total_iters < self.num_upsample_steps:
with torch.no_grad():
new_sdf = sdf_fn(new_samples)
# merge sdf predictions
if sorted_index is not None:
sdf_merge = torch.cat([sdf.squeeze(-1), new_sdf.squeeze(-1)], -1)
sdf = torch.gather(sdf_merge, 1, sorted_index).unsqueeze(-1)
else:
sdf = new_sdf
# compute with fix variances
alphas = self.rendering_sdf_with_fixed_inv_s(
ray_samples, sdf.reshape(ray_samples.shape), inv_s=base_variance * 2**total_iters
)
weights = ray_samples.get_weights_and_transmittance_from_alphas(alphas[..., None], weights_only=True)
weights = torch.cat((weights, torch.zeros_like(weights[:, :1])), dim=1)
new_samples = self.pdf_sampler(
ray_bundle,
ray_samples,
weights,
num_samples=self.num_samples_importance // self.num_upsample_steps,
)
ray_samples, sorted_index = self.merge_ray_samples(ray_bundle, ray_samples, new_samples)
total_iters += 1
return ray_samples
@staticmethod
def rendering_sdf_with_fixed_inv_s(
ray_samples: RaySamples, sdf: TensorType["num_samples", -1], inv_s: int
) -> TensorType["num_samples", -1]:
"""
rendering given a fixed inv_s as NeuS
Args:
ray_samples: samples along ray
sdf: sdf values along ray
inv_s: fixed variance value
Returns:
alpha value
"""
batch_size = ray_samples.shape[0]
prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:]
deltas = ray_samples.deltas[:, :-1, 0]
mid_sdf = (prev_sdf + next_sdf) * 0.5
cos_val = (next_sdf - prev_sdf) / (deltas + 1e-5)
# ----------------------------------------------------------------------------------------------------------
# Use min value of [ cos, prev_cos ]
# Though it makes the sampling (not rendering) a little bit biased, this strategy can make the sampling more
# robust when meeting situations like below:
#
# SDF
# ^
# |\ -----x----...
# | \ /
# | x x
# |---\----/-------------> 0 level
# | \ /
# | \/
# |
# ----------------------------------------------------------------------------------------------------------
prev_cos_val = torch.cat([torch.zeros([batch_size, 1], device=sdf.device), cos_val[:, :-1]], dim=-1)
cos_val = torch.stack([prev_cos_val, cos_val], dim=-1)
cos_val, _ = torch.min(cos_val, dim=-1, keepdim=False)
cos_val = cos_val.clip(-1e3, 0.0)
dist = deltas
prev_esti_sdf = mid_sdf - cos_val * dist * 0.5
next_esti_sdf = mid_sdf + cos_val * dist * 0.5
prev_cdf = torch.sigmoid(prev_esti_sdf * inv_s)
next_cdf = torch.sigmoid(next_esti_sdf * inv_s)
alpha = (prev_cdf - next_cdf + 1e-5) / (prev_cdf + 1e-5)
return alpha
@staticmethod
def merge_ray_samples(ray_bundle: RayBundle, ray_samples_1: RaySamples, ray_samples_2: RaySamples):
"""Merge two set of ray samples and return sorted index which can be used to merge sdf values
Args:
ray_samples_1 : ray_samples to merge
ray_samples_2 : ray_samples to merge
"""
starts_1 = ray_samples_1.spacing_starts[..., 0]
starts_2 = ray_samples_2.spacing_starts[..., 0]
ends = torch.maximum(ray_samples_1.spacing_ends[..., -1:, 0], ray_samples_2.spacing_ends[..., -1:, 0])
bins, sorted_index = torch.sort(torch.cat([starts_1, starts_2], -1), -1)
bins = torch.cat([bins, ends], dim=-1)
# Stop gradients
bins = bins.detach()
euclidean_bins = ray_samples_1.spacing_to_euclidean_fn(bins)
ray_samples = ray_bundle.get_ray_samples(
bin_starts=euclidean_bins[..., :-1, None],
bin_ends=euclidean_bins[..., 1:, None],
spacing_starts=bins[..., :-1, None],
spacing_ends=bins[..., 1:, None],
spacing_to_euclidean_fn=ray_samples_1.spacing_to_euclidean_fn,
)
return ray_samples, sorted_index