mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 14:23:57 +08:00
NeuS Implementation (#1444)
* add initial method configs and data manager * add neus scheduler * add initial surface model and neus * add Neus ray sampler * add sdf field * move scale/shift calc to math utils * add debug and fix sdf_dataset typo * update vscode neus debug * fix dataset and get initial working version * add magic number from original repo * black fixes * clean up get weights/transmittance from alphas * clean up typos in sdf datamanager * type annotations on neus scheduler and remove unused params * clean up and annotate sdf field along w/ other fixes * more clean up * remove unnessary directions norm * resolve comments * update neus scheduler to follow refactor * fix linter errors * fix some typos * remove get_weights_from_alpha * more comment fixes * more linter error fixes * clean up and add comments * more little fixes for types/comments * more comments * rename NeuS Scheduler to DelayedScheduler * rename DelayedScheduler to CosineDecayScheduler
This commit is contained in:
1 parent
1b18b44d81
commit
ddfa463f4d
16 files changed
+1376
-53
No files matched your search
@@ -23,7 +23,7 @@ from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.utils.math import masked_reduction
|
||||
from nerfstudio.utils.math import masked_reduction, normalized_depth_scale_and_shift
|
||||
|
||||
L1Loss = nn.L1Loss
|
||||
MSELoss = nn.MSELoss
|
||||
@@ -355,7 +355,8 @@ class MiDaSMSELoss(nn.Module):
|
||||
"""
|
||||
summed_mask = torch.sum(mask, (1, 2))
|
||||
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
|
||||
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
|
||||
# multiply by 2 magic number?
|
||||
image_loss = masked_reduction(image_loss, 2 * summed_mask, self.reduction_type)
|
||||
|
||||
return image_loss
|
||||
|
||||
@@ -456,47 +457,6 @@ class ScaleAndShiftInvariantLoss(nn.Module):
|
||||
|
||||
self.__prediction_ssi = None
|
||||
|
||||
@classmethod
|
||||
def compute_scale_and_shift(
|
||||
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
):
|
||||
"""
|
||||
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
|
||||
This function computes scale/shift required to normalizes predicted depth map,
|
||||
to allow for using normalized depth maps as input from monocular depth estimation networks.
|
||||
These networks are trained such that they predict normalized depth maps.
|
||||
|
||||
Solves for scale/shift using a least squares approach with a closed form solution:
|
||||
Based on:
|
||||
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
scale and shift for depth prediction
|
||||
"""
|
||||
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
|
||||
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
|
||||
a_01 = torch.sum(mask * prediction, (1, 2))
|
||||
a_11 = torch.sum(mask, (1, 2))
|
||||
|
||||
# right hand side: b = [b_0, b_1]
|
||||
b_0 = torch.sum(mask * prediction * target, (1, 2))
|
||||
b_1 = torch.sum(mask * target, (1, 2))
|
||||
|
||||
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
|
||||
scale = torch.zeros_like(b_0)
|
||||
shift = torch.zeros_like(b_1)
|
||||
|
||||
det = a_00 * a_11 - a_01 * a_01
|
||||
valid = det.nonzero()
|
||||
|
||||
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
|
||||
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
|
||||
|
||||
return scale, shift
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
@@ -508,7 +468,7 @@ class ScaleAndShiftInvariantLoss(nn.Module):
|
||||
Returns:
|
||||
scale and shift invariant loss
|
||||
"""
|
||||
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
|
||||
scale, shift = normalized_depth_scale_and_shift(prediction, target, mask)
|
||||
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
|
||||
|
||||
total = self.__data_loss(self.__prediction_ssi, target, mask)
|
||||
|
||||
@@ -17,7 +17,7 @@ Collection of sampling strategies
|
||||
"""
|
||||
|
||||
from abc import abstractmethod
|
||||
from typing import Callable, List, Optional, Tuple
|
||||
from typing import Callable, List, Optional, Tuple, Union
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
@@ -598,3 +598,166 @@ class ProposalNetworkSampler(Sampler):
|
||||
|
||||
assert ray_samples is not None
|
||||
return ray_samples, weights_list, ray_samples_list
|
||||
|
||||
|
||||
class NeuSSampler(Sampler):
|
||||
"""NeuS sampler that uses a sdf network to generate samples with fixed variance value in each iterations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_samples: int = 64,
|
||||
num_samples_importance: int = 64,
|
||||
num_samples_outside: int = 32,
|
||||
num_upsample_steps: int = 4,
|
||||
base_variance: float = 64,
|
||||
single_jitter: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_samples = num_samples
|
||||
self.num_samples_importance = num_samples_importance
|
||||
self.num_samples_outside = num_samples_outside
|
||||
self.num_upsample_steps = num_upsample_steps
|
||||
self.base_variance = base_variance
|
||||
self.single_jitter = single_jitter
|
||||
|
||||
# samplers
|
||||
self.uniform_sampler = UniformSampler(single_jitter=single_jitter)
|
||||
self.pdf_sampler = PDFSampler(
|
||||
include_original=False,
|
||||
single_jitter=single_jitter,
|
||||
histogram_padding=1e-5,
|
||||
)
|
||||
self.outside_sampler = LinearDisparitySampler()
|
||||
|
||||
def generate_ray_samples(
|
||||
self,
|
||||
ray_bundle: Optional[RayBundle] = None,
|
||||
sdf_fn: Optional[Callable] = None,
|
||||
ray_samples: Optional[RaySamples] = None,
|
||||
) -> Union[Tuple[RaySamples, torch.Tensor], RaySamples]:
|
||||
assert ray_bundle is not None
|
||||
assert sdf_fn is not None
|
||||
|
||||
# Start with uniform sampling
|
||||
if ray_samples is None:
|
||||
ray_samples = self.uniform_sampler(ray_bundle, num_samples=self.num_samples)
|
||||
|
||||
total_iters = 0
|
||||
sorted_index = None
|
||||
new_samples = ray_samples
|
||||
|
||||
base_variance = self.base_variance
|
||||
|
||||
while total_iters < self.num_upsample_steps:
|
||||
|
||||
with torch.no_grad():
|
||||
new_sdf = sdf_fn(new_samples)
|
||||
|
||||
# merge sdf predictions
|
||||
if sorted_index is not None:
|
||||
sdf_merge = torch.cat([sdf.squeeze(-1), new_sdf.squeeze(-1)], -1)
|
||||
sdf = torch.gather(sdf_merge, 1, sorted_index).unsqueeze(-1)
|
||||
else:
|
||||
sdf = new_sdf
|
||||
|
||||
# compute with fix variances
|
||||
alphas = self.rendering_sdf_with_fixed_inv_s(
|
||||
ray_samples, sdf.reshape(ray_samples.shape), inv_s=base_variance * 2**total_iters
|
||||
)
|
||||
|
||||
weights = ray_samples.get_weights_and_transmittance_from_alphas(alphas[..., None], weights_only=True)
|
||||
weights = torch.cat((weights, torch.zeros_like(weights[:, :1])), dim=1)
|
||||
|
||||
new_samples = self.pdf_sampler(
|
||||
ray_bundle,
|
||||
ray_samples,
|
||||
weights,
|
||||
num_samples=self.num_samples_importance // self.num_upsample_steps,
|
||||
)
|
||||
|
||||
ray_samples, sorted_index = self.merge_ray_samples(ray_bundle, ray_samples, new_samples)
|
||||
|
||||
total_iters += 1
|
||||
|
||||
return ray_samples
|
||||
|
||||
@staticmethod
|
||||
def rendering_sdf_with_fixed_inv_s(
|
||||
ray_samples: RaySamples, sdf: TensorType["num_samples", -1], inv_s: int
|
||||
) -> TensorType["num_samples", -1]:
|
||||
"""
|
||||
rendering given a fixed inv_s as NeuS
|
||||
|
||||
Args:
|
||||
ray_samples: samples along ray
|
||||
sdf: sdf values along ray
|
||||
inv_s: fixed variance value
|
||||
Returns:
|
||||
alpha value
|
||||
"""
|
||||
batch_size = ray_samples.shape[0]
|
||||
prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:]
|
||||
deltas = ray_samples.deltas[:, :-1, 0]
|
||||
mid_sdf = (prev_sdf + next_sdf) * 0.5
|
||||
cos_val = (next_sdf - prev_sdf) / (deltas + 1e-5)
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------
|
||||
# Use min value of [ cos, prev_cos ]
|
||||
# Though it makes the sampling (not rendering) a little bit biased, this strategy can make the sampling more
|
||||
# robust when meeting situations like below:
|
||||
#
|
||||
# SDF
|
||||
# ^
|
||||
# |\ -----x----...
|
||||
# | \ /
|
||||
# | x x
|
||||
# |---\----/-------------> 0 level
|
||||
# | \ /
|
||||
# | \/
|
||||
# |
|
||||
# ----------------------------------------------------------------------------------------------------------
|
||||
prev_cos_val = torch.cat([torch.zeros([batch_size, 1], device=sdf.device), cos_val[:, :-1]], dim=-1)
|
||||
cos_val = torch.stack([prev_cos_val, cos_val], dim=-1)
|
||||
cos_val, _ = torch.min(cos_val, dim=-1, keepdim=False)
|
||||
cos_val = cos_val.clip(-1e3, 0.0)
|
||||
|
||||
dist = deltas
|
||||
prev_esti_sdf = mid_sdf - cos_val * dist * 0.5
|
||||
next_esti_sdf = mid_sdf + cos_val * dist * 0.5
|
||||
prev_cdf = torch.sigmoid(prev_esti_sdf * inv_s)
|
||||
next_cdf = torch.sigmoid(next_esti_sdf * inv_s)
|
||||
alpha = (prev_cdf - next_cdf + 1e-5) / (prev_cdf + 1e-5)
|
||||
|
||||
return alpha
|
||||
|
||||
@staticmethod
|
||||
def merge_ray_samples(ray_bundle: RayBundle, ray_samples_1: RaySamples, ray_samples_2: RaySamples):
|
||||
"""Merge two set of ray samples and return sorted index which can be used to merge sdf values
|
||||
Args:
|
||||
ray_samples_1 : ray_samples to merge
|
||||
ray_samples_2 : ray_samples to merge
|
||||
"""
|
||||
|
||||
starts_1 = ray_samples_1.spacing_starts[..., 0]
|
||||
starts_2 = ray_samples_2.spacing_starts[..., 0]
|
||||
|
||||
ends = torch.maximum(ray_samples_1.spacing_ends[..., -1:, 0], ray_samples_2.spacing_ends[..., -1:, 0])
|
||||
|
||||
bins, sorted_index = torch.sort(torch.cat([starts_1, starts_2], -1), -1)
|
||||
|
||||
bins = torch.cat([bins, ends], dim=-1)
|
||||
|
||||
# Stop gradients
|
||||
bins = bins.detach()
|
||||
|
||||
euclidean_bins = ray_samples_1.spacing_to_euclidean_fn(bins)
|
||||
|
||||
ray_samples = ray_bundle.get_ray_samples(
|
||||
bin_starts=euclidean_bins[..., :-1, None],
|
||||
bin_ends=euclidean_bins[..., 1:, None],
|
||||
spacing_starts=bins[..., :-1, None],
|
||||
spacing_ends=bins[..., 1:, None],
|
||||
spacing_to_euclidean_fn=ray_samples_1.spacing_to_euclidean_fn,
|
||||
)
|
||||
|
||||
return ray_samples, sorted_index
|
||||
Reference in new issue
Block a user