mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 06:35:36 +08:00
misc proposal network modifications (#578)
* misc improvements * cleanup * remove distortion * dist loss * only dist changed now * changes * remove comment
This commit is contained in:
1 parent
5bc011e5fa
commit
81fa5f9ba5
7 files changed
+100
-28
No files matched your search
@@ -24,7 +24,6 @@ def distortion_loss(
|
||||
ray_samples: RaySamples,
|
||||
densities: TensorType["bs":..., "num_samples", 1] = None,
|
||||
weights: TensorType["bs":..., "num_samples", 1] = None,
|
||||
scale_factor: float = 1.0,
|
||||
) -> TensorType["bs":..., 1]:
|
||||
"""Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss.
|
||||
|
||||
@@ -40,7 +39,6 @@ def distortion_loss(
|
||||
ray_samples: Ray samples to compute loss over
|
||||
densities: Predicted sample densities
|
||||
weights: Predicted weights from densities and sample locations
|
||||
scale_factor: Scale the starts and ends by this factor
|
||||
"""
|
||||
if torch.is_tensor(densities):
|
||||
assert not torch.is_tensor(weights), "Cannot use both densities and weights"
|
||||
@@ -49,8 +47,8 @@ def distortion_loss(
|
||||
if torch.is_tensor(weights):
|
||||
assert not torch.is_tensor(densities), "Cannot use both densities and weights"
|
||||
|
||||
starts = ray_samples.frustums.starts * scale_factor
|
||||
ends = ray_samples.frustums.ends * scale_factor
|
||||
starts = ray_samples.spacing_starts
|
||||
ends = ray_samples.spacing_ends
|
||||
|
||||
midpoints = (starts + ends) / 2.0 # (..., num_samples, 1)
|
||||
|
||||
|
||||
@@ -82,11 +82,13 @@ class SpacedSampler(Sampler):
|
||||
spacing_fn_inv: Callable,
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
|
||||
self.train_stratified = train_stratified
|
||||
self.single_jitter = single_jitter
|
||||
self.spacing_fn = spacing_fn
|
||||
self.spacing_fn_inv = spacing_fn_inv
|
||||
|
||||
@@ -117,7 +119,10 @@ class SpacedSampler(Sampler):
|
||||
|
||||
# TODO More complicated than it needs to be.
|
||||
if self.train_stratified and self.training:
|
||||
t_rand = torch.rand((num_rays, num_samples + 1), dtype=bins.dtype, device=bins.device)
|
||||
if self.single_jitter:
|
||||
t_rand = torch.rand((num_rays, 1), dtype=bins.dtype, device=bins.device)
|
||||
else:
|
||||
t_rand = torch.rand((num_rays, num_samples + 1), dtype=bins.dtype, device=bins.device)
|
||||
bin_centers = (bins[..., 1:] + bins[..., :-1]) / 2.0
|
||||
bin_upper = torch.cat([bin_centers, bins[..., -1:]], -1)
|
||||
bin_lower = torch.cat([bins[..., :1], bin_centers], -1)
|
||||
@@ -152,6 +157,7 @@ class UniformSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
@@ -160,6 +166,7 @@ class UniformSampler(SpacedSampler):
|
||||
spacing_fn=lambda x: x,
|
||||
spacing_fn_inv=lambda x: x,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
@@ -179,6 +186,7 @@ class LinearDisparitySampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
@@ -187,6 +195,7 @@ class LinearDisparitySampler(SpacedSampler):
|
||||
spacing_fn=lambda x: 1 / x,
|
||||
spacing_fn_inv=lambda x: 1 / x,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
@@ -206,6 +215,7 @@ class SqrtSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
@@ -214,6 +224,7 @@ class SqrtSampler(SpacedSampler):
|
||||
spacing_fn=torch.sqrt,
|
||||
spacing_fn_inv=lambda x: x**2,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
@@ -233,6 +244,7 @@ class LogSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
@@ -241,6 +253,7 @@ class LogSampler(SpacedSampler):
|
||||
spacing_fn=torch.log,
|
||||
spacing_fn_inv=torch.exp,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
@@ -262,6 +275,7 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
@@ -270,6 +284,7 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
|
||||
spacing_fn=lambda x: torch.where(x < 1, x / 2, 1 - 1 / (2 * x)),
|
||||
spacing_fn_inv=lambda x: torch.where(x < 0.5, 2 * x, 1 / (2 - 2 * x)),
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
@@ -531,9 +546,10 @@ class ProposalNetworkSampler(Sampler):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_proposal_samples_per_ray: int = 64,
|
||||
num_proposal_samples_per_ray: Tuple[int] = (64,),
|
||||
num_nerf_samples_per_ray: int = 32,
|
||||
num_proposal_network_iterations: int = 2,
|
||||
single_jitter: bool = True,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
@@ -543,7 +559,7 @@ class ProposalNetworkSampler(Sampler):
|
||||
self.num_proposal_network_iterations = num_proposal_network_iterations
|
||||
|
||||
# samplers
|
||||
self.initial_sampler = UniformLinDispPiecewiseSampler()
|
||||
self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter)
|
||||
self.pdf_sampler = PDFSampler(include_original=False)
|
||||
|
||||
self._anneal = 1.0
|
||||
@@ -566,7 +582,7 @@ class ProposalNetworkSampler(Sampler):
|
||||
n = self.num_proposal_network_iterations
|
||||
for i_level in range(n + 1):
|
||||
is_prop = i_level < n
|
||||
num_samples = self.num_proposal_samples_per_ray if is_prop else self.num_nerf_samples_per_ray
|
||||
num_samples = self.num_proposal_samples_per_ray[i_level] if is_prop else self.num_nerf_samples_per_ray
|
||||
if i_level == 0:
|
||||
# Uniform sampling because we need to start with some samples
|
||||
ray_samples = self.initial_sampler(ray_bundle, num_samples=num_samples)
|
||||
|
||||
@@ -34,10 +34,9 @@ from nerfactory.fields.density_field import DensityField
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.fields.modules.spatial_distortions import SceneContraction
|
||||
from nerfactory.models.base import Model, ModelConfig
|
||||
from nerfactory.models.modules.ray_losses import distortion_loss
|
||||
from nerfactory.models.modules.ray_sampler import ProposalNetworkSampler
|
||||
from nerfactory.models.modules.scene_colliders import NearFarCollider
|
||||
from nerfactory.optimizers.loss import MSELoss, interlevel_loss
|
||||
from nerfactory.optimizers.loss import MSELoss, distortion_loss, interlevel_loss
|
||||
from nerfactory.renderers.renderers import (
|
||||
AccumulationRenderer,
|
||||
DepthRenderer,
|
||||
@@ -62,7 +61,7 @@ class ProposalModelConfig(ModelConfig):
|
||||
"""How far along the ray to stop sampling."""
|
||||
randomize_background: bool = True
|
||||
"""Whether to randomize the background color."""
|
||||
num_proposal_samples_per_ray: int = 64
|
||||
num_proposal_samples_per_ray: Tuple[int] = (64,)
|
||||
"""Number of samples per ray for the proposal network."""
|
||||
num_nerf_samples_per_ray: int = 64
|
||||
"""Number of samples per ray for the nerf network."""
|
||||
@@ -72,16 +71,20 @@ class ProposalModelConfig(ModelConfig):
|
||||
"""Use the same proposal network. Otherwise use different ones."""
|
||||
interlevel_loss_mult: float = 1.0
|
||||
"""Proposal loss multiplier."""
|
||||
distortion_loss_mult: float = 0.01
|
||||
distortion_loss_mult: float = 0.001
|
||||
"""Distortion loss multiplier."""
|
||||
use_appearance_conditioning: bool = True
|
||||
"""Whether to use appearance conditioning."""
|
||||
use_proposal_weight_anneal: bool = True
|
||||
"""Whether to use proposal weight annealing."""
|
||||
use_average_appearance_embedding: bool = True
|
||||
"""Whether to use average appearance embedding or zeros for inference."""
|
||||
proposal_weights_anneal_slope: float = 10.0
|
||||
"""Slope of the annealing function for the proposal weights."""
|
||||
proposal_weights_anneal_max_num_iters: int = 1000
|
||||
"""Max num iterations for the annealing function."""
|
||||
use_single_jitter: bool = False
|
||||
"""Whether use single jitter or not for first proposal network."""
|
||||
|
||||
|
||||
class ProposalModel(Model):
|
||||
@@ -102,7 +105,10 @@ class ProposalModel(Model):
|
||||
# Fields
|
||||
if self.config.use_appearance_conditioning:
|
||||
self.field = TCNNCompoundField(
|
||||
self.scene_bounds.aabb, spatial_distortion=scene_contraction, num_images=self.num_train_data
|
||||
self.scene_bounds.aabb,
|
||||
spatial_distortion=scene_contraction,
|
||||
num_images=self.num_train_data,
|
||||
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError("Only appearance conditioning is supported.")
|
||||
@@ -127,6 +133,7 @@ class ProposalModel(Model):
|
||||
num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray,
|
||||
num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
|
||||
num_proposal_network_iterations=self.config.num_proposal_network_iterations,
|
||||
single_jitter=self.config.use_single_jitter,
|
||||
)
|
||||
|
||||
# renderers
|
||||
@@ -197,6 +204,7 @@ class ProposalModel(Model):
|
||||
metrics_dict = {}
|
||||
image = batch["image"].to(self.device)
|
||||
metrics_dict["psnr"] = self.psnr(outputs["rgb"], image)
|
||||
metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"])
|
||||
return metrics_dict
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None):
|
||||
@@ -206,13 +214,7 @@ class ProposalModel(Model):
|
||||
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * torch.mean(
|
||||
distortion_loss(
|
||||
outputs["ray_samples_list"][-1],
|
||||
weights=outputs["weights_list"][-1],
|
||||
scale_factor=1.0 / (self.config.far_plane - self.config.near_plane),
|
||||
)
|
||||
)
|
||||
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"]
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
|
||||
Reference in new issue
Block a user