misc proposal network modifications (#578)

* misc improvements

* cleanup

* remove distortion

* dist loss

* only dist changed now

* changes

* remove comment
This commit is contained in:
Ethan Weber authored and GitHub committed 2022-09-29 16:31:22 -07:00
1 parent 5bc011e5fa
commit 81fa5f9ba5
7 files changed
+100 -28

No files matched your search

+14 -12
View File
@@ -34,10 +34,9 @@ from nerfactory.fields.density_field import DensityField
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.fields.modules.spatial_distortions import SceneContraction
from nerfactory.models.base import Model, ModelConfig
from nerfactory.models.modules.ray_losses import distortion_loss
from nerfactory.models.modules.ray_sampler import ProposalNetworkSampler
from nerfactory.models.modules.scene_colliders import NearFarCollider
from nerfactory.optimizers.loss import MSELoss, interlevel_loss
from nerfactory.optimizers.loss import MSELoss, distortion_loss, interlevel_loss
from nerfactory.renderers.renderers import (
AccumulationRenderer,
DepthRenderer,
@@ -62,7 +61,7 @@ class ProposalModelConfig(ModelConfig):
"""How far along the ray to stop sampling."""
randomize_background: bool = True
"""Whether to randomize the background color."""
num_proposal_samples_per_ray: int = 64
num_proposal_samples_per_ray: Tuple[int] = (64,)
"""Number of samples per ray for the proposal network."""
num_nerf_samples_per_ray: int = 64
"""Number of samples per ray for the nerf network."""
@@ -72,16 +71,20 @@ class ProposalModelConfig(ModelConfig):
"""Use the same proposal network. Otherwise use different ones."""
interlevel_loss_mult: float = 1.0
"""Proposal loss multiplier."""
distortion_loss_mult: float = 0.01
distortion_loss_mult: float = 0.001
"""Distortion loss multiplier."""
use_appearance_conditioning: bool = True
"""Whether to use appearance conditioning."""
use_proposal_weight_anneal: bool = True
"""Whether to use proposal weight annealing."""
use_average_appearance_embedding: bool = True
"""Whether to use average appearance embedding or zeros for inference."""
proposal_weights_anneal_slope: float = 10.0
"""Slope of the annealing function for the proposal weights."""
proposal_weights_anneal_max_num_iters: int = 1000
"""Max num iterations for the annealing function."""
use_single_jitter: bool = False
"""Whether use single jitter or not for first proposal network."""
class ProposalModel(Model):
@@ -102,7 +105,10 @@ class ProposalModel(Model):
# Fields
if self.config.use_appearance_conditioning:
self.field = TCNNCompoundField(
self.scene_bounds.aabb, spatial_distortion=scene_contraction, num_images=self.num_train_data
self.scene_bounds.aabb,
spatial_distortion=scene_contraction,
num_images=self.num_train_data,
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
)
else:
raise NotImplementedError("Only appearance conditioning is supported.")
@@ -127,6 +133,7 @@ class ProposalModel(Model):
num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray,
num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
num_proposal_network_iterations=self.config.num_proposal_network_iterations,
single_jitter=self.config.use_single_jitter,
)
# renderers
@@ -197,6 +204,7 @@ class ProposalModel(Model):
metrics_dict = {}
image = batch["image"].to(self.device)
metrics_dict["psnr"] = self.psnr(outputs["rgb"], image)
metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"])
return metrics_dict
def get_loss_dict(self, outputs, batch, metrics_dict=None):
@@ -206,13 +214,7 @@ class ProposalModel(Model):
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * torch.mean(
distortion_loss(
outputs["ray_samples_list"][-1],
weights=outputs["weights_list"][-1],
scale_factor=1.0 / (self.config.far_plane - self.config.near_plane),
)
)
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"]
return loss_dict
def get_image_metrics_and_images(