From 1267045aed99ee09851223ae795fda74150cc9b3 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Tue, 27 Sep 2022 12:32:04 -0700 Subject: [PATCH] annealing for proposal network (#560) * merged * adding depth * added annealing * reverted launch.json * reverted hidden dim * changed loss * losses match master * comments * fix black --- .gitignore | 1 + nerfactory/engine/trainer.py | 2 +- nerfactory/fields/density_field.py | 50 +++++++++++++++-------- nerfactory/models/modules/ray_losses.py | 2 +- nerfactory/models/modules/ray_sampler.py | 11 ++++- nerfactory/models/proposal.py | 52 ++++++++++++++++++++++-- nerfactory/optimizers/loss.py | 12 +++--- nerfactory/utils/callbacks.py | 4 ++ 8 files changed, 104 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index f3126029d..27eb63dab 100644 --- a/.gitignore +++ b/.gitignore @@ -187,3 +187,4 @@ scripts/colmap2nerf.py bash/ nerfactory/ngp_pl/ cache/ +renders/ diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py index 149f00966..e133ac98d 100644 --- a/nerfactory/engine/trainer.py +++ b/nerfactory/engine/trainer.py @@ -130,12 +130,12 @@ class Trainer: self._load_checkpoint() - # TODO(ethan): do this for pipeline, not pipeline.model self.callbacks = self.pipeline.get_training_callbacks( TrainingCallbackAttributes( optimizers=self.optimizers, # type: ignore grad_scaler=self.grad_scaler, # type: ignore pipeline=self.pipeline, # type: ignore + config=self.config.trainer, # type: ignore ) ) diff --git a/nerfactory/fields/density_field.py b/nerfactory/fields/density_field.py index fb8c805d9..32902bae2 100644 --- a/nerfactory/fields/density_field.py +++ b/nerfactory/fields/density_field.py @@ -20,6 +20,7 @@ Proposal network field. from typing import Optional import numpy as np +import torch from torch.nn.parameter import Parameter from torchtyping import TensorType @@ -44,15 +45,22 @@ class DensityField(Field): num_layers: number of hidden layers hidden_dim: dimension of hidden layers spatial_distortion: spatial distortion module + use_linear: whether to skip the MLP and use a single linear layer instead """ def __init__( - self, aabb, num_layers: int = 2, hidden_dim: int = 64, spatial_distortion: Optional[SpatialDistortion] = None + self, + aabb, + num_layers: int = 2, + hidden_dim: int = 64, + spatial_distortion: Optional[SpatialDistortion] = None, + use_linear=False, ) -> None: super().__init__() self.aabb = Parameter(aabb, requires_grad=False) self.spatial_distortion = spatial_distortion + self.use_linear = use_linear num_levels = 8 max_res = 1024 @@ -61,18 +69,8 @@ class DensityField(Field): features_per_level = 2 growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) - self.direction_encoding = tcnn.Encoding( - n_input_dims=3, - encoding_config={ - "otype": "SphericalHarmonics", - "degree": 4, - }, - ) - - self.mlp_base = tcnn.NetworkWithInputEncoding( - n_input_dims=3, - n_output_dims=1, - encoding_config={ + config = { + "encoding": { "otype": "HashGrid", "n_levels": num_levels, "n_features_per_level": features_per_level, @@ -80,14 +78,25 @@ class DensityField(Field): "base_resolution": base_res, "per_level_scale": growth_factor, }, - network_config={ + "network": { "otype": "FullyFusedMLP", "activation": "ReLU", "output_activation": "None", "n_neurons": hidden_dim, "n_hidden_layers": num_layers - 1, }, - ) + } + + if not self.use_linear: + self.mlp_base = tcnn.NetworkWithInputEncoding( + n_input_dims=3, + n_output_dims=1, + encoding_config=config["encoding"], + network_config=config["network"], + ) + else: + self.encoding = tcnn.Encoding(n_input_dims=3, encoding_config=config["encoding"]) + self.linear = torch.nn.Linear(self.encoding.n_output_dims, 1) def get_density(self, ray_samples: RaySamples): if self.spatial_distortion is not None: @@ -101,12 +110,19 @@ class DensityField(Field): # assert torch.all(positions >= 0.0) and torch.all( # positions <= 1.0 # ), f"positions: {positions.min()} {positions.max()}" - density_before_activation = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) + + if not self.use_linear: + density_before_activation = ( + self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1).to(positions) + ) + else: + x = self.encoding(positions_flat).to(positions) + density_before_activation = self.linear(x).view(*ray_samples.frustums.shape, -1) # Rectifying the density with an exponential is much more stable than a ReLU or # softplus, because it enables high post-activation (float32) density outputs # from smaller internal (float16) parameters. - density = trunc_exp(density_before_activation.to(positions)) + density = trunc_exp(density_before_activation) return density, None def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None): diff --git a/nerfactory/models/modules/ray_losses.py b/nerfactory/models/modules/ray_losses.py index f74937093..414f1cff7 100644 --- a/nerfactory/models/modules/ray_losses.py +++ b/nerfactory/models/modules/ray_losses.py @@ -26,7 +26,7 @@ def distortion_loss( weights: TensorType["bs":..., "num_samples", 1] = None, scale_factor: float = 1.0, ) -> TensorType["bs":..., 1]: - """Ray baserd distortion loss proposed in MipNeRF-360. Returns distortion Loss. + """Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss. .. math:: diff --git a/nerfactory/models/modules/ray_sampler.py b/nerfactory/models/modules/ray_sampler.py index d3cadbf7c..29a717e4b 100644 --- a/nerfactory/models/modules/ray_sampler.py +++ b/nerfactory/models/modules/ray_sampler.py @@ -551,9 +551,16 @@ class ProposalNetworkSampler(Sampler): self.num_nerf_samples_per_ray = num_nerf_samples_per_ray self.num_proposal_network_iterations = num_proposal_network_iterations + # samplers self.initial_sampler = UniformLinDispPiecewiseSampler() self.pdf_sampler = PDFSampler(include_original=False) + self._anneal = 1.0 + + def set_anneal(self, anneal: float) -> None: + """Set the anneal value for the proposal network.""" + self._anneal = anneal + def generate_ray_samples( self, ray_bundle: Optional[RayBundle] = None, @@ -572,7 +579,9 @@ class ProposalNetworkSampler(Sampler): ray_samples = self.initial_sampler(ray_bundle, num_samples=num_samples) else: # PDF sampling based on the last samples and their weights - ray_samples = self.pdf_sampler(ray_bundle, ray_samples, weights, num_samples=num_samples) + # Perform annealing to the weights. This will be a no-op if self._anneal is 1.0. + annealed_weights = torch.pow(weights, self._anneal) + ray_samples = self.pdf_sampler(ray_bundle, ray_samples, annealed_weights, num_samples=num_samples) if is_prop: density = density_fn(ray_samples.frustums.get_positions()) weights = ray_samples.get_weights(density) diff --git a/nerfactory/models/proposal.py b/nerfactory/models/proposal.py index ca46760b5..2d7109054 100644 --- a/nerfactory/models/proposal.py +++ b/nerfactory/models/proposal.py @@ -21,6 +21,7 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Dict, List, Tuple, Type +import numpy as np import torch from torch.nn import Parameter from torchmetrics import PeakSignalNoiseRatio @@ -44,6 +45,11 @@ from nerfactory.renderers.renderers import ( RGBRenderer, ) from nerfactory.utils import colors, visualization +from nerfactory.utils.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) @dataclass @@ -69,6 +75,12 @@ class ProposalModelConfig(ModelConfig): """Distortion loss multiplier.""" use_appearance_conditioning: bool = True """Whether to use appearance conditioning.""" + use_proposal_weight_anneal: bool = True + """Whether to use proposal weight annealing.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 1000 + """Max num iterations for the annealing function.""" class ProposalModel(Model): @@ -124,6 +136,30 @@ class ProposalModel(Model): param_groups["fields"] = list(self.proposal_network.parameters()) + list(self.field.parameters()) return param_groups + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + callbacks = [] + if self.config.use_proposal_weight_anneal: + # anneal the weights of the proposal network before doing PDF sampling + N = self.config.proposal_weights_anneal_max_num_iters + + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / N, 0, 1) + bias = lambda x, b: (b * x) / ((b - 1) * x + 1) + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ) + ) + return callbacks + def get_outputs(self, ray_bundle: RayBundle): ray_samples, weights_list, ray_samples_list = self.proposal_sampler( ray_bundle, density_fn=self.proposal_network.density_fn @@ -140,6 +176,9 @@ class ProposalModel(Model): outputs = {"rgb": rgb, "accumulation": accumulation, "depth": depth} outputs["weights_list"] = weights_list outputs["ray_samples_list"] = ray_samples_list + + for i in range(self.config.num_proposal_network_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) return outputs def get_metrics_dict(self, outputs, batch): @@ -153,10 +192,7 @@ class ProposalModel(Model): image = batch["image"].to(self.device) loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"]) loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( - outputs["weights_list"], - outputs["ray_samples_list"], - self.config.near_plane, - self.config.far_plane, + outputs["weights_list"], outputs["ray_samples_list"] ) loss_dict["distortion_loss"] = self.config.distortion_loss_mult * torch.mean( distortion_loss( @@ -197,4 +233,12 @@ class ProposalModel(Model): images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + for i in range(self.config.num_proposal_network_iterations): + key = f"prop_depth_{i}" + prop_depth_i = visualization.apply_depth_colormap( + outputs[key], + accumulation=outputs["accumulation"], + ) + images_dict[key] = prop_depth_i + return metrics_dict, images_dict diff --git a/nerfactory/optimizers/loss.py b/nerfactory/optimizers/loss.py index 57b9a7633..8065662d8 100644 --- a/nerfactory/optimizers/loss.py +++ b/nerfactory/optimizers/loss.py @@ -70,25 +70,25 @@ def lossfun_outer(t, w, t_env, w_env): return torch.clip(w - w_outer, min=0) ** 2 / (w + EPS) -def ray_samples_to_sdist(ray_samples, near_plane, far_plane): +def ray_samples_to_sdist(ray_samples): """Convert ray samples to s space""" - starts = (ray_samples.frustums.starts - near_plane) / (far_plane - near_plane) - ends = (ray_samples.frustums.ends - near_plane) / (far_plane - near_plane) + starts = ray_samples.spacing_starts + ends = ray_samples.spacing_ends sdist = torch.cat([starts[..., 0], ends[..., -1:, 0]], dim=-1) # (num_rays, num_samples + 1) return sdist -def interlevel_loss(weights_list, ray_samples_list, near_plane, far_plane): +def interlevel_loss(weights_list, ray_samples_list): """Calculates the proposal loss in the MipNeRF-360 paper. https://github.com/kakaobrain/NeRF-Factory/blob/f61bb8744a5cb4820a4d968fb3bfbed777550f4a/src/model/mipnerf360/model.py#L515 https://github.com/google-research/multinerf/blob/b02228160d3179300c7d499dca28cb9ca3677f32/internal/train_utils.py#L133 """ - c = ray_samples_to_sdist(ray_samples_list[-1], near_plane, far_plane).detach() + c = ray_samples_to_sdist(ray_samples_list[-1]).detach() w = weights_list[-1][..., 0].detach() loss_interlevel = 0.0 for ray_samples, weights in zip(ray_samples_list[:-1], weights_list[:-1]): - sdist = ray_samples_to_sdist(ray_samples, near_plane, far_plane) + sdist = ray_samples_to_sdist(ray_samples) cp = sdist # (num_rays, num_samples + 1) wp = weights[..., 0] # (num_rays, num_samples) loss_interlevel += torch.mean(lossfun_outer(c, w, cp, wp)) diff --git a/nerfactory/utils/callbacks.py b/nerfactory/utils/callbacks.py index 165a7ed47..2604c35d5 100644 --- a/nerfactory/utils/callbacks.py +++ b/nerfactory/utils/callbacks.py @@ -21,6 +21,8 @@ from enum import Enum, auto from inspect import signature from typing import Callable, Dict, List, Optional, Tuple +from nerfactory.configs.base import TrainerConfig + @dataclass class TrainingCallbackAttributes: @@ -36,6 +38,8 @@ class TrainingCallbackAttributes: """gradient scalers""" pipeline: Optional[InitVar] """reference to training pipeline""" + config: TrainerConfig + """the trainer config""" class TrainingCallbackLocation(Enum):