nerfplayer with ngp and nerfacto (#1277)

* nerfplayer with ngp and nerfacto

* update comments

* update cpp extension build

* avoid Windows repeated CUDA compile

---------

Co-authored-by: machenmusik <machenmusik@users.noreply.github.com>
This commit is contained in:
LSongandmachenmusik authored and GitHub committed 2023-01-31 11:56:28 -08:00
1 parent 0e9e26dc7c
commit e73692e599
10 files changed
+1238 -15

No files matched your search

+251
View File
@@ -0,0 +1,251 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
NeRFPlayer (https://arxiv.org/abs/2210.15947) implementation with nerfacto backbone.
"""
from __future__ import annotations
import functools
from dataclasses import dataclass, field
from typing import Dict, List, Type
import numpy as np
import torch
from torchmetrics import PeakSignalNoiseRatio
from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from typing_extensions import Literal
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.nerfplayer_nerfacto_field import (
NerfplayerNerfactoField,
TemporalHashMLPDensityField,
)
from nerfstudio.model_components.losses import (
MSELoss,
interlevel_loss,
orientation_loss,
pred_normal_loss,
)
from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler
from nerfstudio.model_components.renderers import (
AccumulationRenderer,
DepthRenderer,
NormalsRenderer,
RGBRenderer,
)
from nerfstudio.model_components.scene_colliders import NearFarCollider
from nerfstudio.models.base_model import Model
from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig
@dataclass
class NerfplayerNerfactoModelConfig(NerfactoModelConfig):
"""Nerfplayer Model Config with Nerfacto backbone"""
_target: Type = field(default_factory=lambda: NerfplayerNerfactoModel)
near_plane: float = 0.05
"""How far along the ray to start sampling."""
far_plane: float = 1000.0
"""How far along the ray to stop sampling."""
background_color: Literal["random", "last_sample"] = "random"
"""Whether to randomize the background color. (Random is reported to be better on DyCheck.)"""
num_levels: int = 16
"""Hashing grid parameter."""
features_per_level: int = 2
"""Hashing grid parameter."""
log2_hashmap_size: int = 18
"""Hashing grid parameter."""
temporal_dim: int = 32
"""Hashing grid parameter. A higher temporal dim means a higher temporal frequency."""
proposal_net_args_list: List[Dict] = field(
default_factory=lambda: [
{"hidden_dim": 16, "temporal_dim": 32, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 64},
{"hidden_dim": 16, "temporal_dim": 32, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
]
)
"""Arguments for the proposal density fields."""
distortion_loss_mult: float = 1e-2
"""Distortion loss multiplier."""
temporal_tv_weight: float = 1
"""Temporal TV balancing weight for feature channels."""
depth_weight: float = 1e-1
"""depth loss balancing weight for feature channels."""
class NerfplayerNerfactoModel(NerfactoModel):
"""Nerfplayer model with Nerfacto backbone.
Args:
config: Nerfplayer configuration to instantiate model
"""
config: NerfplayerNerfactoModelConfig
def populate_modules(self):
"""Set the fields and modules."""
Model.populate_modules(self)
scene_contraction = SceneContraction(order=float("inf"))
# Fields
self.field = NerfplayerNerfactoField(
self.scene_box.aabb,
temporal_dim=self.config.temporal_dim,
num_levels=self.config.num_levels,
features_per_level=self.config.features_per_level,
log2_hashmap_size=self.config.log2_hashmap_size,
spatial_distortion=scene_contraction,
num_images=self.num_train_data,
use_pred_normals=self.config.predict_normals,
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
)
self.density_fns = []
num_prop_nets = self.config.num_proposal_iterations
# Build the proposal network(s)
self.proposal_networks = torch.nn.ModuleList()
if self.config.use_same_proposal_network:
assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed."
prop_net_args = self.config.proposal_net_args_list[0]
network = TemporalHashMLPDensityField(
self.scene_box.aabb, spatial_distortion=scene_contraction, **prop_net_args
)
self.proposal_networks.append(network)
self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)])
else:
for i in range(num_prop_nets):
prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)]
network = TemporalHashMLPDensityField(
self.scene_box.aabb,
spatial_distortion=scene_contraction,
**prop_net_args,
)
self.proposal_networks.append(network)
self.density_fns.extend([network.density_fn for network in self.proposal_networks])
# Samplers
update_schedule = lambda step: np.clip(
np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]),
1,
self.config.proposal_update_every,
)
self.proposal_sampler = ProposalNetworkSampler(
num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray,
num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
num_proposal_network_iterations=self.config.num_proposal_iterations,
single_jitter=self.config.use_single_jitter,
update_sched=update_schedule,
)
# Collider
self.collider = NearFarCollider(near_plane=self.config.near_plane, far_plane=self.config.far_plane)
# renderers
self.renderer_rgb = RGBRenderer(background_color=self.config.background_color)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer(method="expected") # for depth loss
self.renderer_normals = NormalsRenderer()
# losses
self.rgb_loss = MSELoss()
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.temporal_distortion = True # for viewer
def get_outputs(self, ray_bundle: RayBundle):
assert ray_bundle.times is not None, "Time not provided."
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(
ray_bundle, density_fns=[functools.partial(f, times=ray_bundle.times) for f in self.density_fns]
)
field_outputs = self.field(ray_samples, compute_normals=self.config.predict_normals)
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
weights_list.append(weights)
ray_samples_list.append(ray_samples)
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
accumulation = self.renderer_accumulation(weights=weights)
outputs = {
"rgb": rgb,
"accumulation": accumulation,
"depth": depth,
}
if self.config.predict_normals:
outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
# These use a lot of GPU memory, so we avoid storing them for eval.
if self.training:
outputs["weights_list"] = weights_list
outputs["ray_samples_list"] = ray_samples_list
if self.training and self.config.predict_normals:
outputs["rendered_orientation_loss"] = orientation_loss(
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
)
outputs["rendered_pred_normal_loss"] = pred_normal_loss(
weights.detach(),
field_outputs[FieldHeadNames.NORMALS].detach(),
field_outputs[FieldHeadNames.PRED_NORMALS],
)
for i in range(self.config.num_proposal_iterations):
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
return outputs
def get_loss_dict(self, outputs, batch, metrics_dict=None):
loss_dict = {}
image = batch["image"].to(self.device)
loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"])
if self.training:
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
assert metrics_dict is not None and "distortion" in metrics_dict
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"]
if self.config.predict_normals:
# orientation loss for computed normals
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
outputs["rendered_orientation_loss"]
)
# ground truth supervision for normals
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
outputs["rendered_pred_normal_loss"]
)
if "depth_image" in batch.keys() and self.config.depth_weight > 0:
mask = (batch["depth_image"] != 0).view([-1])
loss_dict["depth_loss"] = 0
l = lambda x: self.config.depth_weight * (x - batch["depth_image"][mask]).pow(2).mean()
loss_dict["depth_loss"] = l(outputs["depth"][mask])
for i in range(self.config.num_proposal_iterations):
loss_dict["depth_loss"] += l(outputs[f"prop_depth_{i}"][mask])
if self.config.temporal_tv_weight > 0:
loss_dict["temporal_tv_loss"] = self.field.mlp_base.get_temporal_tv_loss()
for net in self.proposal_networks:
loss_dict["temporal_tv_loss"] += net.encoding.get_temporal_tv_loss()
loss_dict["temporal_tv_loss"] *= self.config.temporal_tv_weight
return loss_dict
+241
View File
@@ -0,0 +1,241 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Implementation of NeRFPlayer (https://arxiv.org/abs/2210.15947) with InstantNGP backbone.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Type
import nerfacc
import torch
from nerfacc import ContractionType
from torch.nn import Parameter
from torchmetrics import PeakSignalNoiseRatio
from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from typing_extensions import Literal
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.fields.nerfplayer_ngp_field import NerfplayerNGPField
from nerfstudio.model_components.losses import MSELoss
from nerfstudio.model_components.ray_samplers import VolumetricSampler
from nerfstudio.model_components.renderers import (
AccumulationRenderer,
DepthRenderer,
RGBRenderer,
)
from nerfstudio.models.base_model import Model
from nerfstudio.models.instant_ngp import InstantNGPModelConfig, NGPModel
from nerfstudio.utils import colors
@dataclass
class NerfplayerNGPModelConfig(InstantNGPModelConfig):
"""NeRFPlayer Model Config with InstantNGP backbone.
Tips for tuning the performance:
1. If the scene is flickering, this is caused by unwanted high-freq on the temporal dimension.
Try reducing `temporal_dim` first, but don't be too small, otherwise the dynamic object is blurred.
Then try increasing the `temporal_tv_weight`. This is the loss for promoting smoothness among the
temporal channels.
2. If a faster rendering is preferred, then try reducing `log2_hashmap_size`. If more details are
wanted, try increasing `log2_hashmap_size`.
3. If the input cameras are of limited numbers, try reducing `num_levels`. `num_levels` is for
multi-resolution volume sampling, and has a similar behavior to the freq in NeRF. With a small
`num_levels`, a blurred rendering will be generated, but it is unlikely to overfit the training views.
"""
_target: Type = field(default_factory=lambda: NerfplayerNGPModel)
temporal_dim: int = 64
"""Hashing grid parameter. A higher temporal dim means a higher temporal frequency."""
num_levels: int = 16
"""Hashing grid parameter."""
features_per_level: int = 2
"""Hashing grid parameter."""
log2_hashmap_size: int = 17
"""Hashing grid parameter."""
base_resolution: int = 16
"""Hashing grid parameter."""
temporal_tv_weight: float = 1
"""Temporal TV loss balancing weight for feature channels."""
depth_weight: float = 1e-1
"""depth loss balancing weight for feature channels."""
train_background_color: Literal["random", "black", "white"] = "random"
"""The training background color that is given to untrained areas."""
eval_background_color: Literal["random", "black", "white"] = "white"
"""The training background color that is given to untrained areas."""
disable_viewing_dependent: bool = True
"""Disable viewing dependent effects."""
class NerfplayerNGPModel(NGPModel):
"""NeRFPlayer Model with InstantNGP backbone.
Args:
config: NeRFPlayer NGP configuration to instantiate model
"""
config: NerfplayerNGPModelConfig
field: NerfplayerNGPField
def populate_modules(self):
"""Set the fields and modules."""
Model.populate_modules(self)
self.field = NerfplayerNGPField(
aabb=self.scene_box.aabb,
contraction_type=self.config.contraction_type,
use_appearance_embedding=self.config.use_appearance_embedding,
num_images=self.num_train_data,
temporal_dim=self.config.temporal_dim,
num_levels=self.config.num_levels,
features_per_level=self.config.features_per_level,
log2_hashmap_size=self.config.log2_hashmap_size,
base_resolution=self.config.base_resolution,
disable_viewing_dependent=self.config.disable_viewing_dependent,
)
self.scene_aabb = Parameter(self.scene_box.aabb.flatten(), requires_grad=False)
# Occupancy Grid
self.occupancy_grid = nerfacc.OccupancyGrid(
roi_aabb=self.scene_aabb,
resolution=self.config.grid_resolution,
contraction_type=self.config.contraction_type,
)
# Sampler
vol_sampler_aabb = self.scene_box.aabb if self.config.contraction_type == ContractionType.AABB else None
self.sampler = VolumetricSampler(
scene_aabb=vol_sampler_aabb,
occupancy_grid=self.occupancy_grid,
density_fn=self.field.density_fn,
) # need to update the density_fn later during forward (for input time)
# renderers
self.train_background_color = self.config.train_background_color
self.eval_background_color = self.config.eval_background_color
if self.config.train_background_color in ["white", "black"]:
self.train_background_color = colors.COLORS_DICT[self.config.train_background_color]
if self.config.eval_background_color in ["white", "black"]:
self.eval_background_color = colors.COLORS_DICT[self.config.eval_background_color]
self.renderer_rgb = RGBRenderer() # will update bgcolor later during forward
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer(method="expected")
# losses
self.rgb_loss = MSELoss()
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
self.temporal_distortion = True # for viewer
def get_outputs(self, ray_bundle: RayBundle):
num_rays = len(ray_bundle)
# update the density_fn of the sampler so that the density is time aware
self.sampler.density_fn = lambda x: self.field.density_fn(x, ray_bundle.times)
with torch.no_grad():
ray_samples, ray_indices = self.sampler(
ray_bundle=ray_bundle,
near_plane=self.config.near_plane,
far_plane=self.config.far_plane,
render_step_size=self.config.render_step_size,
cone_angle=self.config.cone_angle,
)
field_outputs = self.field(ray_samples)
# accumulation
packed_info = nerfacc.pack_info(ray_indices, num_rays)
weights = nerfacc.render_weight_from_density(
packed_info=packed_info,
sigmas=field_outputs[FieldHeadNames.DENSITY],
t_starts=ray_samples.frustums.starts,
t_ends=ray_samples.frustums.ends,
)
# update bgcolor in the renderer; usually random color for training and fixed color for inference
if self.training:
self.renderer_rgb.background_color = self.train_background_color
else:
self.renderer_rgb.background_color = self.eval_background_color
rgb = self.renderer_rgb(
rgb=field_outputs[FieldHeadNames.RGB],
weights=weights,
ray_indices=ray_indices,
num_rays=num_rays,
)
depth = self.renderer_depth(
weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays
)
accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays)
alive_ray_mask = accumulation.squeeze(-1) > 0
outputs = {
"rgb": rgb,
"accumulation": accumulation,
"depth": depth,
"alive_ray_mask": alive_ray_mask, # the rays we kept from sampler
"num_samples_per_ray": packed_info[:, 1],
}
# adding them to outputs for calculating losses
if self.training and self.config.depth_weight > 0:
outputs["ray_indices"] = ray_indices
outputs["ray_samples"] = ray_samples
outputs["weights"] = weights
outputs["sigmas"] = field_outputs[FieldHeadNames.DENSITY]
return outputs
def get_loss_dict(self, outputs, batch, metrics_dict=None):
image = batch["image"].to(self.device)
mask = outputs["alive_ray_mask"]
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
loss_dict = {"rgb_loss": rgb_loss}
if "depth_image" in batch.keys() and self.config.depth_weight > 0:
mask = batch["depth_image"] != 0
# First we calculate the depth value, just like most of the papers.
loss_dict["depth_loss"] = (outputs["depth"][mask] - batch["depth_image"][mask]).abs().mean()
# But this is not enough -- it will lead to fog like reconstructions, even with depth supervision.
# Because this loss only cares about the mean value.
# (Feel free to try it and see the fog-like effects on DyCheck by commenting out the following loss.)
# > The fog can be effectively penalized by multiview inputs (i.e., from another viewing point).
# > However, it is hard to get multiple views under the setting of dynamic scenes.
# > A new view always indicates another camera, which is expensive and brings sync problems.
# In DyCheck (https://arxiv.org/abs/2210.13445), surface sparsity regularizer
# (i.e., `distortion_loss` in nerfstudio) is used to make surface tight.
# The `distortion_loss` can be used here as well, but personally find it hard to implement...
# (Due to volume sampling, seems that cuda kernels are needed for efficiently computing the loss.)
# (Try nerfplayer with nerfacto backbone for distortion loss.)
# Instead, directly penalizing the empty space is found effective here. Perhaps due to a more
# "clear" loss (as it is directly applied to the network outputs, rather than post-processed values).
# But such a loss also has drawbacks: it tends to overfit wrong (or noise) presented in the depth map.
# Some structures are floating in the air with this loss...
gt_depth_packed = batch["depth_image"][outputs["ray_indices"]]
steps = (outputs["ray_samples"].frustums.starts + outputs["ray_samples"].frustums.ends) / 2
# empty area should not be too close to the given depth, so lets add a margin to the gt depth
margin = (self.scene_box.aabb.max() - self.scene_box.aabb.min()) / 128
density_min_mask = (gt_depth_packed - steps > margin) & (gt_depth_packed != 0)
loss_dict["depth_loss"] += (outputs["sigmas"][density_min_mask[..., 0]].pow(2)).mean() * 1e-2
loss_dict["depth_loss"] *= self.config.depth_weight
if self.config.temporal_tv_weight > 0:
loss_dict["temporal_tv_loss"] = self.config.temporal_tv_weight * self.field.mlp_base.get_temporal_tv_loss()
return loss_dict