Migrate compound model to nerfacc (#474)

This commit is contained in:
Matthew Tancik authored and GitHub committed 2022-09-18 14:14:30 -07:00
1 parent 32943caf5b
commit 2be4625b4f
5 files changed
+136 -200

No files matched your search

-17
View File
@@ -38,7 +38,6 @@ from nerfactory.datamanagers.dataparsers.record3d_parser import Record3D
# model instances
from nerfactory.models.base import Model
from nerfactory.models.compound import CompoundModel
from nerfactory.models.nerfw import NerfWModel
from nerfactory.models.tensorf import TensoRFModel
from nerfactory.optimizers.schedulers import ExponentialDecaySchedule
@@ -398,22 +397,6 @@ class VanillaModelConfig(InstantiateConfig):
"""Vanilla Model Config"""
@dataclass
class CompoundModelConfig(ModelConfig):
"""Compound Model Config"""
_target: Type = CompoundModel
enable_density_field: bool = True
enable_collider: bool = False
field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ...
loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss": 1.0})
num_samples: int = 1024 # instead of course/fine samples
cone_angle: float = 0.0
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
near_plane: float = 0.05
"""How far along ray to start sampling."""
@dataclass
class NerfWModelConfig(ModelConfig):
"""NerfW model config"""
+1 -1
View File
@@ -24,7 +24,6 @@ from typeguard import typeguard_ignore
from nerfactory.configs.base import (
BlenderDataParserConfig,
CompoundModelConfig,
Config,
FriendsDataManagerConfig,
MipNerf360DataParserConfig,
@@ -37,6 +36,7 @@ from nerfactory.configs.base import (
TrainerConfig,
VanillaDataManagerConfig,
)
from nerfactory.models.compound import CompoundModelConfig
from nerfactory.models.instant_ngp import InstantNGPModelConfig
from nerfactory.models.mipnerf import MipNerfModel
from nerfactory.models.mipnerf_360 import MipNerf360Model
+24 -9
View File
@@ -49,24 +49,39 @@ except ImportError:
pass
def get_normalized_directions(directions):
"""SH encoding must be in the range [0, 1]"""
def get_normalized_directions(directions: TensorType["bs":..., 3]):
"""SH encoding must be in the range [0, 1]
Args:
directions: batch of directions
"""
return (directions + 1.0) / 2.0
class TCNNCompoundField(Field):
"""Compound Field that uses TCNN"""
"""Compound Field that uses TCNN
Args:
aabb: parameters of scene aabb bounds
num_layers: number of hidden layers
hidden_dim: dimension of hidden layers
geo_feat_dim: output geo feat dimensions
num_layers_color: number of hidden layers for color network
hidden_dim_color: dimension of hidden layers for color network
appearance_embedding_dim: dimension of appearance embedding
spatial_distortion: spatial distortion to apply to the scene
"""
def __init__(
self,
aabb,
num_images: int,
num_layers=2,
hidden_dim=64,
geo_feat_dim=15,
num_layers_color=3,
hidden_dim_color=64,
appearance_embedding_dim: int = 40,
num_layers: int = 2,
hidden_dim: int = 64,
geo_feat_dim: int = 15,
num_layers_color: int = 3,
hidden_dim_color: int = 64,
appearance_embedding_dim: int = 32,
spatial_distortion: SpatialDistortion = SceneContraction(),
) -> None:
super().__init__()
+111 -70
View File
@@ -28,23 +28,29 @@ TODO:
from __future__ import annotations
from typing import Dict, List, Tuple
from dataclasses import dataclass, field
from typing import Dict, List, Literal, Optional, Tuple, Type
import nerfacc
import torch
from torch.nn import Parameter
from torchmetrics import PeakSignalNoiseRatio
from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
import nerfactory.cuda as nerfactory_cuda
from nerfactory.cameras.rays import RayBundle
from nerfactory.configs import base as cfg
from nerfactory.fields.compound_field import field_implementation_to_class
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.models.base import Model
from nerfactory.models.modules.ray_sampler import NGPSpacedSampler
from nerfactory.models.modules.ray_sampler import VolumetricSampler
from nerfactory.optimizers.loss import MSELoss
from nerfactory.utils import colors, misc, visualization
from nerfactory.renderers.renderers import (
AccumulationRenderer,
DepthRenderer,
RGBRenderer,
)
from nerfactory.utils import colors, visualization
from nerfactory.utils.callbacks import (
TrainingCallback,
TrainingCallbackAttributes,
@@ -52,44 +58,69 @@ from nerfactory.utils.callbacks import (
)
@dataclass
class CompoundModelConfig(cfg.ModelConfig):
"""Compound Model Config"""
_target: Type = field(
default_factory=lambda: CompoundModel
) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet
"""target class to instantiate"""
enable_collider: bool = False
"""Whether to create a scene collider to filter rays."""
collider_params: Optional[Dict[str, float]] = None
"""Instant NGP doesn't use a collider."""
field_implementation: Literal["torch", "tcnn"] = "tcnn"
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
num_samples: int = 1024
"""Number of max samples per ray"""
grid_resolution: int = 128
"""Resolution of the grid used for the field."""
cone_angle: float = 0.0
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
near_plane: float = 0.05
"""How far along ray to start sampling."""
randomize_background: bool = False
"""Whether to randomize the background color."""
class CompoundModel(Model):
"""Compound model
Args:
field_implementation (str): one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'
kwargs: additional params to pass up to the parent class model
config: Compound model configuration to instantiate model
"""
config: cfg.CompoundModelConfig
config: CompoundModelConfig
def __init__(self, config: cfg.CompoundModelConfig, **kwargs) -> None:
def __init__(self, config: CompoundModelConfig, **kwargs) -> None:
assert config.field_implementation in field_implementation_to_class
self.field = None
super().__init__(config=config, **kwargs)
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
) -> List[TrainingCallback]:
assert self.density_field is not None
return [
TrainingCallback(
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
update_every_num_iters=self.density_field.update_every_num_iters,
func=self.density_field.update_density_grid,
kwargs={"density_eval_func": self.field.density_fn}, # type: ignore
)
]
def populate_modules(self):
"""Set the fields and modules."""
super().populate_modules()
# torch or tiny-cuda-nn version
self.field = field_implementation_to_class[self.config.field_implementation](
self.scene_bounds.aabb, self.num_train_data
aabb=self.scene_bounds.aabb, num_images=self.num_train_data
)
# samplers
self.sampler = NGPSpacedSampler(num_samples=self.config.num_samples, density_field=self.density_field)
self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False)
# Sampler
self.sampler = VolumetricSampler(
aabb=self.scene_aabb,
density_fn=self.field.density_fn,
grid_resolution=self.config.grid_resolution,
num_samples=self.config.num_samples,
)
# renderers
background_color = None if self.config.randomize_background else colors.WHITE
self.renderer_rgb = RGBRenderer(background_color=background_color)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer()
# losses
self.rgb_loss = MSELoss()
@@ -101,6 +132,17 @@ class CompoundModel(Model):
# no colliders default
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
) -> List[TrainingCallback]:
return [
TrainingCallback(
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
update_every_num_iters=1,
func=self.sampler.occ_field.every_n_step,
)
]
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
if self.field is None:
@@ -109,75 +151,69 @@ class CompoundModel(Model):
return param_groups
def get_outputs(self, ray_bundle: RayBundle):
# TODO(ruilongli)
# - train test difference
# - visualize "depth_density_grid"
assert self.field is not None
num_rays = len(ray_bundle)
device = ray_bundle.origins.device
if self.field is None:
raise ValueError("populate_fields() must be called before get_outputs")
ray_samples, packed_info, t_min, t_max = self.sampler(
ray_bundle, self.field.aabb, cone_angle=self.config.cone_angle, near_plane=self.config.near_plane
)
with torch.no_grad():
ray_samples, packed_info, ray_indices = self.sampler(
ray_bundle=ray_bundle,
near_plane=self.config.near_plane,
)
field_outputs = self.field.forward(ray_samples)
rgbs = field_outputs[FieldHeadNames.RGB]
sigmas = field_outputs[FieldHeadNames.DENSITY]
# accumulate all the rays start from zero opacity
opacities = torch.zeros((num_rays, 1), device=device)
(
accumulated_weight,
accumulated_depth,
accumulated_color,
alive_ray_mask,
) = nerfactory_cuda.VolumeRenderer.apply(
packed_info,
ray_samples.frustums.starts,
ray_samples.frustums.ends,
sigmas.contiguous(),
rgbs.contiguous(),
opacities,
# accumulation
weights = nerfacc.volumetric_rendering_weights(
packed_info=packed_info,
sigmas=field_outputs[FieldHeadNames.DENSITY],
frustum_starts=ray_samples.frustums.starts,
frustum_ends=ray_samples.frustums.ends,
)
accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None])
accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight)
rgb = self.renderer_rgb(
rgb=field_outputs[FieldHeadNames.RGB],
weights=weights,
ray_indices=ray_indices,
num_rays=num_rays,
)
depth = self.renderer_depth(
weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays
)
accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays)
alive_ray_mask = accumulation.squeeze(-1) > 0
outputs = {
"rgb": accumulated_color,
"accumulation": accumulated_weight,
"depth": accumulated_depth,
"rgb": rgb,
"accumulation": accumulation,
"depth": depth,
"alive_ray_mask": alive_ray_mask, # the rays we kept from sampler
}
return outputs
def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients):
device = self.device
image = batch["image"].to(device)
if "alive_ray_mask" in outputs:
mask = outputs["alive_ray_mask"]
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
else:
rgb_loss = self.rgb_loss(image, outputs["rgb"])
image = batch["image"].to(self.device)
mask = outputs["alive_ray_mask"]
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
loss_dict = {"rgb_loss": rgb_loss}
loss_dict = misc.scale_dict(loss_dict, loss_coefficients)
return loss_dict
def get_image_metrics_and_images(
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
device = self.device
image = batch["image"].to(device)
image = batch["image"].to(self.device)
rgb = outputs["rgb"]
acc = visualization.apply_colormap(outputs["accumulation"])
depth = visualization.apply_depth_colormap(
outputs["depth"],
accumulation=outputs["accumulation"],
)
alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"])
combined_rgb = torch.cat([image, rgb], dim=1)
combined_acc = torch.cat([acc], dim=1)
combined_depth = torch.cat([depth], dim=1)
combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1)
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
image = torch.moveaxis(image, -1, 0)[None, ...]
@@ -187,10 +223,15 @@ class CompoundModel(Model):
ssim = self.ssim(image, rgb)
lpips = self.lpips(image, rgb)
metrics_dict = {
"psnr": float(psnr.item()),
"ssim": float(ssim.item()),
"lpips": float(lpips.item()),
# all of these metrics will be logged as scalars
metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore
# TODO(ethan): return an image dictionary
images_dict = {
"img": combined_rgb,
"accumulation": combined_acc,
"depth": combined_depth,
"alive_ray_mask": combined_alive_ray_mask,
}
images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth}
return metrics_dict, images_dict
-103
View File
@@ -25,7 +25,6 @@ import torch
from torch import nn
from torchtyping import TensorType
import nerfactory.cuda as nerfactory_cuda
from nerfactory.cameras.rays import Frustums, RayBundle, RaySamples
from nerfactory.fields.density_fields.density_grid import DensityGrid
@@ -344,108 +343,6 @@ class PDFSampler(Sampler):
return ray_samples
class NGPSpacedSampler(Sampler):
"""Sampler that matches Instant-NGP paper.
This sampler does ray-box AABB test, ray samples generation and density check, all together.
TODO(ruilongli): check whether fuse AABB test together with `raymarching` can speed things up.
Otherwise we can seperate it out and use collision detecoter that already exists in the repo.
"""
def generate_ray_samples(self) -> RaySamples:
raise RuntimeError("For NGP we fused ray samples and density check together. Please call forward() directly.")
# pylint: disable=arguments-differ
def forward(
self,
ray_bundle: RayBundle,
aabb: TensorType[2, 3],
num_samples: Optional[int] = None,
cone_angle: float = 0.0,
marching_steps: Optional[int] = 128,
near_plane: float = 0.0,
) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]:
"""Generate ray samples in a bounding box.
TODO(ruilongli): write a Packed[Ray_samples] class to ray_samples with packed_info.
TODO(ruilongli): maybe move aabb test to the collision detector?
Args:
ray_bundle: Rays to generate samples for
aabb: Bounding box of the scene.
num_samples: Number of samples per ray
cone_angle: 0.0 for nerf-synthetic and 1.0/256 for real scenes
marching_steps: Number of marching steps (Note: not currently used)
near_plane: Near plane for raymarching
Returns:
First return is ray samples in a packed way where only the valid samples are kept.
Second return contains all the information to recover packed samples into unpacked mode for rendering.
The last two returns are t_min and t_max from ray-aabb test.
"""
if self.density_field is None:
raise ValueError("density_field must be set to use NGPSpacedSampler")
num_samples = num_samples or self.num_samples
aabb = aabb.flatten().contiguous()
rays_o = ray_bundle.origins.contiguous()
rays_d = ray_bundle.directions.contiguous()
scene_scale = (aabb[3:6] - aabb[0:3]).max()
t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb)
t_min = torch.max(t_min, torch.ones_like(t_min) * near_plane)
marching_steps = -1 # disable marching mode for now
# TODO(ruilongli): * 16 is for original impl for training. Need to run
# some profiling test with this choice.
max_samples_per_batch = len(rays_o) * num_samples
# marching
packed_info, origins, dirs, starts, ends = nerfactory_cuda.raymarching(
# rays
rays_o,
rays_d,
t_min,
t_max,
# density grid
self.density_field.center,
self.density_field.base_scale,
self.density_field.num_cascades,
self.density_field.resolution,
self.density_field.density_bitfield,
# sampling args
marching_steps,
max_samples_per_batch,
num_samples,
cone_angle,
scene_scale,
)
# squeeze valid samples
total_samples = max(packed_info[:, -1].sum(), 1)
origins = origins[:total_samples]
dirs = dirs[:total_samples]
starts = starts[:total_samples]
ends = ends[:total_samples]
assert ray_bundle.camera_indices is not None
indices = torch.zeros_like(origins[:, :1], dtype=torch.long)
ray_bundle.camera_indices = ray_bundle.camera_indices.to(torch.long).to(packed_info.device)
indices = nerfactory_cuda.get_camera_indices(packed_info, ray_bundle.camera_indices, indices)
zeros = torch.zeros_like(origins[:, :1])
ray_samples = RaySamples(
frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros),
camera_indices=indices,
)
return ray_samples, packed_info, t_min, t_max
class VolumetricSampler(Sampler):
"""Sampler inspired by the one proposed in the Instant-NGP paper.