mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 05:14:15 +08:00
Migrate compound model to nerfacc (#474)
This commit is contained in:
1 parent
32943caf5b
commit
2be4625b4f
5 files changed
+136
-200
No files matched your search
@@ -38,7 +38,6 @@ from nerfactory.datamanagers.dataparsers.record3d_parser import Record3D
|
||||
|
||||
# model instances
|
||||
from nerfactory.models.base import Model
|
||||
from nerfactory.models.compound import CompoundModel
|
||||
from nerfactory.models.nerfw import NerfWModel
|
||||
from nerfactory.models.tensorf import TensoRFModel
|
||||
from nerfactory.optimizers.schedulers import ExponentialDecaySchedule
|
||||
@@ -398,22 +397,6 @@ class VanillaModelConfig(InstantiateConfig):
|
||||
"""Vanilla Model Config"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompoundModelConfig(ModelConfig):
|
||||
"""Compound Model Config"""
|
||||
|
||||
_target: Type = CompoundModel
|
||||
enable_density_field: bool = True
|
||||
enable_collider: bool = False
|
||||
field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ...
|
||||
loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss": 1.0})
|
||||
num_samples: int = 1024 # instead of course/fine samples
|
||||
cone_angle: float = 0.0
|
||||
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
|
||||
near_plane: float = 0.05
|
||||
"""How far along ray to start sampling."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class NerfWModelConfig(ModelConfig):
|
||||
"""NerfW model config"""
|
||||
|
||||
@@ -24,7 +24,6 @@ from typeguard import typeguard_ignore
|
||||
|
||||
from nerfactory.configs.base import (
|
||||
BlenderDataParserConfig,
|
||||
CompoundModelConfig,
|
||||
Config,
|
||||
FriendsDataManagerConfig,
|
||||
MipNerf360DataParserConfig,
|
||||
@@ -37,6 +36,7 @@ from nerfactory.configs.base import (
|
||||
TrainerConfig,
|
||||
VanillaDataManagerConfig,
|
||||
)
|
||||
from nerfactory.models.compound import CompoundModelConfig
|
||||
from nerfactory.models.instant_ngp import InstantNGPModelConfig
|
||||
from nerfactory.models.mipnerf import MipNerfModel
|
||||
from nerfactory.models.mipnerf_360 import MipNerf360Model
|
||||
|
||||
@@ -49,24 +49,39 @@ except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def get_normalized_directions(directions):
|
||||
"""SH encoding must be in the range [0, 1]"""
|
||||
def get_normalized_directions(directions: TensorType["bs":..., 3]):
|
||||
"""SH encoding must be in the range [0, 1]
|
||||
|
||||
Args:
|
||||
directions: batch of directions
|
||||
"""
|
||||
return (directions + 1.0) / 2.0
|
||||
|
||||
|
||||
class TCNNCompoundField(Field):
|
||||
"""Compound Field that uses TCNN"""
|
||||
"""Compound Field that uses TCNN
|
||||
|
||||
Args:
|
||||
aabb: parameters of scene aabb bounds
|
||||
num_layers: number of hidden layers
|
||||
hidden_dim: dimension of hidden layers
|
||||
geo_feat_dim: output geo feat dimensions
|
||||
num_layers_color: number of hidden layers for color network
|
||||
hidden_dim_color: dimension of hidden layers for color network
|
||||
appearance_embedding_dim: dimension of appearance embedding
|
||||
spatial_distortion: spatial distortion to apply to the scene
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aabb,
|
||||
num_images: int,
|
||||
num_layers=2,
|
||||
hidden_dim=64,
|
||||
geo_feat_dim=15,
|
||||
num_layers_color=3,
|
||||
hidden_dim_color=64,
|
||||
appearance_embedding_dim: int = 40,
|
||||
num_layers: int = 2,
|
||||
hidden_dim: int = 64,
|
||||
geo_feat_dim: int = 15,
|
||||
num_layers_color: int = 3,
|
||||
hidden_dim_color: int = 64,
|
||||
appearance_embedding_dim: int = 32,
|
||||
spatial_distortion: SpatialDistortion = SceneContraction(),
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
+111
-70
@@ -28,23 +28,29 @@ TODO:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Literal, Optional, Tuple, Type
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
from torchmetrics import PeakSignalNoiseRatio
|
||||
from torchmetrics.functional import structural_similarity_index_measure
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
|
||||
import nerfactory.cuda as nerfactory_cuda
|
||||
from nerfactory.cameras.rays import RayBundle
|
||||
from nerfactory.configs import base as cfg
|
||||
from nerfactory.fields.compound_field import field_implementation_to_class
|
||||
from nerfactory.fields.modules.field_heads import FieldHeadNames
|
||||
from nerfactory.models.base import Model
|
||||
from nerfactory.models.modules.ray_sampler import NGPSpacedSampler
|
||||
from nerfactory.models.modules.ray_sampler import VolumetricSampler
|
||||
from nerfactory.optimizers.loss import MSELoss
|
||||
from nerfactory.utils import colors, misc, visualization
|
||||
from nerfactory.renderers.renderers import (
|
||||
AccumulationRenderer,
|
||||
DepthRenderer,
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfactory.utils import colors, visualization
|
||||
from nerfactory.utils.callbacks import (
|
||||
TrainingCallback,
|
||||
TrainingCallbackAttributes,
|
||||
@@ -52,44 +58,69 @@ from nerfactory.utils.callbacks import (
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompoundModelConfig(cfg.ModelConfig):
|
||||
"""Compound Model Config"""
|
||||
|
||||
_target: Type = field(
|
||||
default_factory=lambda: CompoundModel
|
||||
) # We can't write `CompoundModel` directly, because `CompoundModel` doesn't exist yet
|
||||
"""target class to instantiate"""
|
||||
enable_collider: bool = False
|
||||
"""Whether to create a scene collider to filter rays."""
|
||||
collider_params: Optional[Dict[str, float]] = None
|
||||
"""Instant NGP doesn't use a collider."""
|
||||
field_implementation: Literal["torch", "tcnn"] = "tcnn"
|
||||
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
|
||||
num_samples: int = 1024
|
||||
"""Number of max samples per ray"""
|
||||
grid_resolution: int = 128
|
||||
"""Resolution of the grid used for the field."""
|
||||
cone_angle: float = 0.0
|
||||
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
|
||||
near_plane: float = 0.05
|
||||
"""How far along ray to start sampling."""
|
||||
randomize_background: bool = False
|
||||
"""Whether to randomize the background color."""
|
||||
|
||||
|
||||
class CompoundModel(Model):
|
||||
"""Compound model
|
||||
|
||||
Args:
|
||||
field_implementation (str): one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'
|
||||
kwargs: additional params to pass up to the parent class model
|
||||
config: Compound model configuration to instantiate model
|
||||
"""
|
||||
|
||||
config: cfg.CompoundModelConfig
|
||||
config: CompoundModelConfig
|
||||
|
||||
def __init__(self, config: cfg.CompoundModelConfig, **kwargs) -> None:
|
||||
def __init__(self, config: CompoundModelConfig, **kwargs) -> None:
|
||||
assert config.field_implementation in field_implementation_to_class
|
||||
self.field = None
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
assert self.density_field is not None
|
||||
return [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
|
||||
update_every_num_iters=self.density_field.update_every_num_iters,
|
||||
func=self.density_field.update_density_grid,
|
||||
kwargs={"density_eval_func": self.field.density_fn}, # type: ignore
|
||||
)
|
||||
]
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules."""
|
||||
super().populate_modules()
|
||||
# torch or tiny-cuda-nn version
|
||||
self.field = field_implementation_to_class[self.config.field_implementation](
|
||||
self.scene_bounds.aabb, self.num_train_data
|
||||
aabb=self.scene_bounds.aabb, num_images=self.num_train_data
|
||||
)
|
||||
|
||||
# samplers
|
||||
self.sampler = NGPSpacedSampler(num_samples=self.config.num_samples, density_field=self.density_field)
|
||||
self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False)
|
||||
|
||||
# Sampler
|
||||
self.sampler = VolumetricSampler(
|
||||
aabb=self.scene_aabb,
|
||||
density_fn=self.field.density_fn,
|
||||
grid_resolution=self.config.grid_resolution,
|
||||
num_samples=self.config.num_samples,
|
||||
)
|
||||
|
||||
# renderers
|
||||
background_color = None if self.config.randomize_background else colors.WHITE
|
||||
self.renderer_rgb = RGBRenderer(background_color=background_color)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer()
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
@@ -101,6 +132,17 @@ class CompoundModel(Model):
|
||||
|
||||
# no colliders default
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
return [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=self.sampler.occ_field.every_n_step,
|
||||
)
|
||||
]
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
if self.field is None:
|
||||
@@ -109,75 +151,69 @@ class CompoundModel(Model):
|
||||
return param_groups
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
# TODO(ruilongli)
|
||||
# - train test difference
|
||||
# - visualize "depth_density_grid"
|
||||
assert self.field is not None
|
||||
num_rays = len(ray_bundle)
|
||||
device = ray_bundle.origins.device
|
||||
|
||||
if self.field is None:
|
||||
raise ValueError("populate_fields() must be called before get_outputs")
|
||||
ray_samples, packed_info, t_min, t_max = self.sampler(
|
||||
ray_bundle, self.field.aabb, cone_angle=self.config.cone_angle, near_plane=self.config.near_plane
|
||||
)
|
||||
with torch.no_grad():
|
||||
ray_samples, packed_info, ray_indices = self.sampler(
|
||||
ray_bundle=ray_bundle,
|
||||
near_plane=self.config.near_plane,
|
||||
)
|
||||
|
||||
field_outputs = self.field.forward(ray_samples)
|
||||
|
||||
rgbs = field_outputs[FieldHeadNames.RGB]
|
||||
sigmas = field_outputs[FieldHeadNames.DENSITY]
|
||||
|
||||
# accumulate all the rays start from zero opacity
|
||||
opacities = torch.zeros((num_rays, 1), device=device)
|
||||
(
|
||||
accumulated_weight,
|
||||
accumulated_depth,
|
||||
accumulated_color,
|
||||
alive_ray_mask,
|
||||
) = nerfactory_cuda.VolumeRenderer.apply(
|
||||
packed_info,
|
||||
ray_samples.frustums.starts,
|
||||
ray_samples.frustums.ends,
|
||||
sigmas.contiguous(),
|
||||
rgbs.contiguous(),
|
||||
opacities,
|
||||
# accumulation
|
||||
weights = nerfacc.volumetric_rendering_weights(
|
||||
packed_info=packed_info,
|
||||
sigmas=field_outputs[FieldHeadNames.DENSITY],
|
||||
frustum_starts=ray_samples.frustums.starts,
|
||||
frustum_ends=ray_samples.frustums.ends,
|
||||
)
|
||||
accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None])
|
||||
accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight)
|
||||
|
||||
rgb = self.renderer_rgb(
|
||||
rgb=field_outputs[FieldHeadNames.RGB],
|
||||
weights=weights,
|
||||
ray_indices=ray_indices,
|
||||
num_rays=num_rays,
|
||||
)
|
||||
depth = self.renderer_depth(
|
||||
weights=weights, ray_samples=ray_samples, ray_indices=ray_indices, num_rays=num_rays
|
||||
)
|
||||
accumulation = self.renderer_accumulation(weights=weights, ray_indices=ray_indices, num_rays=num_rays)
|
||||
alive_ray_mask = accumulation.squeeze(-1) > 0
|
||||
|
||||
outputs = {
|
||||
"rgb": accumulated_color,
|
||||
"accumulation": accumulated_weight,
|
||||
"depth": accumulated_depth,
|
||||
"rgb": rgb,
|
||||
"accumulation": accumulation,
|
||||
"depth": depth,
|
||||
"alive_ray_mask": alive_ray_mask, # the rays we kept from sampler
|
||||
}
|
||||
return outputs
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients):
|
||||
device = self.device
|
||||
image = batch["image"].to(device)
|
||||
if "alive_ray_mask" in outputs:
|
||||
mask = outputs["alive_ray_mask"]
|
||||
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
|
||||
else:
|
||||
rgb_loss = self.rgb_loss(image, outputs["rgb"])
|
||||
image = batch["image"].to(self.device)
|
||||
mask = outputs["alive_ray_mask"]
|
||||
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
|
||||
loss_dict = {"rgb_loss": rgb_loss}
|
||||
loss_dict = misc.scale_dict(loss_dict, loss_coefficients)
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
|
||||
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
|
||||
device = self.device
|
||||
image = batch["image"].to(device)
|
||||
|
||||
image = batch["image"].to(self.device)
|
||||
rgb = outputs["rgb"]
|
||||
acc = visualization.apply_colormap(outputs["accumulation"])
|
||||
depth = visualization.apply_depth_colormap(
|
||||
outputs["depth"],
|
||||
accumulation=outputs["accumulation"],
|
||||
)
|
||||
alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"])
|
||||
|
||||
combined_rgb = torch.cat([image, rgb], dim=1)
|
||||
combined_acc = torch.cat([acc], dim=1)
|
||||
combined_depth = torch.cat([depth], dim=1)
|
||||
combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1)
|
||||
|
||||
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
|
||||
image = torch.moveaxis(image, -1, 0)[None, ...]
|
||||
@@ -187,10 +223,15 @@ class CompoundModel(Model):
|
||||
ssim = self.ssim(image, rgb)
|
||||
lpips = self.lpips(image, rgb)
|
||||
|
||||
metrics_dict = {
|
||||
"psnr": float(psnr.item()),
|
||||
"ssim": float(ssim.item()),
|
||||
"lpips": float(lpips.item()),
|
||||
# all of these metrics will be logged as scalars
|
||||
metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim), "lpips": float(lpips)} # type: ignore
|
||||
# TODO(ethan): return an image dictionary
|
||||
|
||||
images_dict = {
|
||||
"img": combined_rgb,
|
||||
"accumulation": combined_acc,
|
||||
"depth": combined_depth,
|
||||
"alive_ray_mask": combined_alive_ray_mask,
|
||||
}
|
||||
images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth}
|
||||
|
||||
return metrics_dict, images_dict
|
||||
@@ -25,7 +25,6 @@ import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
import nerfactory.cuda as nerfactory_cuda
|
||||
from nerfactory.cameras.rays import Frustums, RayBundle, RaySamples
|
||||
from nerfactory.fields.density_fields.density_grid import DensityGrid
|
||||
|
||||
@@ -344,108 +343,6 @@ class PDFSampler(Sampler):
|
||||
return ray_samples
|
||||
|
||||
|
||||
class NGPSpacedSampler(Sampler):
|
||||
"""Sampler that matches Instant-NGP paper.
|
||||
|
||||
This sampler does ray-box AABB test, ray samples generation and density check, all together.
|
||||
|
||||
TODO(ruilongli): check whether fuse AABB test together with `raymarching` can speed things up.
|
||||
Otherwise we can seperate it out and use collision detecoter that already exists in the repo.
|
||||
"""
|
||||
|
||||
def generate_ray_samples(self) -> RaySamples:
|
||||
raise RuntimeError("For NGP we fused ray samples and density check together. Please call forward() directly.")
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
def forward(
|
||||
self,
|
||||
ray_bundle: RayBundle,
|
||||
aabb: TensorType[2, 3],
|
||||
num_samples: Optional[int] = None,
|
||||
cone_angle: float = 0.0,
|
||||
marching_steps: Optional[int] = 128,
|
||||
near_plane: float = 0.0,
|
||||
) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]:
|
||||
"""Generate ray samples in a bounding box.
|
||||
|
||||
TODO(ruilongli): write a Packed[Ray_samples] class to ray_samples with packed_info.
|
||||
TODO(ruilongli): maybe move aabb test to the collision detector?
|
||||
|
||||
Args:
|
||||
ray_bundle: Rays to generate samples for
|
||||
aabb: Bounding box of the scene.
|
||||
num_samples: Number of samples per ray
|
||||
cone_angle: 0.0 for nerf-synthetic and 1.0/256 for real scenes
|
||||
marching_steps: Number of marching steps (Note: not currently used)
|
||||
near_plane: Near plane for raymarching
|
||||
|
||||
Returns:
|
||||
First return is ray samples in a packed way where only the valid samples are kept.
|
||||
Second return contains all the information to recover packed samples into unpacked mode for rendering.
|
||||
The last two returns are t_min and t_max from ray-aabb test.
|
||||
"""
|
||||
|
||||
if self.density_field is None:
|
||||
raise ValueError("density_field must be set to use NGPSpacedSampler")
|
||||
|
||||
num_samples = num_samples or self.num_samples
|
||||
|
||||
aabb = aabb.flatten().contiguous()
|
||||
rays_o = ray_bundle.origins.contiguous()
|
||||
rays_d = ray_bundle.directions.contiguous()
|
||||
scene_scale = (aabb[3:6] - aabb[0:3]).max()
|
||||
|
||||
t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb)
|
||||
t_min = torch.max(t_min, torch.ones_like(t_min) * near_plane)
|
||||
|
||||
marching_steps = -1 # disable marching mode for now
|
||||
# TODO(ruilongli): * 16 is for original impl for training. Need to run
|
||||
# some profiling test with this choice.
|
||||
max_samples_per_batch = len(rays_o) * num_samples
|
||||
|
||||
# marching
|
||||
packed_info, origins, dirs, starts, ends = nerfactory_cuda.raymarching(
|
||||
# rays
|
||||
rays_o,
|
||||
rays_d,
|
||||
t_min,
|
||||
t_max,
|
||||
# density grid
|
||||
self.density_field.center,
|
||||
self.density_field.base_scale,
|
||||
self.density_field.num_cascades,
|
||||
self.density_field.resolution,
|
||||
self.density_field.density_bitfield,
|
||||
# sampling args
|
||||
marching_steps,
|
||||
max_samples_per_batch,
|
||||
num_samples,
|
||||
cone_angle,
|
||||
scene_scale,
|
||||
)
|
||||
|
||||
# squeeze valid samples
|
||||
total_samples = max(packed_info[:, -1].sum(), 1)
|
||||
origins = origins[:total_samples]
|
||||
dirs = dirs[:total_samples]
|
||||
starts = starts[:total_samples]
|
||||
ends = ends[:total_samples]
|
||||
|
||||
assert ray_bundle.camera_indices is not None
|
||||
indices = torch.zeros_like(origins[:, :1], dtype=torch.long)
|
||||
|
||||
ray_bundle.camera_indices = ray_bundle.camera_indices.to(torch.long).to(packed_info.device)
|
||||
|
||||
indices = nerfactory_cuda.get_camera_indices(packed_info, ray_bundle.camera_indices, indices)
|
||||
|
||||
zeros = torch.zeros_like(origins[:, :1])
|
||||
ray_samples = RaySamples(
|
||||
frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros),
|
||||
camera_indices=indices,
|
||||
)
|
||||
return ray_samples, packed_info, t_min, t_max
|
||||
|
||||
|
||||
class VolumetricSampler(Sampler):
|
||||
"""Sampler inspired by the one proposed in the Instant-NGP paper.
|
||||
|
||||
|
||||
Reference in new issue
Block a user