From 84024a4e731e4c99a4d67bea0b10c51275932b46 Mon Sep 17 00:00:00 2001 From: Terrance Wang Date: Fri, 16 Sep 2022 14:38:25 -0700 Subject: [PATCH] compound model (wip) (#427) * compound model now uses compound field, passing num_images in the config, working on passing image indices through ngp sampler * renaming cuda func, adding torch compound support * removing unused imports * getting num images from dataloader instead of config * fixing pylint and adding todo * undoing base config change * adding cone angle and near plane to compound model config --- nerfactory/configs/base.py | 4 + nerfactory/cuda/__init__.py | 1 + nerfactory/cuda/csrc/nerfactory.cpp | 6 ++ nerfactory/cuda/csrc/nerfactory.cu | 7 ++ nerfactory/cuda/csrc/raymarching.cu | 34 ++++++ nerfactory/fields/compound_field.py | 125 ++++++++++++++++++----- nerfactory/models/base.py | 2 + nerfactory/models/compound.py | 10 +- nerfactory/models/modules/ray_sampler.py | 9 +- nerfactory/pipelines/base.py | 3 +- 10 files changed, 170 insertions(+), 31 deletions(-) diff --git a/nerfactory/configs/base.py b/nerfactory/configs/base.py index f55abab2c..0b0958067 100644 --- a/nerfactory/configs/base.py +++ b/nerfactory/configs/base.py @@ -411,6 +411,10 @@ class CompoundModelConfig(ModelConfig): field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ... loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss": 1.0}) num_samples: int = 1024 # instead of course/fine samples + cone_angle: float = 0.0 + """Should be set to 0.0 for blender scenes but 1./256 for real scenes.""" + near_plane: float = 0.05 + """How far along ray to start sampling.""" @dataclass diff --git a/nerfactory/cuda/__init__.py b/nerfactory/cuda/__init__.py index 36802b445..e7affe6d7 100644 --- a/nerfactory/cuda/__init__.py +++ b/nerfactory/cuda/__init__.py @@ -37,6 +37,7 @@ raymarching = _make_lazy_cuda("raymarching") volumetric_rendering_forward = _make_lazy_cuda("volumetric_rendering_forward") volumetric_rendering_backward = _make_lazy_cuda("volumetric_rendering_backward") occupancy_query = _make_lazy_cuda("occupancy_query") +get_camera_indices = _make_lazy_cuda("get_camera_indices") # pylint: disable=abstract-method,arguments-differ class VolumeRenderer(torch.autograd.Function): diff --git a/nerfactory/cuda/csrc/nerfactory.cpp b/nerfactory/cuda/csrc/nerfactory.cpp index 38a39e91c..7e3818076 100644 --- a/nerfactory/cuda/csrc/nerfactory.cpp +++ b/nerfactory/cuda/csrc/nerfactory.cpp @@ -59,6 +59,12 @@ std::vector volumetric_rendering_backward( torch::Tensor opacities ) {}; +torch::Tensor get_camera_indices( + const int* packed_info, + const int* camera_indices, + torch::Tensor mapping +) {}; + std::vector occupancy_query( // samples const torch::Tensor positions, diff --git a/nerfactory/cuda/csrc/nerfactory.cu b/nerfactory/cuda/csrc/nerfactory.cu index 1b328b068..ced053728 100644 --- a/nerfactory/cuda/csrc/nerfactory.cu +++ b/nerfactory/cuda/csrc/nerfactory.cu @@ -59,6 +59,12 @@ std::vector volumetric_rendering_backward( torch::Tensor opacities ); +torch::Tensor get_camera_indices( + torch::Tensor packed_info, + torch::Tensor camera_indices, + torch::Tensor mapping +); + std::vector occupancy_query( // samples const torch::Tensor positions, @@ -81,4 +87,5 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) m.def("volumetric_rendering_forward", &volumetric_rendering_forward); m.def("volumetric_rendering_backward", &volumetric_rendering_backward); m.def("occupancy_query", &occupancy_query); + m.def("get_camera_indices", &get_camera_indices); } \ No newline at end of file diff --git a/nerfactory/cuda/csrc/raymarching.cu b/nerfactory/cuda/csrc/raymarching.cu index c073ece12..60444b0e1 100644 --- a/nerfactory/cuda/csrc/raymarching.cu +++ b/nerfactory/cuda/csrc/raymarching.cu @@ -428,6 +428,40 @@ std::vector raymarching( return {packed_info, origins, dirs, starts, ends}; } +__global__ void kernel_get_camera_indices( + const uint32_t n_rays, + const int* packed_info, + const long* camera_indices, + long* mapping +) { + CUDA_GET_THREAD_ID(thread_id, n_rays); + + const int i = packed_info[thread_id * 3 + 0]; // ray idx in {rays_o, rays_d} + const int base = packed_info[thread_id * 3 + 1]; // point idx start. + const int numsteps = packed_info[thread_id * 3 + 2]; // point idx shift. + const long camera_index = camera_indices[i]; + + int j = 0; + for (; j < numsteps; ++j) { + mapping[base + j] = camera_index; + } + + return; +} + +torch::Tensor get_camera_indices( + const torch::Tensor packed_info, + const torch::Tensor camera_indices, + torch::Tensor mapping +) { + const int n_rays = packed_info.size(0); + const int threads = 256; + const int blocks = CUDA_N_BLOCKS_NEEDED(n_rays, threads); + + kernel_get_camera_indices<<>>(n_rays, packed_info.data_ptr(), camera_indices.data_ptr(), mapping.data_ptr()); + return mapping; +} + std::vector occupancy_query( // samples diff --git a/nerfactory/fields/compound_field.py b/nerfactory/fields/compound_field.py index 85dd6a771..56ebd44c2 100644 --- a/nerfactory/fields/compound_field.py +++ b/nerfactory/fields/compound_field.py @@ -17,23 +17,29 @@ Field for compound nerf model, adds scene contraction and image embeddings to in """ -from typing import Optional, Tuple +from typing import Dict, Optional, Tuple import torch +from torch import nn from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfactory.cameras.rays import RaySamples from nerfactory.datamanagers.structs import SceneBounds -from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField +from nerfactory.fields.base import Field from nerfactory.fields.modules.embedding import Embedding from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding -from nerfactory.fields.modules.field_heads import FieldHeadNames +from nerfactory.fields.modules.field_heads import ( + DensityFieldHead, + FieldHead, + FieldHeadNames, + RGBFieldHead, +) +from nerfactory.fields.modules.mlp import MLP from nerfactory.fields.modules.spatial_distortions import ( SceneContraction, SpatialDistortion, ) -from nerfactory.fields.nerf_field import NeRFField from nerfactory.utils.activations import trunc_exp try: @@ -48,7 +54,7 @@ def get_normalized_directions(directions): return (directions + 1.0) / 2.0 -class TCNNCompoundField(TCNNInstantNGPField): +class TCNNCompoundField(Field): """Compound Field that uses TCNN""" def __init__( @@ -63,7 +69,9 @@ class TCNNCompoundField(TCNNInstantNGPField): appearance_embedding_dim: int = 40, spatial_distortion: SpatialDistortion = SceneContraction(), ) -> None: - super().__init__(self, aabb) + super().__init__() + + self.aabb = Parameter(aabb, requires_grad=False) self.geo_feat_dim = geo_feat_dim self.spatial_distortion = spatial_distortion @@ -71,10 +79,6 @@ class TCNNCompoundField(TCNNInstantNGPField): self.appearance_embedding_dim = appearance_embedding_dim self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) - self.aabb = Parameter(aabb, requires_grad=False) - - self.geo_feat_dim = geo_feat_dim - # TODO: set this properly based on the aabb per_level_scale = 1.4472692012786865 @@ -120,6 +124,7 @@ class TCNNCompoundField(TCNNInstantNGPField): def get_density(self, ray_samples: RaySamples): """Computes and returns the densities.""" + # TODO: add spatial distortion to ngp sampler # positions = self.spatial_distortion(ray_samples.frustums.get_positions()) positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) positions_flat = positions.view(-1, 3) @@ -137,25 +142,36 @@ class TCNNCompoundField(TCNNInstantNGPField): # tcnn requires directions in the range [0,1] if ray_samples.camera_indices is None: raise AttributeError("Camera indices are not provided.") - camera_indices = ray_samples.camera_indices.squeeze().to(ray_samples.frustums.origins.device) - embedded_appearance = self.embedding_appearance(camera_indices) - directions = get_normalized_directions(ray_samples.frustums.directions) directions_flat = directions.view(-1, 3) d = self.direction_encoding(directions_flat) + if self.training: + embedded_appearance = self.embedding_appearance(camera_indices) + else: + embedded_appearance = torch.zeros((*directions.shape[:-1], self.appearance_embedding_dim)).to( + directions.device + ) + if density_embedding is None: positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) h = torch.cat([d, positions.view(-1, 3), embedded_appearance], dim=-1) else: - h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim), embedded_appearance], dim=-1) + h = torch.cat( + [ + d, + density_embedding.view(-1, self.geo_feat_dim), + embedded_appearance.view(-1, self.appearance_embedding_dim), + ], + dim=-1, + ) rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions) return {FieldHeadNames.RGB: rgb} -class TorchCompoundField(NeRFField): +class TorchCompoundField(Field): """ PyTorch implementation of the compound field. """ @@ -163,26 +179,85 @@ class TorchCompoundField(NeRFField): def __init__( self, aabb, + num_images: int, position_encoding: Encoding = HashEncoding(), direction_encoding: Encoding = SHEncoding(), base_mlp_num_layers: int = 3, base_mlp_layer_width: int = 64, head_mlp_num_layers: int = 2, head_mlp_layer_width: int = 32, + appearance_embedding_dim: int = 40, skip_connections: Tuple = (4,), + field_heads: Tuple[FieldHead] = (RGBFieldHead(),), spatial_distortion: SpatialDistortion = SceneContraction(), ) -> None: - super().__init__( - position_encoding, - direction_encoding, - base_mlp_num_layers, - base_mlp_layer_width, - head_mlp_num_layers, - head_mlp_layer_width, - skip_connections, - spatial_distortion=spatial_distortion, - ) + super().__init__() self.aabb = Parameter(aabb, requires_grad=False) + self.spatial_distortion = spatial_distortion + self.num_images = num_images + self.appearance_embedding_dim = appearance_embedding_dim + self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) + + self.position_encoding = position_encoding + self.direction_encoding = direction_encoding + self.spatial_distortion = spatial_distortion + + self.mlp_base = MLP( + in_dim=self.position_encoding.get_out_dim(), + num_layers=base_mlp_num_layers, + layer_width=base_mlp_layer_width, + skip_connections=skip_connections, + ) + + self.mlp_head = MLP( + in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim() + self.appearance_embedding_dim, + num_layers=head_mlp_num_layers, + layer_width=head_mlp_layer_width, + ) + + self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim()) + self.field_heads = nn.ModuleList(field_heads) + for field_head in self.field_heads: + field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore + + def get_density(self, ray_samples: RaySamples): + # TODO: add spatial distortion to ngp sampler + # positions = self.spatial_distortion(ray_samples.frustums.get_positions()) + positions = ray_samples.frustums.get_positions() + encoded_xyz = self.position_encoding(positions) + base_mlp_out = self.mlp_base(encoded_xyz) + density = self.field_output_density(base_mlp_out) + return density, base_mlp_out + + def get_outputs( + self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None + ) -> Dict[FieldHeadNames, TensorType]: + + if ray_samples.camera_indices is None: + raise AttributeError("Camera indices are not provided.") + camera_indices = ray_samples.camera_indices.squeeze().to(ray_samples.frustums.origins.device) + if self.training: + embedded_appearance = self.embedding_appearance(camera_indices) + else: + embedded_appearance = torch.zeros( + (*ray_samples.frustums.directions.shape[:-1], self.appearance_embedding_dim) + ).to(ray_samples.frustums.directions.device) + + outputs = {} + for field_head in self.field_heads: + encoded_dir = self.direction_encoding(ray_samples.frustums.directions) + mlp_out = self.mlp_head( + torch.cat( + [ + encoded_dir, + density_embedding, # type:ignore + embedded_appearance.view(-1, self.appearance_embedding_dim), + ], + dim=-1, # type:ignore + ) + ) + outputs[field_head.field_head_name] = field_head(mlp_out) + return outputs field_implementation_to_class = {"tcnn": TCNNCompoundField, "torch": TorchCompoundField} diff --git a/nerfactory/models/base.py b/nerfactory/models/base.py index 8ec1acca0..4f1a931f2 100644 --- a/nerfactory/models/base.py +++ b/nerfactory/models/base.py @@ -51,11 +51,13 @@ class Model(nn.Module): self, config: cfg.ModelConfig, scene_bounds: SceneBounds, + num_train_data: int, **kwargs, ) -> None: super().__init__() self.config = config self.scene_bounds = scene_bounds + self.num_train_data = num_train_data self.density_field = None self.kwargs = kwargs self.collider = None diff --git a/nerfactory/models/compound.py b/nerfactory/models/compound.py index 488dc7633..087171957 100644 --- a/nerfactory/models/compound.py +++ b/nerfactory/models/compound.py @@ -39,7 +39,7 @@ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity import nerfactory.cuda as nerfactory_cuda from nerfactory.cameras.rays import RayBundle from nerfactory.configs import base as cfg -from nerfactory.fields.instant_ngp_field import field_implementation_to_class +from nerfactory.fields.compound_field import field_implementation_to_class from nerfactory.fields.modules.field_heads import FieldHeadNames from nerfactory.models.base import Model from nerfactory.models.modules.ray_sampler import NGPSpacedSampler @@ -84,7 +84,9 @@ class CompoundModel(Model): """Set the fields and modules.""" super().populate_modules() # torch or tiny-cuda-nn version - self.field = field_implementation_to_class[self.config.field_implementation](self.scene_bounds.aabb) + self.field = field_implementation_to_class[self.config.field_implementation]( + self.scene_bounds.aabb, self.num_train_data + ) # samplers self.sampler = NGPSpacedSampler(num_samples=self.config.num_samples, density_field=self.density_field) @@ -163,14 +165,14 @@ class CompoundModel(Model): return loss_dict def log_test_image_outputs(self, image_idx, step, batch, outputs): - image = batch["image"] + device = self.device + image = batch["image"].to(device) rgb = outputs["rgb"] acc = visualization.apply_colormap(outputs["accumulation"]) depth = visualization.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], ) - combined_rgb = torch.cat([image, rgb], dim=1) combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) diff --git a/nerfactory/models/modules/ray_sampler.py b/nerfactory/models/modules/ray_sampler.py index f24357680..3255c6c3a 100644 --- a/nerfactory/models/modules/ray_sampler.py +++ b/nerfactory/models/modules/ray_sampler.py @@ -430,10 +430,17 @@ class NGPSpacedSampler(Sampler): starts = starts[:total_samples] ends = ends[:total_samples] - # return samples + assert ray_bundle.camera_indices is not None + indices = torch.zeros_like(origins[:, :1], dtype=torch.long) + + ray_bundle.camera_indices = ray_bundle.camera_indices.to(torch.long).to(packed_info.device) + + indices = nerfactory_cuda.get_camera_indices(packed_info, ray_bundle.camera_indices, indices) + zeros = torch.zeros_like(origins[:, :1]) ray_samples = RaySamples( frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros), + camera_indices=indices, ) return ray_samples, packed_info, t_min, t_max diff --git a/nerfactory/pipelines/base.py b/nerfactory/pipelines/base.py index 96c87e8ab..19ca0091d 100644 --- a/nerfactory/pipelines/base.py +++ b/nerfactory/pipelines/base.py @@ -88,7 +88,8 @@ class Pipeline(nn.Module): # TODO(ethan): get rid of scene_bounds from the model assert self.datamanager.train_input_dataset is not None, "Missing input dataset" self.model: Model = config.model.setup( - scene_bounds=self.datamanager.train_input_dataset.dataset_inputs.scene_bounds + scene_bounds=self.datamanager.train_input_dataset.dataset_inputs.scene_bounds, + num_train_data=len(self.datamanager.train_input_dataset), ) self.model.to(device)