compound model (wip) (#427)

* compound model now uses compound field, passing num_images in the config, working on passing image indices through ngp sampler

* renaming cuda func, adding torch compound support

* removing unused imports

* getting num images from dataloader instead of config

* fixing pylint and adding todo

* undoing base config change

* adding cone angle and near plane to compound model config
This commit is contained in:
Terrance Wang authored and GitHub committed 2022-09-16 14:38:25 -07:00
1 parent 2495cfe1ee
commit 84024a4e73
10 files changed
+170 -31

No files matched your search

+100 -25
View File
@@ -17,23 +17,29 @@ Field for compound nerf model, adds scene contraction and image embeddings to in
"""
from typing import Optional, Tuple
from typing import Dict, Optional, Tuple
import torch
from torch import nn
from torch.nn.parameter import Parameter
from torchtyping import TensorType
from nerfactory.cameras.rays import RaySamples
from nerfactory.datamanagers.structs import SceneBounds
from nerfactory.fields.instant_ngp_field import TCNNInstantNGPField
from nerfactory.fields.base import Field
from nerfactory.fields.modules.embedding import Embedding
from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.fields.modules.field_heads import (
DensityFieldHead,
FieldHead,
FieldHeadNames,
RGBFieldHead,
)
from nerfactory.fields.modules.mlp import MLP
from nerfactory.fields.modules.spatial_distortions import (
SceneContraction,
SpatialDistortion,
)
from nerfactory.fields.nerf_field import NeRFField
from nerfactory.utils.activations import trunc_exp
try:
@@ -48,7 +54,7 @@ def get_normalized_directions(directions):
return (directions + 1.0) / 2.0
class TCNNCompoundField(TCNNInstantNGPField):
class TCNNCompoundField(Field):
"""Compound Field that uses TCNN"""
def __init__(
@@ -63,7 +69,9 @@ class TCNNCompoundField(TCNNInstantNGPField):
appearance_embedding_dim: int = 40,
spatial_distortion: SpatialDistortion = SceneContraction(),
) -> None:
super().__init__(self, aabb)
super().__init__()
self.aabb = Parameter(aabb, requires_grad=False)
self.geo_feat_dim = geo_feat_dim
self.spatial_distortion = spatial_distortion
@@ -71,10 +79,6 @@ class TCNNCompoundField(TCNNInstantNGPField):
self.appearance_embedding_dim = appearance_embedding_dim
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
self.aabb = Parameter(aabb, requires_grad=False)
self.geo_feat_dim = geo_feat_dim
# TODO: set this properly based on the aabb
per_level_scale = 1.4472692012786865
@@ -120,6 +124,7 @@ class TCNNCompoundField(TCNNInstantNGPField):
def get_density(self, ray_samples: RaySamples):
"""Computes and returns the densities."""
# TODO: add spatial distortion to ngp sampler
# positions = self.spatial_distortion(ray_samples.frustums.get_positions())
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
positions_flat = positions.view(-1, 3)
@@ -137,25 +142,36 @@ class TCNNCompoundField(TCNNInstantNGPField):
# tcnn requires directions in the range [0,1]
if ray_samples.camera_indices is None:
raise AttributeError("Camera indices are not provided.")
camera_indices = ray_samples.camera_indices.squeeze().to(ray_samples.frustums.origins.device)
embedded_appearance = self.embedding_appearance(camera_indices)
directions = get_normalized_directions(ray_samples.frustums.directions)
directions_flat = directions.view(-1, 3)
d = self.direction_encoding(directions_flat)
if self.training:
embedded_appearance = self.embedding_appearance(camera_indices)
else:
embedded_appearance = torch.zeros((*directions.shape[:-1], self.appearance_embedding_dim)).to(
directions.device
)
if density_embedding is None:
positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
h = torch.cat([d, positions.view(-1, 3), embedded_appearance], dim=-1)
else:
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim), embedded_appearance], dim=-1)
h = torch.cat(
[
d,
density_embedding.view(-1, self.geo_feat_dim),
embedded_appearance.view(-1, self.appearance_embedding_dim),
],
dim=-1,
)
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
return {FieldHeadNames.RGB: rgb}
class TorchCompoundField(NeRFField):
class TorchCompoundField(Field):
"""
PyTorch implementation of the compound field.
"""
@@ -163,26 +179,85 @@ class TorchCompoundField(NeRFField):
def __init__(
self,
aabb,
num_images: int,
position_encoding: Encoding = HashEncoding(),
direction_encoding: Encoding = SHEncoding(),
base_mlp_num_layers: int = 3,
base_mlp_layer_width: int = 64,
head_mlp_num_layers: int = 2,
head_mlp_layer_width: int = 32,
appearance_embedding_dim: int = 40,
skip_connections: Tuple = (4,),
field_heads: Tuple[FieldHead] = (RGBFieldHead(),),
spatial_distortion: SpatialDistortion = SceneContraction(),
) -> None:
super().__init__(
position_encoding,
direction_encoding,
base_mlp_num_layers,
base_mlp_layer_width,
head_mlp_num_layers,
head_mlp_layer_width,
skip_connections,
spatial_distortion=spatial_distortion,
)
super().__init__()
self.aabb = Parameter(aabb, requires_grad=False)
self.spatial_distortion = spatial_distortion
self.num_images = num_images
self.appearance_embedding_dim = appearance_embedding_dim
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
self.position_encoding = position_encoding
self.direction_encoding = direction_encoding
self.spatial_distortion = spatial_distortion
self.mlp_base = MLP(
in_dim=self.position_encoding.get_out_dim(),
num_layers=base_mlp_num_layers,
layer_width=base_mlp_layer_width,
skip_connections=skip_connections,
)
self.mlp_head = MLP(
in_dim=self.mlp_base.get_out_dim() + self.direction_encoding.get_out_dim() + self.appearance_embedding_dim,
num_layers=head_mlp_num_layers,
layer_width=head_mlp_layer_width,
)
self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim())
self.field_heads = nn.ModuleList(field_heads)
for field_head in self.field_heads:
field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore
def get_density(self, ray_samples: RaySamples):
# TODO: add spatial distortion to ngp sampler
# positions = self.spatial_distortion(ray_samples.frustums.get_positions())
positions = ray_samples.frustums.get_positions()
encoded_xyz = self.position_encoding(positions)
base_mlp_out = self.mlp_base(encoded_xyz)
density = self.field_output_density(base_mlp_out)
return density, base_mlp_out
def get_outputs(
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
) -> Dict[FieldHeadNames, TensorType]:
if ray_samples.camera_indices is None:
raise AttributeError("Camera indices are not provided.")
camera_indices = ray_samples.camera_indices.squeeze().to(ray_samples.frustums.origins.device)
if self.training:
embedded_appearance = self.embedding_appearance(camera_indices)
else:
embedded_appearance = torch.zeros(
(*ray_samples.frustums.directions.shape[:-1], self.appearance_embedding_dim)
).to(ray_samples.frustums.directions.device)
outputs = {}
for field_head in self.field_heads:
encoded_dir = self.direction_encoding(ray_samples.frustums.directions)
mlp_out = self.mlp_head(
torch.cat(
[
encoded_dir,
density_embedding, # type:ignore
embedded_appearance.view(-1, self.appearance_embedding_dim),
],
dim=-1, # type:ignore
)
)
outputs[field_head.field_head_name] = field_head(mlp_out)
return outputs
field_implementation_to_class = {"tcnn": TCNNCompoundField, "torch": TorchCompoundField}