Remove point samples

This commit is contained in:
Matt Tancik authored and Matthew Tancik committed 2022-07-01 09:36:59 -07:00
1 parent 191dfc6a34
commit 5a1c8a1c86
13 files changed
+85 -107

No files matched your search

+16 -16
View File
@@ -24,7 +24,7 @@ import torch
from torchtyping import TensorType
from pyrad.fields.modules.field_heads import FieldHeadNames
from pyrad.cameras.rays import Frustums, PointSamples
from pyrad.cameras.rays import Frustums, RaySamples
from pyrad.utils.misc import is_not_none
@@ -34,7 +34,7 @@ class Field(nn.Module):
def density_fn(self, positions):
"""Returns only the density. Used primarily with the occupancy grid."""
# Need to figure out a better way to descibe positions with a ray.
point_samples = PointSamples(
ray_samples = RaySamples(
frustums=Frustums(
origins=positions,
directions=torch.ones_like(positions),
@@ -43,15 +43,15 @@ class Field(nn.Module):
pixel_area=torch.ones_like((positions[..., :1])),
)
)
density, _ = self.get_density(point_samples)
density, _ = self.get_density(ray_samples)
return density
@abstractmethod
def get_density(self, point_samples: PointSamples) -> Tuple[TensorType[..., 1], TensorType[..., "num_features"]]:
def get_density(self, ray_samples: RaySamples) -> Tuple[TensorType[..., 1], TensorType[..., "num_features"]]:
"""Computes and returns the densities.
Args:
point_samples (PointSamples): Samples locations to compute density.
ray_samples (RaySamples): Samples locations to compute density.
Returns:
Tuple[TensorType[...,1], TensorType[...,"num_features"]]: A tensor of densities and a tensor of features.
@@ -59,34 +59,34 @@ class Field(nn.Module):
@abstractmethod
def get_outputs(
self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
) -> Dict[FieldHeadNames, TensorType]:
"""Computes and returns the colors.
Args:
point_samples (PointSamples): Samples locations to compute outputs.
ray_samples (RaySamples): Samples locations to compute outputs.
density_embedding (TensorType, optional): Density embeddings to condition on.
Returns:
Dict[FieldHeadNames, TensorType]: Output field values.
"""
def forward(self, point_samples: PointSamples):
def forward(self, ray_samples: RaySamples):
"""Evaluates the field at points along the ray.
Args:
point_samples (PointSamples): Samples to evaluate field on.
ray_samples (RaySamples): Samples to evaluate field on.
"""
valid_mask = point_samples.valid_mask
valid_mask = ray_samples.valid_mask
if is_not_none(valid_mask):
# Hacky handling of empty masks. Tests on a single ray but doesn't use results
if not valid_mask.any():
point_samples = PointSamples(frustums=Frustums.get_mock_frustum())
ray_samples = RaySamples(frustums=Frustums.get_mock_frustum())
else:
point_samples = point_samples.apply_masks()
density_masked, density_embedding_masked = self.get_density(point_samples)
field_outputs_masked = self.get_outputs(point_samples, density_embedding=density_embedding_masked)
ray_samples = ray_samples.apply_masks()
density_masked, density_embedding_masked = self.get_density(ray_samples)
field_outputs_masked = self.get_outputs(ray_samples, density_embedding=density_embedding_masked)
field_outputs = {}
for k, value in field_outputs_masked.items():
@@ -100,8 +100,8 @@ class Field(nn.Module):
if valid_mask.any():
density[valid_mask] = density_masked
else:
density, density_embedding = self.get_density(point_samples)
field_outputs = self.get_outputs(point_samples, density_embedding=density_embedding)
density, density_embedding = self.get_density(ray_samples)
field_outputs = self.get_outputs(ray_samples, density_embedding=density_embedding)
field_outputs[FieldHeadNames.DENSITY] = density
return field_outputs
+12 -12
View File
@@ -27,7 +27,7 @@ from pyrad.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
from pyrad.fields.modules.field_heads import FieldHeadNames
from pyrad.fields.base import Field
from pyrad.fields.nerf_field import NeRFField
from pyrad.cameras.rays import PointSamples
from pyrad.cameras.rays import RaySamples
try:
import tinycudann as tcnn
@@ -107,26 +107,26 @@ class TCNNInstantNGPField(Field):
},
)
def get_density(self, point_samples: PointSamples):
def get_density(self, ray_samples: RaySamples):
"""Computes and returns the densities."""
positions = get_normalized_positions(point_samples.frustums.get_positions(), self.aabb)
positions = get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
positions_flat = positions.view(-1, 3)
dtype = positions_flat.dtype
x = self.position_encoding(positions_flat)
h = self.mlp_base(x).view(*point_samples.frustums.get_positions().shape[:-1], -1).to(dtype)
h = self.mlp_base(x).view(*ray_samples.frustums.get_positions().shape[:-1], -1).to(dtype)
density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1)
density = F.softplus(density_before_activation)
return density, base_mlp_out
def get_outputs(self, point_samples: PointSamples, density_embedding=None):
def get_outputs(self, ray_samples: RaySamples, density_embedding=None):
# TODO: add valid_mask masking!
# tcnn requires directions in the range [0,1]
directions = get_normalized_directions(point_samples.frustums.directions)
directions = get_normalized_directions(ray_samples.frustums.directions)
directions_flat = directions.view(-1, 3)
dtype = directions_flat.dtype
d = self.direction_encoding(directions_flat)
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
h = self.mlp_head(h).view(*point_samples.frustums.directions.shape[:-1], -1).to(dtype)
h = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(dtype)
rgb = torch.sigmoid(h)
return {FieldHeadNames.RGB: rgb}
@@ -158,12 +158,12 @@ class TorchInstantNGPField(NeRFField):
)
self.aabb = Parameter(aabb, requires_grad=False)
def get_density(self, point_samples: PointSamples):
normalized_point_samples = point_samples
normalized_point_samples.positions = get_normalized_positions(
normalized_point_samples.frustums.get_positions(), self.aabb
def get_density(self, ray_samples: RaySamples):
normalized_ray_samples = ray_samples
normalized_ray_samples.positions = get_normalized_positions(
normalized_ray_samples.frustums.get_positions(), self.aabb
)
return super().get_density(normalized_point_samples)
return super().get_density(normalized_ray_samples)
field_implementation_to_class = {"tcnn": TCNNInstantNGPField, "torch": TorchInstantNGPField}
+6 -6
View File
@@ -25,7 +25,7 @@ from pyrad.fields.modules.encoding import Encoding, Identity
from pyrad.fields.modules.field_heads import DensityFieldHead, FieldHead, FieldHeadNames, RGBFieldHead
from pyrad.fields.modules.mlp import MLP
from pyrad.fields.base import Field
from pyrad.cameras.rays import PointSamples
from pyrad.cameras.rays import RaySamples
from pyrad.fields.modules.spatial_distortions import SpatialDistortion
@@ -81,14 +81,14 @@ class NeRFField(Field):
for field_head in self.field_heads:
field_head.set_in_dim(self.mlp_head.get_out_dim())
def get_density(self, point_samples: PointSamples):
def get_density(self, ray_samples: RaySamples):
if self.use_integrated_encoding:
gaussian_samples = point_samples.frustums.get_gaussian_blob()
gaussian_samples = ray_samples.frustums.get_gaussian_blob()
if self.spatial_distortion is not None:
gaussian_samples = self.spatial_distortion(gaussian_samples)
encoded_xyz = self.position_encoding(gaussian_samples.mean, covs=gaussian_samples.cov)
else:
positions = point_samples.frustums.get_positions()
positions = ray_samples.frustums.get_positions()
if self.spatial_distortion is not None:
positions = self.spatial_distortion(positions)
encoded_xyz = self.position_encoding(positions)
@@ -97,11 +97,11 @@ class NeRFField(Field):
return density, base_mlp_out
def get_outputs(
self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
) -> Dict[FieldHeadNames, TensorType]:
outputs = {}
for field_head in self.field_heads:
encoded_dir = self.direction_encoding(point_samples.frustums.directions)
encoded_dir = self.direction_encoding(ray_samples.frustums.directions)
mlp_out = self.mlp_head(torch.cat([encoded_dir, density_embedding], dim=-1))
outputs[field_head.field_head_name] = field_head(mlp_out)
return outputs
+7 -7
View File
@@ -32,7 +32,7 @@ from pyrad.fields.modules.field_heads import (
)
from pyrad.fields.modules.mlp import MLP
from pyrad.fields.base import Field
from pyrad.cameras.rays import PointSamples
from pyrad.cameras.rays import RaySamples
class VanillaNerfWField(Field):
@@ -106,22 +106,22 @@ class VanillaNerfWField(Field):
self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim())
self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim())
def get_density(self, point_samples: PointSamples):
def get_density(self, ray_samples: RaySamples):
"""Computes and returns the densities."""
encoded_xyz = self.position_encoding(point_samples.frustums.positions())
encoded_xyz = self.position_encoding(ray_samples.frustums.positions())
base_mlp_out = self.mlp_base(encoded_xyz)
density = self.field_head_density(base_mlp_out)
return density, base_mlp_out
def get_outputs(
self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
) -> Dict[FieldHeadNames, TensorType]:
outputs = {}
encoded_dir = self.direction_encoding(point_samples.directions)
embedded_appearance = self.embedding_appearance(point_samples.camera_indices)
encoded_dir = self.direction_encoding(ray_samples.frustums.directions)
embedded_appearance = self.embedding_appearance(ray_samples.camera_indices)
mlp_head_out = self.mlp_head(torch.cat([density_embedding, encoded_dir, embedded_appearance], dim=-1))
outputs[self.field_head_rgb.field_head_name] = self.field_head_rgb(mlp_head_out) # static rgb
embedded_transient = self.embedding_transient(point_samples.camera_indices)
embedded_transient = self.embedding_transient(ray_samples.camera_indices)
transient_mlp_out = self.mlp_transient(torch.cat([density_embedding, embedded_transient], dim=-1))
outputs[self.field_head_transient_uncertainty.field_head_name] = self.field_head_transient_uncertainty(
transient_mlp_out