mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 07:05:44 +08:00
Remove point samples
This commit is contained in:
1 parent
191dfc6a34
commit
5a1c8a1c86
13 files changed
+85
-107
No files matched your search
+16
-16
@@ -24,7 +24,7 @@ import torch
|
||||
from torchtyping import TensorType
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
|
||||
from pyrad.cameras.rays import Frustums, PointSamples
|
||||
from pyrad.cameras.rays import Frustums, RaySamples
|
||||
from pyrad.utils.misc import is_not_none
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class Field(nn.Module):
|
||||
def density_fn(self, positions):
|
||||
"""Returns only the density. Used primarily with the occupancy grid."""
|
||||
# Need to figure out a better way to descibe positions with a ray.
|
||||
point_samples = PointSamples(
|
||||
ray_samples = RaySamples(
|
||||
frustums=Frustums(
|
||||
origins=positions,
|
||||
directions=torch.ones_like(positions),
|
||||
@@ -43,15 +43,15 @@ class Field(nn.Module):
|
||||
pixel_area=torch.ones_like((positions[..., :1])),
|
||||
)
|
||||
)
|
||||
density, _ = self.get_density(point_samples)
|
||||
density, _ = self.get_density(ray_samples)
|
||||
return density
|
||||
|
||||
@abstractmethod
|
||||
def get_density(self, point_samples: PointSamples) -> Tuple[TensorType[..., 1], TensorType[..., "num_features"]]:
|
||||
def get_density(self, ray_samples: RaySamples) -> Tuple[TensorType[..., 1], TensorType[..., "num_features"]]:
|
||||
"""Computes and returns the densities.
|
||||
|
||||
Args:
|
||||
point_samples (PointSamples): Samples locations to compute density.
|
||||
ray_samples (RaySamples): Samples locations to compute density.
|
||||
|
||||
Returns:
|
||||
Tuple[TensorType[...,1], TensorType[...,"num_features"]]: A tensor of densities and a tensor of features.
|
||||
@@ -59,34 +59,34 @@ class Field(nn.Module):
|
||||
|
||||
@abstractmethod
|
||||
def get_outputs(
|
||||
self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None
|
||||
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
|
||||
) -> Dict[FieldHeadNames, TensorType]:
|
||||
"""Computes and returns the colors.
|
||||
|
||||
Args:
|
||||
point_samples (PointSamples): Samples locations to compute outputs.
|
||||
ray_samples (RaySamples): Samples locations to compute outputs.
|
||||
density_embedding (TensorType, optional): Density embeddings to condition on.
|
||||
|
||||
Returns:
|
||||
Dict[FieldHeadNames, TensorType]: Output field values.
|
||||
"""
|
||||
|
||||
def forward(self, point_samples: PointSamples):
|
||||
def forward(self, ray_samples: RaySamples):
|
||||
"""Evaluates the field at points along the ray.
|
||||
|
||||
Args:
|
||||
point_samples (PointSamples): Samples to evaluate field on.
|
||||
ray_samples (RaySamples): Samples to evaluate field on.
|
||||
"""
|
||||
valid_mask = point_samples.valid_mask
|
||||
valid_mask = ray_samples.valid_mask
|
||||
|
||||
if is_not_none(valid_mask):
|
||||
# Hacky handling of empty masks. Tests on a single ray but doesn't use results
|
||||
if not valid_mask.any():
|
||||
point_samples = PointSamples(frustums=Frustums.get_mock_frustum())
|
||||
ray_samples = RaySamples(frustums=Frustums.get_mock_frustum())
|
||||
else:
|
||||
point_samples = point_samples.apply_masks()
|
||||
density_masked, density_embedding_masked = self.get_density(point_samples)
|
||||
field_outputs_masked = self.get_outputs(point_samples, density_embedding=density_embedding_masked)
|
||||
ray_samples = ray_samples.apply_masks()
|
||||
density_masked, density_embedding_masked = self.get_density(ray_samples)
|
||||
field_outputs_masked = self.get_outputs(ray_samples, density_embedding=density_embedding_masked)
|
||||
|
||||
field_outputs = {}
|
||||
for k, value in field_outputs_masked.items():
|
||||
@@ -100,8 +100,8 @@ class Field(nn.Module):
|
||||
if valid_mask.any():
|
||||
density[valid_mask] = density_masked
|
||||
else:
|
||||
density, density_embedding = self.get_density(point_samples)
|
||||
field_outputs = self.get_outputs(point_samples, density_embedding=density_embedding)
|
||||
density, density_embedding = self.get_density(ray_samples)
|
||||
field_outputs = self.get_outputs(ray_samples, density_embedding=density_embedding)
|
||||
|
||||
field_outputs[FieldHeadNames.DENSITY] = density
|
||||
return field_outputs
|
||||
@@ -27,7 +27,7 @@ from pyrad.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
from pyrad.fields.base import Field
|
||||
from pyrad.fields.nerf_field import NeRFField
|
||||
from pyrad.cameras.rays import PointSamples
|
||||
from pyrad.cameras.rays import RaySamples
|
||||
|
||||
try:
|
||||
import tinycudann as tcnn
|
||||
@@ -107,26 +107,26 @@ class TCNNInstantNGPField(Field):
|
||||
},
|
||||
)
|
||||
|
||||
def get_density(self, point_samples: PointSamples):
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
"""Computes and returns the densities."""
|
||||
positions = get_normalized_positions(point_samples.frustums.get_positions(), self.aabb)
|
||||
positions = get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
positions_flat = positions.view(-1, 3)
|
||||
dtype = positions_flat.dtype
|
||||
x = self.position_encoding(positions_flat)
|
||||
h = self.mlp_base(x).view(*point_samples.frustums.get_positions().shape[:-1], -1).to(dtype)
|
||||
h = self.mlp_base(x).view(*ray_samples.frustums.get_positions().shape[:-1], -1).to(dtype)
|
||||
density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1)
|
||||
density = F.softplus(density_before_activation)
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(self, point_samples: PointSamples, density_embedding=None):
|
||||
def get_outputs(self, ray_samples: RaySamples, density_embedding=None):
|
||||
# TODO: add valid_mask masking!
|
||||
# tcnn requires directions in the range [0,1]
|
||||
directions = get_normalized_directions(point_samples.frustums.directions)
|
||||
directions = get_normalized_directions(ray_samples.frustums.directions)
|
||||
directions_flat = directions.view(-1, 3)
|
||||
dtype = directions_flat.dtype
|
||||
d = self.direction_encoding(directions_flat)
|
||||
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
|
||||
h = self.mlp_head(h).view(*point_samples.frustums.directions.shape[:-1], -1).to(dtype)
|
||||
h = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(dtype)
|
||||
rgb = torch.sigmoid(h)
|
||||
return {FieldHeadNames.RGB: rgb}
|
||||
|
||||
@@ -158,12 +158,12 @@ class TorchInstantNGPField(NeRFField):
|
||||
)
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
|
||||
def get_density(self, point_samples: PointSamples):
|
||||
normalized_point_samples = point_samples
|
||||
normalized_point_samples.positions = get_normalized_positions(
|
||||
normalized_point_samples.frustums.get_positions(), self.aabb
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
normalized_ray_samples = ray_samples
|
||||
normalized_ray_samples.positions = get_normalized_positions(
|
||||
normalized_ray_samples.frustums.get_positions(), self.aabb
|
||||
)
|
||||
return super().get_density(normalized_point_samples)
|
||||
return super().get_density(normalized_ray_samples)
|
||||
|
||||
|
||||
field_implementation_to_class = {"tcnn": TCNNInstantNGPField, "torch": TorchInstantNGPField}
|
||||
@@ -25,7 +25,7 @@ from pyrad.fields.modules.encoding import Encoding, Identity
|
||||
from pyrad.fields.modules.field_heads import DensityFieldHead, FieldHead, FieldHeadNames, RGBFieldHead
|
||||
from pyrad.fields.modules.mlp import MLP
|
||||
from pyrad.fields.base import Field
|
||||
from pyrad.cameras.rays import PointSamples
|
||||
from pyrad.cameras.rays import RaySamples
|
||||
from pyrad.fields.modules.spatial_distortions import SpatialDistortion
|
||||
|
||||
|
||||
@@ -81,14 +81,14 @@ class NeRFField(Field):
|
||||
for field_head in self.field_heads:
|
||||
field_head.set_in_dim(self.mlp_head.get_out_dim())
|
||||
|
||||
def get_density(self, point_samples: PointSamples):
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
if self.use_integrated_encoding:
|
||||
gaussian_samples = point_samples.frustums.get_gaussian_blob()
|
||||
gaussian_samples = ray_samples.frustums.get_gaussian_blob()
|
||||
if self.spatial_distortion is not None:
|
||||
gaussian_samples = self.spatial_distortion(gaussian_samples)
|
||||
encoded_xyz = self.position_encoding(gaussian_samples.mean, covs=gaussian_samples.cov)
|
||||
else:
|
||||
positions = point_samples.frustums.get_positions()
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
if self.spatial_distortion is not None:
|
||||
positions = self.spatial_distortion(positions)
|
||||
encoded_xyz = self.position_encoding(positions)
|
||||
@@ -97,11 +97,11 @@ class NeRFField(Field):
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(
|
||||
self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None
|
||||
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
|
||||
) -> Dict[FieldHeadNames, TensorType]:
|
||||
outputs = {}
|
||||
for field_head in self.field_heads:
|
||||
encoded_dir = self.direction_encoding(point_samples.frustums.directions)
|
||||
encoded_dir = self.direction_encoding(ray_samples.frustums.directions)
|
||||
mlp_out = self.mlp_head(torch.cat([encoded_dir, density_embedding], dim=-1))
|
||||
outputs[field_head.field_head_name] = field_head(mlp_out)
|
||||
return outputs
|
||||
@@ -32,7 +32,7 @@ from pyrad.fields.modules.field_heads import (
|
||||
)
|
||||
from pyrad.fields.modules.mlp import MLP
|
||||
from pyrad.fields.base import Field
|
||||
from pyrad.cameras.rays import PointSamples
|
||||
from pyrad.cameras.rays import RaySamples
|
||||
|
||||
|
||||
class VanillaNerfWField(Field):
|
||||
@@ -106,22 +106,22 @@ class VanillaNerfWField(Field):
|
||||
self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim())
|
||||
self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim())
|
||||
|
||||
def get_density(self, point_samples: PointSamples):
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
"""Computes and returns the densities."""
|
||||
encoded_xyz = self.position_encoding(point_samples.frustums.positions())
|
||||
encoded_xyz = self.position_encoding(ray_samples.frustums.positions())
|
||||
base_mlp_out = self.mlp_base(encoded_xyz)
|
||||
density = self.field_head_density(base_mlp_out)
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(
|
||||
self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None
|
||||
self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None
|
||||
) -> Dict[FieldHeadNames, TensorType]:
|
||||
outputs = {}
|
||||
encoded_dir = self.direction_encoding(point_samples.directions)
|
||||
embedded_appearance = self.embedding_appearance(point_samples.camera_indices)
|
||||
encoded_dir = self.direction_encoding(ray_samples.frustums.directions)
|
||||
embedded_appearance = self.embedding_appearance(ray_samples.camera_indices)
|
||||
mlp_head_out = self.mlp_head(torch.cat([density_embedding, encoded_dir, embedded_appearance], dim=-1))
|
||||
outputs[self.field_head_rgb.field_head_name] = self.field_head_rgb(mlp_head_out) # static rgb
|
||||
embedded_transient = self.embedding_transient(point_samples.camera_indices)
|
||||
embedded_transient = self.embedding_transient(ray_samples.camera_indices)
|
||||
transient_mlp_out = self.mlp_transient(torch.cat([density_embedding, embedded_transient], dim=-1))
|
||||
outputs[self.field_head_transient_uncertainty.field_head_name] = self.field_head_transient_uncertainty(
|
||||
transient_mlp_out
|
||||
|
||||
Reference in new issue
Block a user