:wqMerge branch 'master' of github.com:ethanweber/mattport into HEAD

This commit is contained in:
evonneng committed 2022-04-09 05:14:05 +00:00
commit 9fc60d61a1
4 files changed
+164 -6

No files matched your search

+51
View File
@@ -0,0 +1,51 @@
"""
Collection of render heads
"""
from typing import Optional
from torch import nn
from torchtyping import TensorType
class RenderHead(nn.Module):
"""Base Render head"""
def __init__(self, in_dim: int, out_dim: int, activation: Optional[nn.Module]) -> None:
"""
Args:
in_dim (int): input dimension
out_dim (int): output dimension for renderer
activation (Optional[nn.Module]): output head activation
"""
super().__init__()
layers = [nn.Linear(in_dim, out_dim)]
if activation:
layers.append(activation)
self.net = nn.Sequential(*layers)
def forward(self, in_tensor: TensorType[..., "in_dim"]) -> TensorType[..., "out_dim"]:
"""Process network output for renderer
Args:
in_tensor (TensorType[..., "in_dim]): Network input
Returns:
TensorType[..., "out_dim"]: Render head output
"""
return self.net(in_tensor)
class DensityHead(RenderHead):
"""Density head"""
def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus) -> None:
super().__init__(in_dim, 1, activation)
class RGBHead(RenderHead):
"""RGB head"""
def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Sigmoid) -> None:
super().__init__(in_dim, 3, activation)
View File
Whitespace-only changes.
+94
View File
@@ -0,0 +1,94 @@
"""
Collection of renderers
"""
from typing import Dict, Set
import torch
from torch import nn
from torchtyping import TensorType
from mattport.nerf import render_heads as rh
class Renderer(nn.Module):
"""Base Renderer. Intended to be subclassed"""
def required_render_heads(self) -> Set[rh.RenderHead]:
"""
Returns:
Set[RenderHead]: RenderHeads required for this renderer.
"""
raise NotImplementedError
def forward(
self,
render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]],
deltas: TensorType[..., "num_samples"],
) -> TensorType[..., "out_dim"]:
"""Composite samples along ray and render image
Args:
in_tensor (TensorType[..., "num_samples", "features"]): feature for each sample
deltas (TensorType[..., "num_samples"]): depth of each sample
Returns:
TensorType[..., "out_dim"]: Rendered ray
"""
raise NotImplementedError
class RGB(Renderer):
"""Standard volumetic rendering."""
def __init__(self, rgb_name, density_name) -> None:
super().__init__()
self.rgb_name = rgb_name
self.density_name = density_name
def required_render_heads(self) -> Set[rh.RenderHead]:
return set(rh.RGBHead, rh.DensityHead)
def forward(
self,
render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]],
deltas: TensorType[..., "num_samples"],
) -> TensorType[..., "out_dim"]:
"""Composite samples along ray and render color image
Args:
in_tensor (TensorType[..., "num_samples", 3]): RGB for each sample
deltas (TensorType[..., "num_samples"]): depth influence for each sample
Returns:
TensorType[..., "out_dim"]: Composited RGB ray
"""
delta_density = deltas * render_head_outputs[self.density_name]
alphas = 1 - torch.exp(-delta_density)
transmittance = torch.cat(0, torch.cumsum(delta_density[..., :-1], dim=-1), dim=-1)
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
weights = alphas * transmittance # [..., "num_samples"]
rgb = torch.sum(weights[..., None] * render_head_outputs[self.rgb_name], dim=-2)
return rgb
class Depth(Renderer):
"""_summary_
Args:
Renderer (_type_): _description_
"""
def required_render_heads(self) -> Set[rh.RenderHead]:
return set(rh.DensityHead)
def forward(
self,
render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]],
deltas: TensorType[..., "num_samples"],
) -> TensorType[..., "out_dim"]:
raise NotImplementedError
+19 -6
View File
@@ -1,12 +1,18 @@
"""
Collection of sampling strategies
"""
from typing import NamedTuple
from torchtyping import TensorType
from torch import nn
import torch
import nerf.cameras as cameras
RaySamples = NamedTuple(
"RaySamples", [("locations", TensorType[..., "num_samples", 3]), ("deltas", TensorType[..., "num_samples"])]
)
class Sampler(nn.Module):
"""Base Sampler. Intended to be subclassed"""
@@ -26,17 +32,18 @@ class Sampler(nn.Module):
def generate_samples(
self, camera_rays: cameras.Rays, near_plane: float, far_plane: float, num_samples: int
) -> TensorType[..., "num_samples", 3]:
) -> RaySamples:
"""Encodes an input tensor.
Args:
camera_rays (cameras.Rays): Rays to generate samples for
Returns:
TensorType[..., "output_dim"]: A encoded input tensor
near_plane (float): Minimum distance along ray to sample
far_plane (float): Maximum distance along ray to sample
num_samples (int): Number of samples per ray
Returns:
RaySamples: Positions and deltas for samples along a ray
"""
raise NotImplementedError
@@ -55,15 +62,21 @@ class UniformSampler(Sampler):
Args:
camera_rays (cameras.Rays): Rays to generate samples for
Returns:
TensorType[..., "output_dim"]: A encoded input tensor
near_plane (float): Minimum distance along ray to sample
far_plane (float): Maximum distance along ray to sample
num_samples (int): Number of samples per ray
Returns:
RaySamples: Positions and deltas for samples along a ray
"""
near_plane = near_plane or self.near_plane
far_plane = far_plane or self.far_plane
num_samples = num_samples or self.num_samples
return torch.linspace(near_plane, far_plane, num_samples)
bins = torch.linspace(near_plane, far_plane, num_samples + 1)
positions = (bins[..., 1:] + bins[..., :-1]) / 2.0
deltas = bins[..., 1:] - bins[..., :-1]
return RaySamples(locations=positions, deltas=deltas)