mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 21:03:46 +08:00
:wqMerge branch 'master' of github.com:ethanweber/mattport into HEAD
This commit is contained in:
commit
9fc60d61a1
4 files changed
+164
-6
No files matched your search
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
Collection of render heads
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
|
||||
class RenderHead(nn.Module):
|
||||
"""Base Render head"""
|
||||
|
||||
def __init__(self, in_dim: int, out_dim: int, activation: Optional[nn.Module]) -> None:
|
||||
"""
|
||||
|
||||
Args:
|
||||
in_dim (int): input dimension
|
||||
out_dim (int): output dimension for renderer
|
||||
activation (Optional[nn.Module]): output head activation
|
||||
"""
|
||||
super().__init__()
|
||||
layers = [nn.Linear(in_dim, out_dim)]
|
||||
if activation:
|
||||
layers.append(activation)
|
||||
self.net = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, in_tensor: TensorType[..., "in_dim"]) -> TensorType[..., "out_dim"]:
|
||||
"""Process network output for renderer
|
||||
|
||||
Args:
|
||||
in_tensor (TensorType[..., "in_dim]): Network input
|
||||
|
||||
Returns:
|
||||
TensorType[..., "out_dim"]: Render head output
|
||||
"""
|
||||
|
||||
return self.net(in_tensor)
|
||||
|
||||
|
||||
class DensityHead(RenderHead):
|
||||
"""Density head"""
|
||||
|
||||
def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus) -> None:
|
||||
super().__init__(in_dim, 1, activation)
|
||||
|
||||
|
||||
class RGBHead(RenderHead):
|
||||
"""RGB head"""
|
||||
|
||||
def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Sigmoid) -> None:
|
||||
super().__init__(in_dim, 3, activation)
|
||||
Whitespace-only changes.
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
Collection of renderers
|
||||
"""
|
||||
from typing import Dict, Set
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
from mattport.nerf import render_heads as rh
|
||||
|
||||
|
||||
class Renderer(nn.Module):
|
||||
"""Base Renderer. Intended to be subclassed"""
|
||||
|
||||
def required_render_heads(self) -> Set[rh.RenderHead]:
|
||||
"""
|
||||
Returns:
|
||||
Set[RenderHead]: RenderHeads required for this renderer.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def forward(
|
||||
self,
|
||||
render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]],
|
||||
deltas: TensorType[..., "num_samples"],
|
||||
) -> TensorType[..., "out_dim"]:
|
||||
"""Composite samples along ray and render image
|
||||
|
||||
Args:
|
||||
in_tensor (TensorType[..., "num_samples", "features"]): feature for each sample
|
||||
deltas (TensorType[..., "num_samples"]): depth of each sample
|
||||
|
||||
Returns:
|
||||
TensorType[..., "out_dim"]: Rendered ray
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RGB(Renderer):
|
||||
"""Standard volumetic rendering."""
|
||||
|
||||
def __init__(self, rgb_name, density_name) -> None:
|
||||
super().__init__()
|
||||
self.rgb_name = rgb_name
|
||||
self.density_name = density_name
|
||||
|
||||
def required_render_heads(self) -> Set[rh.RenderHead]:
|
||||
return set(rh.RGBHead, rh.DensityHead)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]],
|
||||
deltas: TensorType[..., "num_samples"],
|
||||
) -> TensorType[..., "out_dim"]:
|
||||
"""Composite samples along ray and render color image
|
||||
|
||||
Args:
|
||||
in_tensor (TensorType[..., "num_samples", 3]): RGB for each sample
|
||||
deltas (TensorType[..., "num_samples"]): depth influence for each sample
|
||||
|
||||
Returns:
|
||||
TensorType[..., "out_dim"]: Composited RGB ray
|
||||
"""
|
||||
|
||||
delta_density = deltas * render_head_outputs[self.density_name]
|
||||
alphas = 1 - torch.exp(-delta_density)
|
||||
|
||||
transmittance = torch.cat(0, torch.cumsum(delta_density[..., :-1], dim=-1), dim=-1)
|
||||
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
|
||||
|
||||
weights = alphas * transmittance # [..., "num_samples"]
|
||||
|
||||
rgb = torch.sum(weights[..., None] * render_head_outputs[self.rgb_name], dim=-2)
|
||||
|
||||
return rgb
|
||||
|
||||
|
||||
class Depth(Renderer):
|
||||
"""_summary_
|
||||
|
||||
Args:
|
||||
Renderer (_type_): _description_
|
||||
"""
|
||||
|
||||
def required_render_heads(self) -> Set[rh.RenderHead]:
|
||||
return set(rh.DensityHead)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]],
|
||||
deltas: TensorType[..., "num_samples"],
|
||||
) -> TensorType[..., "out_dim"]:
|
||||
raise NotImplementedError
|
||||
@@ -1,12 +1,18 @@
|
||||
"""
|
||||
Collection of sampling strategies
|
||||
"""
|
||||
from typing import NamedTuple
|
||||
from torchtyping import TensorType
|
||||
from torch import nn
|
||||
import torch
|
||||
import nerf.cameras as cameras
|
||||
|
||||
|
||||
RaySamples = NamedTuple(
|
||||
"RaySamples", [("locations", TensorType[..., "num_samples", 3]), ("deltas", TensorType[..., "num_samples"])]
|
||||
)
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
"""Base Sampler. Intended to be subclassed"""
|
||||
|
||||
@@ -26,17 +32,18 @@ class Sampler(nn.Module):
|
||||
|
||||
def generate_samples(
|
||||
self, camera_rays: cameras.Rays, near_plane: float, far_plane: float, num_samples: int
|
||||
) -> TensorType[..., "num_samples", 3]:
|
||||
) -> RaySamples:
|
||||
"""Encodes an input tensor.
|
||||
|
||||
Args:
|
||||
camera_rays (cameras.Rays): Rays to generate samples for
|
||||
|
||||
Returns:
|
||||
TensorType[..., "output_dim"]: A encoded input tensor
|
||||
near_plane (float): Minimum distance along ray to sample
|
||||
far_plane (float): Maximum distance along ray to sample
|
||||
num_samples (int): Number of samples per ray
|
||||
|
||||
Returns:
|
||||
RaySamples: Positions and deltas for samples along a ray
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -55,15 +62,21 @@ class UniformSampler(Sampler):
|
||||
|
||||
Args:
|
||||
camera_rays (cameras.Rays): Rays to generate samples for
|
||||
|
||||
Returns:
|
||||
TensorType[..., "output_dim"]: A encoded input tensor
|
||||
near_plane (float): Minimum distance along ray to sample
|
||||
far_plane (float): Maximum distance along ray to sample
|
||||
num_samples (int): Number of samples per ray
|
||||
|
||||
Returns:
|
||||
RaySamples: Positions and deltas for samples along a ray
|
||||
"""
|
||||
near_plane = near_plane or self.near_plane
|
||||
far_plane = far_plane or self.far_plane
|
||||
num_samples = num_samples or self.num_samples
|
||||
|
||||
return torch.linspace(near_plane, far_plane, num_samples)
|
||||
bins = torch.linspace(near_plane, far_plane, num_samples + 1)
|
||||
|
||||
positions = (bins[..., 1:] + bins[..., :-1]) / 2.0
|
||||
deltas = bins[..., 1:] - bins[..., :-1]
|
||||
|
||||
return RaySamples(locations=positions, deltas=deltas)
|
||||
Reference in new issue
Block a user