From 5ef9b41ab134962a489e2b90ac349413ad05190c Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Thu, 7 Apr 2022 17:17:58 -0700 Subject: [PATCH 1/4] add render heads --- mattport/nerf/render_heads.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mattport/nerf/render_heads.py diff --git a/mattport/nerf/render_heads.py b/mattport/nerf/render_heads.py new file mode 100644 index 000000000..7c8de087c --- /dev/null +++ b/mattport/nerf/render_heads.py @@ -0,0 +1,54 @@ +""" +Collection of render heads +""" +from enum import Enum +from typing import Optional + +from torch import nn +from torchtyping import TensorType + + +class RenderHeadKeys(Enum): + """Keys for model outputs""" + + DENSITY = "density" + DISTS = "dists" + RGB = "rgb" + SH_RGB = "sh_rgb" + + +class RenderHead(nn.Module): + """Base Render head""" + + def __init__(self, in_dim: int, out_dim: int, activation: Optional[nn.Module]) -> None: + super().__init__() + layers = [nn.Linear(in_dim, out_dim)] + if activation: + layers.append(activation) + self.net = nn.Sequential(*layers) + + def forward(self, in_tensor: TensorType[..., "in_dim"]) -> TensorType[..., "out_dim"]: + """Process network output for renderer + + Args: + in_tensor (TensorType[..., "in_dim]): Network input + + Returns: + TensorType[..., "out_dim"]: Render head output + """ + + return self.net(in_tensor) + + +class DensityHead(RenderHead): + """RGB head""" + + def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus) -> None: + super().__init__(in_dim, 1, activation) + + +class RGBHead(RenderHead): + """RGB head""" + + def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Sigmoid) -> None: + super().__init__(in_dim, 3, activation) From 4e705685101e4c1380a1be6cba9428a17d3ff286 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Thu, 7 Apr 2022 17:20:01 -0700 Subject: [PATCH 2/4] update render heads --- mattport/nerf/render_heads.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mattport/nerf/render_heads.py b/mattport/nerf/render_heads.py index 7c8de087c..7dabcade8 100644 --- a/mattport/nerf/render_heads.py +++ b/mattport/nerf/render_heads.py @@ -8,19 +8,17 @@ from torch import nn from torchtyping import TensorType -class RenderHeadKeys(Enum): - """Keys for model outputs""" - - DENSITY = "density" - DISTS = "dists" - RGB = "rgb" - SH_RGB = "sh_rgb" - - class RenderHead(nn.Module): """Base Render head""" def __init__(self, in_dim: int, out_dim: int, activation: Optional[nn.Module]) -> None: + """ + + Args: + in_dim (int): input dimension + out_dim (int): output dimension for renderer + activation (Optional[nn.Module]): output head activation + """ super().__init__() layers = [nn.Linear(in_dim, out_dim)] if activation: From 0bc278c84880577e616af03ce0e11fcb0b2dd912 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Fri, 8 Apr 2022 18:02:17 -0700 Subject: [PATCH 3/4] add structure for renderer --- mattport/nerf/render_heads.py | 3 +-- mattport/nerf/renderer.py | 0 mattport/nerf/sampler.py | 25 +++++++++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) delete mode 100644 mattport/nerf/renderer.py diff --git a/mattport/nerf/render_heads.py b/mattport/nerf/render_heads.py index 7dabcade8..be509d80b 100644 --- a/mattport/nerf/render_heads.py +++ b/mattport/nerf/render_heads.py @@ -1,7 +1,6 @@ """ Collection of render heads """ -from enum import Enum from typing import Optional from torch import nn @@ -39,7 +38,7 @@ class RenderHead(nn.Module): class DensityHead(RenderHead): - """RGB head""" + """Density head""" def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus) -> None: super().__init__(in_dim, 1, activation) diff --git a/mattport/nerf/renderer.py b/mattport/nerf/renderer.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/mattport/nerf/sampler.py b/mattport/nerf/sampler.py index 52da02629..90428f872 100644 --- a/mattport/nerf/sampler.py +++ b/mattport/nerf/sampler.py @@ -1,12 +1,18 @@ """ Collection of sampling strategies """ +from typing import NamedTuple from torchtyping import TensorType from torch import nn import torch import nerf.cameras as cameras +RaySamples = NamedTuple( + "RaySamples", [("locations", TensorType[..., "num_samples", 3]), ("deltas", TensorType[..., "num_samples"])] +) + + class Sampler(nn.Module): """Base Sampler. Intended to be subclassed""" @@ -26,17 +32,18 @@ class Sampler(nn.Module): def generate_samples( self, camera_rays: cameras.Rays, near_plane: float, far_plane: float, num_samples: int - ) -> TensorType[..., "num_samples", 3]: + ) -> RaySamples: """Encodes an input tensor. Args: camera_rays (cameras.Rays): Rays to generate samples for - - Returns: TensorType[..., "output_dim"]: A encoded input tensor near_plane (float): Minimum distance along ray to sample far_plane (float): Maximum distance along ray to sample num_samples (int): Number of samples per ray + + Returns: + RaySamples: Positions and deltas for samples along a ray """ raise NotImplementedError @@ -55,15 +62,21 @@ class UniformSampler(Sampler): Args: camera_rays (cameras.Rays): Rays to generate samples for - - Returns: TensorType[..., "output_dim"]: A encoded input tensor near_plane (float): Minimum distance along ray to sample far_plane (float): Maximum distance along ray to sample num_samples (int): Number of samples per ray + + Returns: + RaySamples: Positions and deltas for samples along a ray """ near_plane = near_plane or self.near_plane far_plane = far_plane or self.far_plane num_samples = num_samples or self.num_samples - return torch.linspace(near_plane, far_plane, num_samples) + bins = torch.linspace(near_plane, far_plane, num_samples + 1) + + positions = (bins[..., 1:] + bins[..., :-1]) / 2.0 + deltas = bins[..., 1:] - bins[..., :-1] + + return RaySamples(locations=positions, deltas=deltas) From ba5e0a68a7bf2abed52795779e90b1c4115b2efa Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Fri, 8 Apr 2022 18:04:50 -0700 Subject: [PATCH 4/4] add renderers --- mattport/nerf/renderers.py | 94 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 mattport/nerf/renderers.py diff --git a/mattport/nerf/renderers.py b/mattport/nerf/renderers.py new file mode 100644 index 000000000..e7b3c4a46 --- /dev/null +++ b/mattport/nerf/renderers.py @@ -0,0 +1,94 @@ +""" +Collection of renderers +""" +from typing import Dict, Set + +import torch +from torch import nn +from torchtyping import TensorType + +from mattport.nerf import render_heads as rh + + +class Renderer(nn.Module): + """Base Renderer. Intended to be subclassed""" + + def required_render_heads(self) -> Set[rh.RenderHead]: + """ + Returns: + Set[RenderHead]: RenderHeads required for this renderer. + """ + raise NotImplementedError + + def forward( + self, + render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]], + deltas: TensorType[..., "num_samples"], + ) -> TensorType[..., "out_dim"]: + """Composite samples along ray and render image + + Args: + in_tensor (TensorType[..., "num_samples", "features"]): feature for each sample + deltas (TensorType[..., "num_samples"]): depth of each sample + + Returns: + TensorType[..., "out_dim"]: Rendered ray + """ + raise NotImplementedError + + +class RGB(Renderer): + """Standard volumetic rendering.""" + + def __init__(self, rgb_name, density_name) -> None: + super().__init__() + self.rgb_name = rgb_name + self.density_name = density_name + + def required_render_heads(self) -> Set[rh.RenderHead]: + return set(rh.RGBHead, rh.DensityHead) + + def forward( + self, + render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]], + deltas: TensorType[..., "num_samples"], + ) -> TensorType[..., "out_dim"]: + """Composite samples along ray and render color image + + Args: + in_tensor (TensorType[..., "num_samples", 3]): RGB for each sample + deltas (TensorType[..., "num_samples"]): depth influence for each sample + + Returns: + TensorType[..., "out_dim"]: Composited RGB ray + """ + + delta_density = deltas * render_head_outputs[self.density_name] + alphas = 1 - torch.exp(-delta_density) + + transmittance = torch.cat(0, torch.cumsum(delta_density[..., :-1], dim=-1), dim=-1) + transmittance = torch.exp(-transmittance) # [..., "num_samples"] + + weights = alphas * transmittance # [..., "num_samples"] + + rgb = torch.sum(weights[..., None] * render_head_outputs[self.rgb_name], dim=-2) + + return rgb + + +class Depth(Renderer): + """_summary_ + + Args: + Renderer (_type_): _description_ + """ + + def required_render_heads(self) -> Set[rh.RenderHead]: + return set(rh.DensityHead) + + def forward( + self, + render_head_outputs: Dict["str", TensorType[..., "num_samples", "features"]], + deltas: TensorType[..., "num_samples"], + ) -> TensorType[..., "out_dim"]: + raise NotImplementedError