Implement nerfacto alpha transparency training (#2165)

* Implement nerfacto alpha transparency training

* Stop adding all ones alpha channel and clean up blend_background

* Make background_color a required argument in blend_background

* Enable alpha transparent training by default

* Improve background_color typing

* Blend uniform background color for metric computation

* Fix typing error

* Clean up typing

* Fix background_color to outputs assignment

* Model outputs returns unblended RGB (black bg) for random bg

* Update renderers

* Fix comments

* Add support for transparent BG for other methods

* Minor refac

* Fix get_background_color

* Fix typing error for blend_background

* Fix comments

* Export utils support transparency

* Pylint fix

* Fix pcd export masking

* Filter pcd view directions based on opacity

* Clean up get_rgba_image

---------

Co-authored-by: Jonas Kulhanek <jonas.kulhanek@live.com>
This commit is contained in:
Nicholas PfaffandJonas Kulhanek authored and GitHub committed 2023-07-18 12:54:52 +02:00
1 parent dbd4092dbb
commit 9f487a464f
14 files changed
+225 -45

No files matched your search

+96 -11
View File
@@ -28,7 +28,7 @@ Example:
"""
import contextlib
import math
from typing import Generator, Literal, Optional, Union
from typing import Generator, Literal, Optional, Tuple, Union
import nerfacc
import torch
@@ -39,7 +39,7 @@ from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils import colors
from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize
BackgroundColor = Union[Literal["random", "last_sample", "black", "white"], Float[Tensor, "3"]]
BackgroundColor = Union[Literal["random", "last_sample", "black", "white"], Float[Tensor, "3"], Float[Tensor, "*bs 3"]]
BACKGROUND_COLOR_OVERRIDE: Optional[Float[Tensor, "3"]] = None
@@ -75,7 +75,8 @@ class RGBRenderer(nn.Module):
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
num_rays: Optional[int] = None,
) -> Float[Tensor, "*bs 3"]:
"""Composite samples along ray and render color image
"""Composite samples along ray and render color image.
If background color is random, no BG color is added - as if the background was black!
Args:
rgb: RGB for each sample
@@ -101,19 +102,99 @@ class RGBRenderer(nn.Module):
comp_rgb = torch.sum(weights * rgb, dim=-2)
accumulated_weight = torch.sum(weights, dim=-2)
if background_color == "random":
# If background color is random, the predicted color is returned without blending,
# as if the background color was black.
return comp_rgb
elif background_color == "last_sample":
# Note, this is only supported for non-packed samples.
background_color = rgb[..., -1, :]
else:
background_color = cls.get_background_color(background_color, shape=comp_rgb.shape, device=comp_rgb.device)
assert isinstance(background_color, torch.Tensor)
comp_rgb = comp_rgb + background_color * (1.0 - accumulated_weight)
return comp_rgb
@classmethod
def get_background_color(
cls, background_color: BackgroundColor, shape: Tuple[int, ...], device: torch.device
) -> Union[Float[Tensor, "3"], Float[Tensor, "*bs 3"]]:
"""Returns the RGB background color for a specified background color.
Note:
This function CANNOT be called for background_color being either "last_sample" or "random".
Args:
rgb: RGB for each sample.
background_color: The background color specification.
Returns:
Background color as RGB.
"""
assert background_color not in {"last_sample", "random"}
assert shape[-1] == 3, "Background color must be RGB."
if BACKGROUND_COLOR_OVERRIDE is not None:
background_color = BACKGROUND_COLOR_OVERRIDE
if background_color == "last_sample":
background_color = rgb[..., -1, :]
if background_color == "random":
background_color = torch.rand_like(comp_rgb).to(rgb.device)
if isinstance(background_color, str) and background_color in colors.COLORS_DICT:
background_color = colors.COLORS_DICT[background_color].to(rgb.device)
background_color = colors.COLORS_DICT[background_color]
assert isinstance(background_color, Tensor)
# Ensure correct shape
return background_color.expand(shape).to(device)
def blend_background(
self,
image: Tensor,
background_color: Optional[BackgroundColor] = None,
) -> Float[Tensor, "*bs 3"]:
"""Blends the background color into the image if image is RGBA.
Otherwise no blending is performed (we assume opacity of 1).
Args:
image: RGB/RGBA per pixel.
opacity: Alpha opacity per pixel.
background_color: Background color.
Returns:
Blended RGB.
"""
if image.size(-1) < 4:
return image
rgb, opacity = image[..., :3], image[..., 3:]
if background_color is None:
background_color = self.background_color
if background_color in {"last_sample", "random"}:
background_color = "black"
background_color = self.get_background_color(background_color, shape=rgb.shape, device=rgb.device)
assert isinstance(background_color, torch.Tensor)
comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
return rgb * opacity + background_color.to(rgb.device) * (1 - opacity)
return comp_rgb
def blend_background_for_loss_computation(
self,
pred_image: Tensor,
pred_accumulation: Tensor,
gt_image: Tensor,
) -> Tuple[Tensor, Tensor]:
"""Blends a background color into the ground truth and predicted image for
loss computation.
Args:
gt_image: The ground truth image.
pred_image: The predicted RGB values (without background blending).
pred_accumulation: The predicted opacity/ accumulation.
Returns:
A tuple of the predicted and ground truth RGB values.
"""
background_color = self.background_color
if background_color == "last_sample":
background_color = "black" # No background blending for GT
elif background_color == "random":
background_color = torch.rand_like(pred_image)
pred_image = pred_image + background_color * (1.0 - pred_accumulation)
gt_image = self.blend_background(gt_image, background_color=background_color)
return pred_image, gt_image
def forward(
self,
@@ -121,6 +202,7 @@ class RGBRenderer(nn.Module):
weights: Float[Tensor, "*bs num_samples 1"],
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
num_rays: Optional[int] = None,
background_color: Optional[BackgroundColor] = None,
) -> Float[Tensor, "*bs 3"]:
"""Composite samples along ray and render color image
@@ -129,15 +211,18 @@ class RGBRenderer(nn.Module):
weights: Weights for each sample
ray_indices: Ray index for each sample, used when samples are packed.
num_rays: Number of rays, used when samples are packed.
background_color: The background color to use for rendering.
Returns:
Outputs of rgb values.
"""
if background_color is None:
background_color = self.background_color
if not self.training:
rgb = torch.nan_to_num(rgb)
rgb = self.combine_rgb(
rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays
rgb, weights, background_color=background_color, ray_indices=ray_indices, num_rays=num_rays
)
if not self.training:
torch.clamp_(rgb, min=0.0, max=1.0)