mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-22 23:55:36 +08:00
Implement nerfacto alpha transparency training (#2165)
* Implement nerfacto alpha transparency training * Stop adding all ones alpha channel and clean up blend_background * Make background_color a required argument in blend_background * Enable alpha transparent training by default * Improve background_color typing * Blend uniform background color for metric computation * Fix typing error * Clean up typing * Fix background_color to outputs assignment * Model outputs returns unblended RGB (black bg) for random bg * Update renderers * Fix comments * Add support for transparent BG for other methods * Minor refac * Fix get_background_color * Fix typing error for blend_background * Fix comments * Export utils support transparency * Pylint fix * Fix pcd export masking * Filter pcd view directions based on opacity * Clean up get_rgba_image --------- Co-authored-by: Jonas Kulhanek <jonas.kulhanek@live.com>
This commit is contained in:
1 parent
dbd4092dbb
commit
9f487a464f
14 files changed
+225
-45
No files matched your search
@@ -28,7 +28,7 @@ Example:
|
||||
"""
|
||||
import contextlib
|
||||
import math
|
||||
from typing import Generator, Literal, Optional, Union
|
||||
from typing import Generator, Literal, Optional, Tuple, Union
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
@@ -39,7 +39,7 @@ from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.utils import colors
|
||||
from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize
|
||||
|
||||
BackgroundColor = Union[Literal["random", "last_sample", "black", "white"], Float[Tensor, "3"]]
|
||||
BackgroundColor = Union[Literal["random", "last_sample", "black", "white"], Float[Tensor, "3"], Float[Tensor, "*bs 3"]]
|
||||
BACKGROUND_COLOR_OVERRIDE: Optional[Float[Tensor, "3"]] = None
|
||||
|
||||
|
||||
@@ -75,7 +75,8 @@ class RGBRenderer(nn.Module):
|
||||
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
|
||||
num_rays: Optional[int] = None,
|
||||
) -> Float[Tensor, "*bs 3"]:
|
||||
"""Composite samples along ray and render color image
|
||||
"""Composite samples along ray and render color image.
|
||||
If background color is random, no BG color is added - as if the background was black!
|
||||
|
||||
Args:
|
||||
rgb: RGB for each sample
|
||||
@@ -101,19 +102,99 @@ class RGBRenderer(nn.Module):
|
||||
comp_rgb = torch.sum(weights * rgb, dim=-2)
|
||||
accumulated_weight = torch.sum(weights, dim=-2)
|
||||
|
||||
if background_color == "random":
|
||||
# If background color is random, the predicted color is returned without blending,
|
||||
# as if the background color was black.
|
||||
return comp_rgb
|
||||
|
||||
elif background_color == "last_sample":
|
||||
# Note, this is only supported for non-packed samples.
|
||||
background_color = rgb[..., -1, :]
|
||||
else:
|
||||
background_color = cls.get_background_color(background_color, shape=comp_rgb.shape, device=comp_rgb.device)
|
||||
assert isinstance(background_color, torch.Tensor)
|
||||
comp_rgb = comp_rgb + background_color * (1.0 - accumulated_weight)
|
||||
return comp_rgb
|
||||
|
||||
@classmethod
|
||||
def get_background_color(
|
||||
cls, background_color: BackgroundColor, shape: Tuple[int, ...], device: torch.device
|
||||
) -> Union[Float[Tensor, "3"], Float[Tensor, "*bs 3"]]:
|
||||
"""Returns the RGB background color for a specified background color.
|
||||
|
||||
Note:
|
||||
This function CANNOT be called for background_color being either "last_sample" or "random".
|
||||
|
||||
Args:
|
||||
rgb: RGB for each sample.
|
||||
background_color: The background color specification.
|
||||
|
||||
Returns:
|
||||
Background color as RGB.
|
||||
"""
|
||||
assert background_color not in {"last_sample", "random"}
|
||||
assert shape[-1] == 3, "Background color must be RGB."
|
||||
if BACKGROUND_COLOR_OVERRIDE is not None:
|
||||
background_color = BACKGROUND_COLOR_OVERRIDE
|
||||
if background_color == "last_sample":
|
||||
background_color = rgb[..., -1, :]
|
||||
if background_color == "random":
|
||||
background_color = torch.rand_like(comp_rgb).to(rgb.device)
|
||||
if isinstance(background_color, str) and background_color in colors.COLORS_DICT:
|
||||
background_color = colors.COLORS_DICT[background_color].to(rgb.device)
|
||||
background_color = colors.COLORS_DICT[background_color]
|
||||
assert isinstance(background_color, Tensor)
|
||||
|
||||
# Ensure correct shape
|
||||
return background_color.expand(shape).to(device)
|
||||
|
||||
def blend_background(
|
||||
self,
|
||||
image: Tensor,
|
||||
background_color: Optional[BackgroundColor] = None,
|
||||
) -> Float[Tensor, "*bs 3"]:
|
||||
"""Blends the background color into the image if image is RGBA.
|
||||
Otherwise no blending is performed (we assume opacity of 1).
|
||||
|
||||
Args:
|
||||
image: RGB/RGBA per pixel.
|
||||
opacity: Alpha opacity per pixel.
|
||||
background_color: Background color.
|
||||
|
||||
Returns:
|
||||
Blended RGB.
|
||||
"""
|
||||
if image.size(-1) < 4:
|
||||
return image
|
||||
|
||||
rgb, opacity = image[..., :3], image[..., 3:]
|
||||
if background_color is None:
|
||||
background_color = self.background_color
|
||||
if background_color in {"last_sample", "random"}:
|
||||
background_color = "black"
|
||||
background_color = self.get_background_color(background_color, shape=rgb.shape, device=rgb.device)
|
||||
assert isinstance(background_color, torch.Tensor)
|
||||
comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
|
||||
return rgb * opacity + background_color.to(rgb.device) * (1 - opacity)
|
||||
|
||||
return comp_rgb
|
||||
def blend_background_for_loss_computation(
|
||||
self,
|
||||
pred_image: Tensor,
|
||||
pred_accumulation: Tensor,
|
||||
gt_image: Tensor,
|
||||
) -> Tuple[Tensor, Tensor]:
|
||||
"""Blends a background color into the ground truth and predicted image for
|
||||
loss computation.
|
||||
|
||||
Args:
|
||||
gt_image: The ground truth image.
|
||||
pred_image: The predicted RGB values (without background blending).
|
||||
pred_accumulation: The predicted opacity/ accumulation.
|
||||
Returns:
|
||||
A tuple of the predicted and ground truth RGB values.
|
||||
"""
|
||||
background_color = self.background_color
|
||||
if background_color == "last_sample":
|
||||
background_color = "black" # No background blending for GT
|
||||
elif background_color == "random":
|
||||
background_color = torch.rand_like(pred_image)
|
||||
pred_image = pred_image + background_color * (1.0 - pred_accumulation)
|
||||
gt_image = self.blend_background(gt_image, background_color=background_color)
|
||||
return pred_image, gt_image
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -121,6 +202,7 @@ class RGBRenderer(nn.Module):
|
||||
weights: Float[Tensor, "*bs num_samples 1"],
|
||||
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
|
||||
num_rays: Optional[int] = None,
|
||||
background_color: Optional[BackgroundColor] = None,
|
||||
) -> Float[Tensor, "*bs 3"]:
|
||||
"""Composite samples along ray and render color image
|
||||
|
||||
@@ -129,15 +211,18 @@ class RGBRenderer(nn.Module):
|
||||
weights: Weights for each sample
|
||||
ray_indices: Ray index for each sample, used when samples are packed.
|
||||
num_rays: Number of rays, used when samples are packed.
|
||||
background_color: The background color to use for rendering.
|
||||
|
||||
Returns:
|
||||
Outputs of rgb values.
|
||||
"""
|
||||
if background_color is None:
|
||||
background_color = self.background_color
|
||||
|
||||
if not self.training:
|
||||
rgb = torch.nan_to_num(rgb)
|
||||
rgb = self.combine_rgb(
|
||||
rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays
|
||||
rgb, weights, background_color=background_color, ray_indices=ray_indices, num_rays=num_rays
|
||||
)
|
||||
if not self.training:
|
||||
torch.clamp_(rgb, min=0.0, max=1.0)
|
||||
|
||||
Reference in new issue
Block a user