Files
nerfstudio/nerfstudio/exporter/exporter_utils.py
T
Nicholas PfaffandJonas Kulhanek 9f487a464f Implement nerfacto alpha transparency training (#2165)
* Implement nerfacto alpha transparency training

* Stop adding all ones alpha channel and clean up blend_background

* Make background_color a required argument in blend_background

* Enable alpha transparent training by default

* Improve background_color typing

* Blend uniform background color for metric computation

* Fix typing error

* Clean up typing

* Fix background_color to outputs assignment

* Model outputs returns unblended RGB (black bg) for random bg

* Update renderers

* Fix comments

* Add support for transparent BG for other methods

* Minor refac

* Fix get_background_color

* Fix typing error for blend_background

* Fix comments

* Export utils support transparency

* Pylint fix

* Fix pcd export masking

* Filter pcd view directions based on opacity

* Clean up get_rgba_image

---------

Co-authored-by: Jonas Kulhanek <jonas.kulhanek@live.com>
2023-07-18 12:54:52 +02:00

340 lines
13 KiB
Python

# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Export utils such as structs, point cloud generation, and rendering code.
"""
from __future__ import annotations
import sys
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
import open3d as o3d
import pymeshlab
import torch
from jaxtyping import Float
from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeRemainingColumn
from torch import Tensor
from nerfstudio.cameras.cameras import Cameras
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.pipelines.base_pipeline import Pipeline, VanillaPipeline
from nerfstudio.utils.rich_utils import CONSOLE, ItersPerSecColumn
@dataclass
class Mesh:
"""Class for a mesh."""
vertices: Float[Tensor, "num_verts 3"]
"""Vertices of the mesh."""
faces: Float[Tensor, "num_faces 3"]
"""Faces of the mesh."""
normals: Float[Tensor, "num_verts 3"]
"""Normals of the mesh."""
colors: Optional[Float[Tensor, "num_verts 3"]] = None
"""Colors of the mesh."""
def get_mesh_from_pymeshlab_mesh(mesh: pymeshlab.Mesh) -> Mesh: # type: ignore
"""Get a Mesh from a pymeshlab mesh.
See https://pymeshlab.readthedocs.io/en/0.1.5/classes/mesh.html for details.
"""
return Mesh(
vertices=torch.from_numpy(mesh.vertex_matrix()).float(),
faces=torch.from_numpy(mesh.face_matrix()).long(),
normals=torch.from_numpy(np.copy(mesh.vertex_normal_matrix())).float(),
colors=torch.from_numpy(mesh.vertex_color_matrix()).float(),
)
def get_mesh_from_filename(filename: str, target_num_faces: Optional[int] = None) -> Mesh:
"""Get a Mesh from a filename."""
ms = pymeshlab.MeshSet() # type: ignore
ms.load_new_mesh(filename)
if target_num_faces is not None:
CONSOLE.print("Running meshing decimation with quadric edge collapse")
ms.meshing_decimation_quadric_edge_collapse(targetfacenum=target_num_faces)
mesh = ms.current_mesh()
return get_mesh_from_pymeshlab_mesh(mesh)
def generate_point_cloud(
pipeline: Pipeline,
num_points: int = 1000000,
remove_outliers: bool = True,
estimate_normals: bool = False,
reorient_normals: bool = False,
rgb_output_name: str = "rgb",
depth_output_name: str = "depth",
normal_output_name: Optional[str] = None,
use_bounding_box: bool = True,
bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0),
bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0),
std_ratio: float = 10.0,
) -> o3d.geometry.PointCloud:
"""Generate a point cloud from a nerf.
Args:
pipeline: Pipeline to evaluate with.
num_points: Number of points to generate. May result in less if outlier removal is used.
remove_outliers: Whether to remove outliers.
reorient_normals: Whether to re-orient the normals based on the view direction.
estimate_normals: Whether to estimate normals.
rgb_output_name: Name of the RGB output.
depth_output_name: Name of the depth output.
normal_output_name: Name of the normal output.
use_bounding_box: Whether to use a bounding box to sample points.
bounding_box_min: Minimum of the bounding box.
bounding_box_max: Maximum of the bounding box.
std_ratio: Threshold based on STD of the average distances across the point cloud to remove outliers.
Returns:
Point cloud.
"""
progress = Progress(
TextColumn(":cloud: Computing Point Cloud :cloud:"),
BarColumn(),
TaskProgressColumn(show_speed=True),
TimeRemainingColumn(elapsed_when_finished=True, compact=True),
console=CONSOLE,
)
points = []
rgbs = []
normals = []
view_directions = []
with progress as progress_bar:
task = progress_bar.add_task("Generating Point Cloud", total=num_points)
while not progress_bar.finished:
normal = None
with torch.no_grad():
ray_bundle, _ = pipeline.datamanager.next_train(0)
outputs = pipeline.model(ray_bundle)
if rgb_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
if depth_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
rgba = pipeline.model.get_rgba_image(outputs, rgb_output_name)
depth = outputs[depth_output_name]
if normal_output_name is not None:
if normal_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {normal_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
normal = outputs[normal_output_name]
assert (
torch.min(normal) >= 0.0 and torch.max(normal) <= 1.0
), "Normal values from method output must be in [0, 1]"
normal = (normal * 2.0) - 1.0
point = ray_bundle.origins + ray_bundle.directions * depth
view_direction = ray_bundle.directions
# Filter points with opacity lower than 0.5
mask = rgba[..., -1] > 0.5
point = point[mask]
view_direction = view_direction[mask]
rgb = rgba[mask][..., :3]
if normal is not None:
normal = normal[mask]
if use_bounding_box:
comp_l = torch.tensor(bounding_box_min, device=point.device)
comp_m = torch.tensor(bounding_box_max, device=point.device)
assert torch.all(
comp_l < comp_m
), f"Bounding box min {bounding_box_min} must be smaller than max {bounding_box_max}"
mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1)
point = point[mask]
rgb = rgb[mask]
view_direction = view_direction[mask]
if normal is not None:
normal = normal[mask]
points.append(point)
rgbs.append(rgb)
view_directions.append(view_direction)
if normal is not None:
normals.append(normal)
progress.advance(task, point.shape[0])
points = torch.cat(points, dim=0)
rgbs = torch.cat(rgbs, dim=0)
view_directions = torch.cat(view_directions, dim=0).cpu()
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points.double().cpu().numpy())
pcd.colors = o3d.utility.Vector3dVector(rgbs.double().cpu().numpy())
ind = None
if remove_outliers:
CONSOLE.print("Cleaning Point Cloud")
pcd, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=std_ratio)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Cleaning Point Cloud")
if ind is not None:
view_directions = view_directions[ind]
# either estimate_normals or normal_output_name, not both
if estimate_normals:
if normal_output_name is not None:
CONSOLE.rule("Error", style="red")
CONSOLE.print("Cannot estimate normals and use normal_output_name at the same time", justify="center")
sys.exit(1)
CONSOLE.print("Estimating Point Cloud Normals")
pcd.estimate_normals()
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Estimating Point Cloud Normals")
elif normal_output_name is not None:
normals = torch.cat(normals, dim=0)
if ind is not None:
# mask out normals for points that were removed with remove_outliers
normals = normals[ind]
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
# re-orient the normals
if reorient_normals:
normals = torch.from_numpy(np.array(pcd.normals)).float()
mask = torch.sum(view_directions * normals, dim=-1) > 0
normals[mask] *= -1
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
return pcd
def render_trajectory(
pipeline: Pipeline,
cameras: Cameras,
rgb_output_name: str,
depth_output_name: str,
rendered_resolution_scaling_factor: float = 1.0,
disable_distortion: bool = False,
return_rgba_images: bool = False,
) -> Tuple[List[np.ndarray], List[np.ndarray]]:
"""Helper function to create a video of a trajectory.
Args:
pipeline: Pipeline to evaluate with.
cameras: Cameras to render.
rgb_output_name: Name of the RGB output.
depth_output_name: Name of the depth output.
rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution.
disable_distortion: Whether to disable distortion.
return_rgba_images: Whether to return RGBA images (default RGB).
Returns:
List of rgb images, list of depth images.
"""
images = []
depths = []
cameras.rescale_output_resolution(rendered_resolution_scaling_factor)
progress = Progress(
TextColumn(":cloud: Computing rgb and depth images :cloud:"),
BarColumn(),
TaskProgressColumn(show_speed=True),
ItersPerSecColumn(suffix="fps"),
TimeRemainingColumn(elapsed_when_finished=True, compact=True),
)
with progress:
for camera_idx in progress.track(range(cameras.size), description=""):
camera_ray_bundle = cameras.generate_rays(
camera_indices=camera_idx, disable_distortion=disable_distortion
).to(pipeline.device)
with torch.no_grad():
outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle)
if rgb_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
if depth_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
if return_rgba_images:
image = pipeline.model.get_rgba_image(outputs, rgb_output_name)
else:
image = outputs[rgb_output_name]
images.append(image.cpu().numpy())
depths.append(outputs[depth_output_name].cpu().numpy())
return images, depths
def collect_camera_poses_for_dataset(dataset: Optional[InputDataset]) -> List[Dict[str, Any]]:
"""Collects rescaled, translated and optimised camera poses for a dataset.
Args:
dataset: Dataset to collect camera poses for.
Returns:
List of dicts containing camera poses.
"""
if dataset is None:
return []
cameras = dataset.cameras
image_filenames = dataset.image_filenames
frames: List[Dict[str, Any]] = []
# new cameras are in cameras, whereas image paths are stored in a private member of the dataset
for idx in range(len(cameras)):
image_filename = image_filenames[idx]
transform = cameras.camera_to_worlds[idx].tolist()
frames.append(
{
"file_path": str(image_filename),
"transform": transform,
}
)
return frames
def collect_camera_poses(pipeline: VanillaPipeline) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
"""Collects camera poses for train and eval datasets.
Args:
pipeline: Pipeline to evaluate with.
Returns:
List of train camera poses, list of eval camera poses.
"""
train_dataset = pipeline.datamanager.train_dataset
assert isinstance(train_dataset, InputDataset)
eval_dataset = pipeline.datamanager.eval_dataset
assert isinstance(eval_dataset, InputDataset)
train_frames = collect_camera_poses_for_dataset(train_dataset)
eval_frames = collect_camera_poses_for_dataset(eval_dataset)
return train_frames, eval_frames