Files
nerfstudio/nerfstudio/exporter/exporter_utils.py
T

357 lines
14 KiB
Python

# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Export utils such as structs, point cloud generation, and rendering code.
"""
from __future__ import annotations
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
import numpy as np
import torch
from jaxtyping import Float
from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeRemainingColumn
from torch import Tensor
from nerfstudio.cameras.camera_optimizers import CameraOptimizer
from nerfstudio.cameras.cameras import Cameras
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.scene_box import OrientedBox
from nerfstudio.pipelines.base_pipeline import Pipeline, VanillaPipeline
from nerfstudio.utils.rich_utils import CONSOLE, ItersPerSecColumn
if TYPE_CHECKING:
# Importing open3d can take ~1 second, so only do it below if we actually
# need it.
import open3d as o3d
import pymeshlab
@dataclass
class Mesh:
"""Class for a mesh."""
vertices: Float[Tensor, "num_verts 3"]
"""Vertices of the mesh."""
faces: Float[Tensor, "num_faces 3"]
"""Faces of the mesh."""
normals: Float[Tensor, "num_verts 3"]
"""Normals of the mesh."""
colors: Optional[Float[Tensor, "num_verts 3"]] = None
"""Colors of the mesh."""
def get_mesh_from_pymeshlab_mesh(mesh: pymeshlab.Mesh) -> Mesh: # type: ignore
"""Get a Mesh from a pymeshlab mesh.
See https://pymeshlab.readthedocs.io/en/0.1.5/classes/mesh.html for details.
"""
return Mesh(
vertices=torch.from_numpy(mesh.vertex_matrix()).float(),
faces=torch.from_numpy(mesh.face_matrix()).long(),
normals=torch.from_numpy(np.copy(mesh.vertex_normal_matrix())).float(),
colors=torch.from_numpy(mesh.vertex_color_matrix()).float(),
)
def get_mesh_from_filename(filename: str, target_num_faces: Optional[int] = None) -> Mesh:
"""Get a Mesh from a filename."""
import pymeshlab
ms = pymeshlab.MeshSet() # type: ignore
ms.load_new_mesh(filename)
if target_num_faces is not None:
CONSOLE.print("Running meshing decimation with quadric edge collapse")
ms.meshing_decimation_quadric_edge_collapse(targetfacenum=target_num_faces)
mesh = ms.current_mesh()
return get_mesh_from_pymeshlab_mesh(mesh)
def generate_point_cloud(
pipeline: Pipeline,
num_points: int = 1000000,
remove_outliers: bool = True,
estimate_normals: bool = False,
reorient_normals: bool = False,
rgb_output_name: str = "rgb",
depth_output_name: str = "depth",
normal_output_name: Optional[str] = None,
crop_obb: Optional[OrientedBox] = None,
std_ratio: float = 10.0,
) -> o3d.geometry.PointCloud:
"""Generate a point cloud from a nerf.
Args:
pipeline: Pipeline to evaluate with.
num_points: Number of points to generate. May result in less if outlier removal is used.
remove_outliers: Whether to remove outliers.
reorient_normals: Whether to re-orient the normals based on the view direction.
estimate_normals: Whether to estimate normals.
rgb_output_name: Name of the RGB output.
depth_output_name: Name of the depth output.
normal_output_name: Name of the normal output.
std_ratio: Threshold based on STD of the average distances across the point cloud to remove outliers.
Returns:
Point cloud.
"""
progress = Progress(
TextColumn(":cloud: Computing Point Cloud :cloud:"),
BarColumn(),
TaskProgressColumn(show_speed=True),
TimeRemainingColumn(elapsed_when_finished=True, compact=True),
console=CONSOLE,
)
points = []
rgbs = []
normals = []
view_directions = []
with progress as progress_bar:
task = progress_bar.add_task("Generating Point Cloud", total=num_points)
while not progress_bar.finished:
normal = None
with torch.no_grad():
ray_bundle, _ = pipeline.datamanager.next_train(0)
assert isinstance(ray_bundle, RayBundle)
outputs = pipeline.model(ray_bundle)
if rgb_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
if depth_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
rgba = pipeline.model.get_rgba_image(outputs, rgb_output_name)
depth = outputs[depth_output_name]
if normal_output_name is not None:
if normal_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {normal_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
normal = outputs[normal_output_name]
assert torch.min(normal) >= 0.0 and torch.max(normal) <= 1.0, (
"Normal values from method output must be in [0, 1]"
)
normal = (normal * 2.0) - 1.0
point = ray_bundle.origins + ray_bundle.directions * depth
view_direction = ray_bundle.directions
# Filter points with opacity lower than 0.5
mask = rgba[..., -1] > 0.5
point = point[mask]
view_direction = view_direction[mask]
rgb = rgba[mask][..., :3]
if normal is not None:
normal = normal[mask]
if crop_obb is not None:
mask = crop_obb.within(point)
point = point[mask]
rgb = rgb[mask]
view_direction = view_direction[mask]
if normal is not None:
normal = normal[mask]
points.append(point)
rgbs.append(rgb)
view_directions.append(view_direction)
if normal is not None:
normals.append(normal)
progress.advance(task, point.shape[0])
points = torch.cat(points, dim=0)
rgbs = torch.cat(rgbs, dim=0)
view_directions = torch.cat(view_directions, dim=0).cpu()
import open3d as o3d
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points.double().cpu().numpy())
pcd.colors = o3d.utility.Vector3dVector(rgbs.double().cpu().numpy())
ind = None
if remove_outliers:
CONSOLE.print("Cleaning Point Cloud")
pcd, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=std_ratio)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Cleaning Point Cloud")
if ind is not None:
view_directions = view_directions[ind]
# either estimate_normals or normal_output_name, not both
if estimate_normals:
if normal_output_name is not None:
CONSOLE.rule("Error", style="red")
CONSOLE.print("Cannot estimate normals and use normal_output_name at the same time", justify="center")
sys.exit(1)
CONSOLE.print("Estimating Point Cloud Normals")
pcd.estimate_normals()
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Estimating Point Cloud Normals")
elif normal_output_name is not None:
normals = torch.cat(normals, dim=0)
if ind is not None:
# mask out normals for points that were removed with remove_outliers
normals = normals[ind]
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
# re-orient the normals
if reorient_normals:
normals = torch.from_numpy(np.array(pcd.normals)).float()
mask = torch.sum(view_directions * normals, dim=-1) > 0
normals[mask] *= -1
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
return pcd
def render_trajectory(
pipeline: Pipeline,
cameras: Cameras,
rgb_output_name: str,
depth_output_name: str,
rendered_resolution_scaling_factor: float = 1.0,
disable_distortion: bool = False,
return_rgba_images: bool = False,
) -> Tuple[List[np.ndarray], List[np.ndarray]]:
"""Helper function to create a video of a trajectory.
Args:
pipeline: Pipeline to evaluate with.
cameras: Cameras to render.
rgb_output_name: Name of the RGB output.
depth_output_name: Name of the depth output.
rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution.
disable_distortion: Whether to disable distortion.
return_rgba_images: Whether to return RGBA images (default RGB).
Returns:
List of rgb images, list of depth images.
"""
images = []
depths = []
cameras.rescale_output_resolution(rendered_resolution_scaling_factor)
progress = Progress(
TextColumn(":cloud: Computing rgb and depth images :cloud:"),
BarColumn(),
TaskProgressColumn(show_speed=True),
ItersPerSecColumn(suffix="fps"),
TimeRemainingColumn(elapsed_when_finished=True, compact=True),
)
with progress:
for camera_idx in progress.track(range(cameras.size), description=""):
camera_ray_bundle = cameras.generate_rays(
camera_indices=camera_idx, disable_distortion=disable_distortion
).to(pipeline.device)
with torch.no_grad():
outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle)
if rgb_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
if depth_output_name not in outputs:
CONSOLE.rule("Error", style="red")
CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center")
CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
if return_rgba_images:
image = pipeline.model.get_rgba_image(outputs, rgb_output_name)
else:
image = outputs[rgb_output_name]
images.append(image.cpu().numpy())
depths.append(outputs[depth_output_name].cpu().numpy())
return images, depths
def collect_camera_poses_for_dataset(
dataset: Optional[InputDataset], camera_optimizer: Optional[CameraOptimizer] = None
) -> List[Dict[str, Any]]:
"""Collects rescaled, translated and optimised camera poses for a dataset.
Args:
dataset: Dataset to collect camera poses for.
camera_optimizer: Camera optimizer that has been used for adjusting the poses
Returns:
List of dicts containing camera poses.
"""
if dataset is None:
return []
cameras = dataset.cameras
image_filenames = dataset.image_filenames
frames: List[Dict[str, Any]] = []
# new cameras are in cameras, whereas image paths are stored in a private member of the dataset
for idx in range(len(cameras)):
image_filename = image_filenames[idx]
if camera_optimizer is None:
transform = cameras.camera_to_worlds[idx].tolist()
else:
# print('exporting optimized camera pose for camera %d' % idx)
camera = cameras[idx : idx + 1]
assert camera.metadata is not None
camera.metadata["cam_idx"] = idx
transform = camera_optimizer.apply_to_camera(camera).tolist()[0]
frames.append(
{
"file_path": str(image_filename),
"transform": transform,
}
)
return frames
def collect_camera_poses(pipeline: VanillaPipeline) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
"""Collects camera poses for train and eval datasets.
Args:
pipeline: Pipeline to evaluate with.
Returns:
List of train camera poses, list of eval camera poses.
"""
train_dataset = pipeline.datamanager.train_dataset
assert isinstance(train_dataset, InputDataset)
eval_dataset = pipeline.datamanager.eval_dataset
assert isinstance(eval_dataset, InputDataset)
camera_optimizer = None
if hasattr(pipeline.model, "camera_optimizer"):
camera_optimizer = pipeline.model.camera_optimizer
assert isinstance(camera_optimizer, CameraOptimizer)
train_frames = collect_camera_poses_for_dataset(train_dataset, camera_optimizer)
# Note: returning original poses, even if --eval-mode=all
eval_frames = collect_camera_poses_for_dataset(eval_dataset)
return train_frames, eval_frames