From 1052fccec25b406562f40c7a8f1d31825773a4cc Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Fri, 2 Dec 2022 10:49:19 -0800 Subject: [PATCH] Mesh export (#809) * start of mesh PR * point cloud export mvp * start of mesh PR * point cloud export mvp * Save out mesh. Poor quality atm * reorganized mesh algorithms and specified CLI * CLI reorg * tsdf algorithm in progress * debugging * tsdf functional but camera distortion params not in * playing with distortion params * TSDF works but texture looks bad * added trimesh * moved to exporter * removed files * saving progress * save texture image * seems to work w/o barycentric * barycentric start * gap between triangles * blocking out diagonal * new barcentric and scaling * meshing working. need to remove extra rows * fixed triangle packing * gitignore changes * removed package * fix Literal import * comments * update .mtl file * poisson mesh with model output normals & resolved some PR comments on git * starting docs. breaking up big function. todo: add xatlas * xatlas and nvdiffrast working * triangle_id * progress on expensive texture coordinates method * pytorch meshing works * removing nvdiffrast * remove trimesh * cleanup * few changes * pred normals by default and add minimum additions to viewer * fix cli in viewer * add xatlas as dependency * make point cloud more dense * reverted viewer changes * turn off normals by default * fix docs * finished todos on github * remove xatlas * xatlas * black Co-authored-by: Matt Tancik --- .gitignore | 5 +- docs/index.md | 1 + docs/quickstart/export_geometry.md | 64 +++ nerfstudio/cameras/cameras.py | 40 +- nerfstudio/configs/method_configs.py | 4 +- nerfstudio/exporter/__init__.py | 13 + nerfstudio/exporter/exporter_utils.py | 258 +++++++++ nerfstudio/exporter/texture_utils.py | 494 ++++++++++++++++++ nerfstudio/exporter/tsdf_utils.py | 351 +++++++++++++ nerfstudio/model_components/exporters.py | 128 ----- .../model_components/scene_colliders.py | 18 +- nerfstudio/utils/rich_utils.py | 19 +- pyproject.toml | 5 +- scripts/exporter.py | 214 +++++++- scripts/texture.py | 78 +++ 15 files changed, 1526 insertions(+), 166 deletions(-) create mode 100644 docs/quickstart/export_geometry.md create mode 100644 nerfstudio/exporter/__init__.py create mode 100644 nerfstudio/exporter/exporter_utils.py create mode 100644 nerfstudio/exporter/texture_utils.py create mode 100644 nerfstudio/exporter/tsdf_utils.py delete mode 100644 nerfstudio/model_components/exporters.py create mode 100644 scripts/texture.py diff --git a/.gitignore b/.gitignore index 72951da16..d6f0cf5d9 100644 --- a/.gitignore +++ b/.gitignore @@ -162,8 +162,9 @@ cython_debug/ #.idea/ # Experiments and outputs -runs/ outputs/ +exports/ +renders/ # tensorboard log files events.out.* @@ -178,10 +179,8 @@ temp* .nfs* external/ __MACOSX/ -outputs*/ node_modules/ bash/ cache/ -renders/ package-lock.json camera_paths/ diff --git a/docs/index.md b/docs/index.md index 69b149711..ae2270986 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,6 +50,7 @@ quickstart/installation quickstart/first_nerf quickstart/custom_dataset quickstart/viewer_quickstart +quickstart/export_geometry quickstart/data_conventions Contributing ``` diff --git a/docs/quickstart/export_geometry.md b/docs/quickstart/export_geometry.md new file mode 100644 index 000000000..0a50e8a0d --- /dev/null +++ b/docs/quickstart/export_geometry.md @@ -0,0 +1,64 @@ +# Export geometry + +Here we document how to export point clouds and meshes from nerfstudio. The main command you'll be working with is `ns-export`. Our point clouds are exported as `.ply` files and the textured meshes are exported as `.obj` files. + +## Exporting a mesh + +### 1. TSDF Fusion + + TSDF (truncated signed distance function) Fusion is a meshing algorithm that uses depth maps to extract a surface as a mesh. This method works for all models. + + ```python + ns-export tsdf --load-config CONFIG.yml --output-dir OUTPUT_DIR + ``` + +### 2. Poisson surface reconstruction + + Poisson surface reconstruction gives the highest quality meshes. See the steps below to use Poisson surface reconstruction in our repo. + + :::{admonition} Note + :class: info + + This will only work with a Model that computes or predicts normals, e.g., nerfacto. + ::: + + 1. Train nerfacto with network settings that predict normals. + + ```bash + ns-train nerfacto --pipeline.model.predict-normals True + ``` + + 2. Export a mesh with the Poisson meshing algorithm. + + ```bash + ns-export poisson --load-config CONFIG.yml --output-dir OUTPUT_DIR + ``` + +## Exporting a point cloud + +```bash +ns-export pointcloud --help +``` + +## Other exporting methods + +Run the folowing command to see other export methods that may exist. + +```python +ns-export --help +``` + +## Texturing an existing mesh with NeRF + +Say you want to simplify and/or smooth a mesh offline, and then you want to texture it with NeRF. You can do that with the following command. It will work for any mesh filetypes that [PyMeshLab](https://pymeshlab.readthedocs.io/en/latest/) can support, for example a `.ply`. + +```python +python scripts/texture.py --load-config CONFIG.yml --input-mesh-filename FILENAME --output-dir OUTPUT_DIR +``` + +## Dependencies + +Our dependencies are shipped with the pip package in the pyproject.toml file. These are the following: + +- [xatlas-python](https://github.com/mworchel/xatlas-python) for unwrapping meshes to a UV map +- [pymeshlab](https://pymeshlab.readthedocs.io/en/latest/) for reducing the number of faces in a mesh diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index a84d39abf..0299fa4bb 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -308,6 +308,7 @@ class Cameras(TensorDataclass): camera_opt_to_camera: Optional[TensorType["num_rays":..., 3, 4]] = None, distortion_params_delta: Optional[TensorType["num_rays":..., 6]] = None, keep_shape: Optional[bool] = None, + disable_distortion: bool = False, ) -> RayBundle: """Generates rays for the given camera indices. @@ -349,6 +350,7 @@ class Cameras(TensorDataclass): keep_shape: If None, then we default to the regular behavior of flattening if cameras is jagged, otherwise keeping dimensions. If False, we flatten at the end. If True, then we keep the shape of the camera_indices and coords tensors (if we can). + disable_distortion: If True, disables distortion. Returns: Rays for the given camera indices and coords. @@ -441,7 +443,7 @@ class Cameras(TensorDataclass): # raybundle.shape == (num_rays) when done # pylint: disable=protected-access raybundle = cameras._generate_rays_from_coords( - camera_indices, coords, camera_opt_to_camera, distortion_params_delta + camera_indices, coords, camera_opt_to_camera, distortion_params_delta, disable_distortion=disable_distortion ) # If we have mandated that we don't keep the shape, then we flatten @@ -460,6 +462,7 @@ class Cameras(TensorDataclass): coords: TensorType["num_rays":..., 2], camera_opt_to_camera: Optional[TensorType["num_rays":..., 3, 4]] = None, distortion_params_delta: Optional[TensorType["num_rays":..., 6]] = None, + disable_distortion: bool = False, ) -> RayBundle: """Generates rays for the given camera indices and coords where self isn't jagged @@ -525,6 +528,8 @@ class Cameras(TensorDataclass): In terms of shape, it follows the same rules as coords, but indexing into it with [i:...] gets you the 1D tensor with the 6 distortion parameters for the camera optimization at RayBundle[i:...]. + disable_distortion: If True, disables distortion. + Returns: Rays for the given camera indices and coords. RayBundle.shape == num_rays """ @@ -582,23 +587,24 @@ class Cameras(TensorDataclass): assert coord_stack.shape == (3,) + num_rays_shape + (2,) # Undistorts our images according to our distortion parameters - distortion_params = None - if self.distortion_params is not None: - distortion_params = self.distortion_params[true_indices] - if distortion_params_delta is not None: - distortion_params = distortion_params + distortion_params_delta - elif distortion_params_delta is not None: - distortion_params = distortion_params_delta + if not disable_distortion: + distortion_params = None + if self.distortion_params is not None: + distortion_params = self.distortion_params[true_indices] + if distortion_params_delta is not None: + distortion_params = distortion_params + distortion_params_delta + elif distortion_params_delta is not None: + distortion_params = distortion_params_delta - # Do not apply distortion for equirectangular images - if distortion_params is not None: - mask = (self.camera_type[true_indices] != CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays) - coord_mask = torch.stack([mask, mask, mask], dim=0) - if mask.any(): - coord_stack[coord_mask, :] = camera_utils.radial_and_tangential_undistort( - coord_stack[coord_mask, :].reshape(3, -1, 2), - distortion_params[mask, :], - ).reshape(-1, 2) + # Do not apply distortion for equirectangular images + if distortion_params is not None: + mask = (self.camera_type[true_indices] != CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays) + coord_mask = torch.stack([mask, mask, mask], dim=0) + if mask.any(): + coord_stack[coord_mask, :] = camera_utils.radial_and_tangential_undistort( + coord_stack[coord_mask, :].reshape(3, -1, 2), + distortion_params[mask, :], + ).reshape(-1, 2) # Make sure after we have undistorted our images, the shapes are still correct assert coord_stack.shape == (3,) + num_rays_shape + (2,) diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 6a05f835d..f666b5ed1 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -135,7 +135,9 @@ method_configs["mipnerf"] = Config( method_configs["semantic-nerfw"] = Config( method_name="semantic-nerfw", - trainer=TrainerConfig(steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True), + trainer=TrainerConfig( + steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True + ), pipeline=VanillaPipelineConfig( datamanager=SemanticDataManagerConfig( dataparser=FriendsDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=8192 diff --git a/nerfstudio/exporter/__init__.py b/nerfstudio/exporter/__init__.py new file mode 100644 index 000000000..d926cf95c --- /dev/null +++ b/nerfstudio/exporter/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py new file mode 100644 index 000000000..c3db219ed --- /dev/null +++ b/nerfstudio/exporter/exporter_utils.py @@ -0,0 +1,258 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Export utils such as structs, point cloud generation, and rendering code. +""" + +# pylint: disable=no-member + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np +import open3d as o3d +import pymeshlab +import torch +from rich.console import Console +from rich.progress import ( + BarColumn, + Progress, + TaskProgressColumn, + TextColumn, + TimeRemainingColumn, +) +from torchtyping import TensorType + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.configs.base_config import Config # pylint: disable=unused-import +from nerfstudio.pipelines.base_pipeline import Pipeline +from nerfstudio.utils.rich_utils import ItersPerSecColumn + +CONSOLE = Console(width=120) + + +@dataclass +class Mesh: + """Class for a mesh.""" + + vertices: TensorType["num_verts", 3] + """Vertices of the mesh.""" + faces: TensorType["num_faces", 3] + """Faces of the mesh.""" + normals: TensorType["num_verts", 3] + """Normals of the mesh.""" + colors: Optional[TensorType["num_verts", 3]] = None + """Colors of the mesh.""" + + +def get_mesh_from_pymeshlab_mesh(mesh: pymeshlab.Mesh) -> Mesh: + """Get a Mesh from a pymeshlab mesh. + See https://pymeshlab.readthedocs.io/en/0.1.5/classes/mesh.html for details. + """ + return Mesh( + vertices=torch.from_numpy(mesh.vertex_matrix()).float(), + faces=torch.from_numpy(mesh.face_matrix()).long(), + normals=torch.from_numpy(np.copy(mesh.vertex_normal_matrix())).float(), + colors=torch.from_numpy(mesh.vertex_color_matrix()).float(), + ) + + +def get_mesh_from_filename(filename: str, target_num_faces: Optional[int] = None) -> Mesh: + """Get a Mesh from a filename.""" + ms = pymeshlab.MeshSet() + ms.load_new_mesh(filename) + if target_num_faces is not None: + CONSOLE.print("Running meshing decimation with quadric edge collapse") + ms.meshing_decimation_quadric_edge_collapse(targetfacenum=target_num_faces) + mesh = ms.current_mesh() + return get_mesh_from_pymeshlab_mesh(mesh) + + +def generate_point_cloud( + pipeline: Pipeline, + num_points: int = 1000000, + remove_outliers: bool = True, + estimate_normals: bool = False, + rgb_output_name: str = "rgb", + depth_output_name: str = "depth", + normal_output_name: Optional[str] = None, + use_bounding_box: bool = True, + bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0), + bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0), + std_ratio: float = 10.0, +) -> o3d.geometry.PointCloud: + """Generate a point cloud from a nerf. + + Args: + pipeline: Pipeline to evaluate with. + num_points: Number of points to generate. May result in less if outlier removal is used. + remove_outliers: Whether to remove outliers. + estimate_normals: Whether to estimate normals. + rgb_output_name: Name of the RGB output. + depth_output_name: Name of the depth output. + normal_output_name: Name of the normal output. + use_bounding_box: Whether to use a bounding box to sample points. + bounding_box_min: Minimum of the bounding box. + bounding_box_max: Maximum of the bounding box. + std_ratio: Threshold based on STD of the average distances across the point cloud to remove outliers. + + Returns: + Point cloud. + """ + + # pylint: disable=too-many-statements + + progress = Progress( + TextColumn(":cloud: Computing Point Cloud :cloud:"), + BarColumn(), + TaskProgressColumn(show_speed=True), + TimeRemainingColumn(elapsed_when_finished=True, compact=True), + ) + points = [] + rgbs = [] + normals = [] + with progress as progress_bar: + task = progress_bar.add_task("Generating Point Cloud", total=num_points) + while not progress_bar.finished: + with torch.no_grad(): + ray_bundle, _ = pipeline.datamanager.next_train(0) + outputs = pipeline.model(ray_bundle) + if rgb_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center") + CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center") + sys.exit(1) + if depth_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center") + CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center") + sys.exit(1) + rgb = outputs[rgb_output_name] + depth = outputs[depth_output_name] + if normal_output_name is not None: + if normal_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {normal_output_name} in the model outputs", justify="center") + CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center") + sys.exit(1) + normal = outputs[normal_output_name] + point = ray_bundle.origins + ray_bundle.directions * depth + + if use_bounding_box: + comp_l = torch.tensor(bounding_box_min, device=point.device) + comp_m = torch.tensor(bounding_box_max, device=point.device) + assert torch.all( + comp_l < comp_m + ), f"Bounding box min {bounding_box_min} must be smaller than max {bounding_box_max}" + mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1) + point = point[mask] + rgb = rgb[mask] + if normal_output_name is not None: + normal = normal[mask] + + points.append(point) + rgbs.append(rgb) + if normal_output_name is not None: + normals.append(normal) + progress.advance(task, point.shape[0]) + points = torch.cat(points, dim=0) + rgbs = torch.cat(rgbs, dim=0) + + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(points.float().cpu().numpy()) + pcd.colors = o3d.utility.Vector3dVector(rgbs.float().cpu().numpy()) + + ind = None + if remove_outliers: + CONSOLE.print("Cleaning Point Cloud") + pcd, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=std_ratio) + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Cleaning Point Cloud") + + # either estimate_normals or normal_output_name, not both + if estimate_normals: + if normal_output_name is not None: + CONSOLE.rule("Error", style="red") + CONSOLE.print("Cannot estimate normals and use normal_output_name at the same time", justify="center") + sys.exit(1) + CONSOLE.print("Estimating Point Cloud Normals") + pcd.estimate_normals() + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Estimating Point Cloud Normals") + elif normal_output_name is not None: + normals = torch.cat(normals, dim=0) + if ind is not None: + # mask out normals for points that were removed with remove_outliers + normals = normals[ind] + pcd.normals = o3d.utility.Vector3dVector(normals.float().cpu().numpy()) + + return pcd + + +def render_trajectory( + pipeline: Pipeline, + cameras: Cameras, + rgb_output_name: str, + depth_output_name: str, + rendered_resolution_scaling_factor: float = 1.0, + disable_distortion: bool = False, +) -> Tuple[List[np.ndarray], List[np.ndarray]]: + """Helper function to create a video of a trajectory. + + Args: + pipeline: Pipeline to evaluate with. + cameras: Cameras to render. + rgb_output_name: Name of the RGB output. + depth_output_name: Name of the depth output. + rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution. + disable_distortion: Whether to disable distortion. + + Returns: + List of rgb images, list of depth images. + """ + images = [] + depths = [] + cameras.rescale_output_resolution(rendered_resolution_scaling_factor) + + progress = Progress( + TextColumn(":cloud: Computing rgb and depth images :cloud:"), + BarColumn(), + TaskProgressColumn(show_speed=True), + ItersPerSecColumn(suffix="fps"), + TimeRemainingColumn(elapsed_when_finished=True, compact=True), + ) + with progress: + for camera_idx in progress.track(range(cameras.size), description=""): + camera_ray_bundle = cameras.generate_rays( + camera_indices=camera_idx, disable_distortion=disable_distortion + ).to(pipeline.device) + with torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if rgb_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center") + CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center") + sys.exit(1) + if depth_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center") + CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center") + sys.exit(1) + images.append(outputs[rgb_output_name].cpu().numpy()) + depths.append(outputs[depth_output_name].cpu().numpy()) + return images, depths diff --git a/nerfstudio/exporter/texture_utils.py b/nerfstudio/exporter/texture_utils.py new file mode 100644 index 000000000..9f2825ce3 --- /dev/null +++ b/nerfstudio/exporter/texture_utils.py @@ -0,0 +1,494 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Texture utils. +""" + +# pylint: disable=no-member + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Optional, Tuple + +import mediapy as media +import numpy as np +import torch +import xatlas +from rich.console import Console +from torchtyping import TensorType +from typing_extensions import Literal + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.exporter.exporter_utils import Mesh +from nerfstudio.pipelines.base_pipeline import Pipeline +from nerfstudio.utils.rich_utils import get_progress + +CONSOLE = Console(width=120) + + +def get_parallelogram_area( + p: TensorType["bs":..., 2], v0: TensorType["bs":..., 2], v1: TensorType["bs":..., 2] +) -> TensorType["bs":...]: + """Given three 2D points, return the area defined by the parallelogram. I.e., 2x the triangle area. + + Args: + p: The origin of the parallelogram. + v0: The first vector of the parallelogram. + v1: The second vector of the parallelogram. + + Returns: + The area of the parallelogram. + """ + return (p[..., 0] - v0[..., 0]) * (v1[..., 1] - v0[..., 1]) - (p[..., 1] - v0[..., 1]) * (v1[..., 0] - v0[..., 0]) + + +def get_texture_image(num_pixels_w, num_pixels_h, device): + """Get a texture image.""" + px_w = 1.0 / num_pixels_w + px_h = 1.0 / num_pixels_h + uv_indices = torch.stack( + torch.meshgrid( + torch.arange(num_pixels_w, device=device), torch.arange(num_pixels_h, device=device), indexing="xy" + ), + dim=-1, + ) + linspace_h = torch.linspace(px_h / 2, 1 - px_h / 2, num_pixels_h, device=device) + linspace_w = torch.linspace(px_w / 2, 1 - px_w / 2, num_pixels_w, device=device) + uv_coords = torch.stack( + torch.meshgrid(linspace_w, linspace_h, indexing="xy"), dim=-1 + ) # (num_pixels_h, num_pixels_w, 2) + + return uv_coords, uv_indices + + +def unwrap_mesh_per_uv_triangle( + vertices: TensorType["num_verts", 3], + faces: TensorType["num_faces", 3], + vertex_normals: TensorType["num_verts", 3], + px_per_uv_triangle: int, +): + """Unwrap a mesh to a UV texture. This is done by making a grid of rectangles in the UV texture map + and then having two triangles per rectangle. Then the texture image is rasterized and uses barycentric + interpolation to get the origins and directions, per pixel, that are needed to render the NeRF with. + + Args: + vertices: The vertices of the mesh. + faces: The faces of the mesh. + vertex_normals: The vertex normals of the mesh. + px_per_uv_triangle: The number of pixels per UV triangle. + """ + + # pylint: disable=too-many-statements + + assert len(vertices) == len(vertex_normals), "Number of vertices and vertex normals must be equal" + device = vertices.device + + # calculate the number of rectangles needed + triangle_padding = 3 + num_squares = math.ceil(len(faces) / 2) + squares_per_side_w = math.ceil(math.sqrt(num_squares)) + squares_per_side_h = math.ceil(num_squares / squares_per_side_w) + px_per_square_w = px_per_uv_triangle + triangle_padding + px_per_square_h = px_per_uv_triangle + num_pixels_w = squares_per_side_w * px_per_square_w + num_pixels_h = squares_per_side_h * px_per_square_h + + # Construct what one square would look like + # The height is equal to px_per_uv_triangle pixels. + # The width is equal to px_per_uv_triangle + 3 pixels. + # v0---------------v1------------------------v2 + # --Triangle 1--------------------------------- + # -----------------3px gap--------------------- + # --------------------------------Triangle 2--- + # v2-----------------------v1----------------v0 + + lr_w = (px_per_uv_triangle + triangle_padding) / num_pixels_w + lr_h = (px_per_uv_triangle) / num_pixels_h + lr = torch.tensor([lr_w, lr_h], device=device) + px_w = 1.0 / num_pixels_w + px_h = 1.0 / num_pixels_h + px = torch.tensor([px_w, px_h], device=device) + edge_len_w = px_per_uv_triangle / num_pixels_w + edge_len_h = px_per_uv_triangle / num_pixels_h + scalar = (px_per_uv_triangle - 1) / px_per_uv_triangle + # uv coords (upper left and lower right) + uv_coords_upper_left = torch.tensor([[0, 0], [edge_len_w, 0], [0, edge_len_h]], device=device) + # scale for bilinear interpolation reasons + uv_coords_upper_left = uv_coords_upper_left * scalar + px / 2 + lower_right = [lr_w, lr_h] + uv_coords_lower_right = torch.tensor( + [ + lower_right, # lower right + [3 * px_w, lr_h], # lower left + [lr_w, 0], # upper right + ], + device=device, + ) + # scale for bilinear interpolation reasons + uv_coords_lower_right = ( + (uv_coords_lower_right - torch.tensor(lower_right, device=device)) * scalar + + torch.tensor(lower_right, device=device) + - px / 2 + ) + + # Tile this pattern across the entire texture + uv_coords_square = torch.stack([uv_coords_upper_left, uv_coords_lower_right], dim=0) # (2, 3, 2) + uv_coords_square = uv_coords_square.reshape(1, 1, 6, 2) # (6, 2) + square_offsets = ( + torch.stack( + torch.meshgrid( + torch.arange(squares_per_side_w, device=device), + torch.arange(squares_per_side_h, device=device), + indexing="xy", + ), + dim=-1, + ) + * lr + ) + uv_coords_square = uv_coords_square + square_offsets.view( + squares_per_side_h, squares_per_side_w, 1, 2 + ) # (num_squares_h, num_squares_w, 6, 2) + texture_coordinates = uv_coords_square.view(-1, 3, 2)[: len(faces)] # (num_faces, 3, 2) + + # Now find the triangle indices for every pixel and the barycentric coordinates + # which can be used to interpolate the XYZ and normal values to then query with NeRF + uv_coords, uv_indices = get_texture_image(num_pixels_w, num_pixels_h, device) + + u_index = torch.div(uv_indices[..., 0], px_per_square_w, rounding_mode="floor") + v_index = torch.div(uv_indices[..., 1], px_per_square_h, rounding_mode="floor") + square_index = v_index * squares_per_side_w + u_index + u_offset = uv_indices[..., 0] % px_per_square_w + v_offset = uv_indices[..., 1] % px_per_square_h + lower_right = (u_offset + v_offset) >= (px_per_square_w - 2) + triangle_index = square_index * 2 + lower_right + triangle_index = torch.clamp(triangle_index, min=0, max=len(faces) - 1) + + nearby_uv_coords = texture_coordinates[triangle_index] # (num_pixels_h, num_pixels_w, 3, 2) + nearby_vertices = vertices[faces[triangle_index]] # (num_pixels_h, num_pixels_w, 3, 3) + nearby_normals = vertex_normals[faces[triangle_index]] # (num_pixels_h, num_pixels_w, 3, 3) + + # compute barycentric coordinates + v0 = nearby_uv_coords[..., 0, :] # (num_pixels, num_pixels, 2) + v1 = nearby_uv_coords[..., 1, :] # (num_pixels, num_pixels, 2) + v2 = nearby_uv_coords[..., 2, :] # (num_pixels, num_pixels, 2) + p = uv_coords # (num_pixels, num_pixels, 2) + area = get_parallelogram_area(v2, v0, v1) # 2x face area. + w0 = get_parallelogram_area(p, v1, v2) / area + w1 = get_parallelogram_area(p, v2, v0) / area + w2 = get_parallelogram_area(p, v0, v1) / area + + origins = ( + nearby_vertices[..., 0, :] * w0[..., None] + + nearby_vertices[..., 1, :] * w1[..., None] + + nearby_vertices[..., 2, :] * w2[..., None] + ).float() + directions = -( + nearby_normals[..., 0, :] * w0[..., None] + + nearby_normals[..., 1, :] * w1[..., None] + + nearby_normals[..., 2, :] * w2[..., None] + ).float() + # normalize the direction vector to make it a unit vector + directions = torch.nn.functional.normalize(directions, dim=-1) + + return texture_coordinates, origins, directions + + +def unwrap_mesh_with_xatlas( + vertices: TensorType["num_verts", 3], + faces: TensorType["num_faces", 3, torch.long], + vertex_normals: TensorType["num_verts", 3], + num_pixels_per_side=1024, + num_faces_per_barycentric_chunk=10, +) -> Tuple[ + TensorType["num_faces", 3, 2], + TensorType["num_pixels", "num_pixels", 3], + TensorType["num_pixels", "num_pixels", "num_pixels"], +]: + """Unwrap a mesh using xatlas. We use xatlas to unwrap the mesh with UV coordinates. + Then we rasterize the mesh with a square pattern. We interpolate the XYZ and normal + values for every pixel in the texture image. We return the texture coordinates, the + origins, and the directions for every pixel. + + Args: + vertices: Tensor of mesh vertices. + faces: Tensor of mesh faces. + vertex_normals: Tensor of mesh vertex normals. + num_pixels_per_side: Number of pixels per side of the texture image. We use a square. + num_faces_per_barycentric_chunk: Number of faces to use for barycentric chunk computation. + + Returns: + texture_coordinates: Tensor of texture coordinates for every face. + origins: Tensor of origins for every pixel. + directions: Tensor of directions for every pixel. + """ + + # pylint: disable=unused-variable + # pylint: disable=too-many-statements + + device = vertices.device + + # unwrap the mesh + vertices_np = vertices.cpu().numpy() + faces_np = faces.cpu().numpy() + vertex_normals_np = vertex_normals.cpu().cpu().numpy() + vmapping, indices, uvs = xatlas.parametrize( # pylint: disable=c-extension-no-member + vertices_np, faces_np, vertex_normals_np + ) + + # vertices texture coordinates + vertices_tc = torch.from_numpy(uvs.astype(np.float32)).to(device) + + # render uv maps + vertices_tc = vertices_tc * 2.0 - 1.0 # uvs to range [-1, 1] + vertices_tc = torch.cat( + (vertices_tc, torch.zeros_like(vertices_tc[..., :1]), torch.ones_like(vertices_tc[..., :1])), dim=-1 + ) # [num_verts, 4] + + texture_coordinates = torch.from_numpy(uvs[indices]).to(device) # (num_faces, 3, 2) + + # Now find the triangle indices for every pixel and the barycentric coordinates + # which can be used to interpolate the XYZ and normal values to then query with NeRF + uv_coords, _ = get_texture_image(num_pixels_per_side, num_pixels_per_side, device) + uv_coords_shape = uv_coords.shape + p = uv_coords.reshape(1, -1, 2) # (1, N, 2) + num_vertices = p.shape[1] + num_faces = texture_coordinates.shape[0] + triangle_distances = torch.ones_like(p[..., 0]) * torch.finfo(torch.float32).max # (1, N) + triangle_indices = torch.zeros_like(p[..., 0]).long() # (1, N) + triangle_w0 = torch.zeros_like(p[..., 0]) # (1, N) + triangle_w1 = torch.zeros_like(p[..., 0]) # (1, N) + triangle_w2 = torch.zeros_like(p[..., 0]) # (1, N) + arange_list = torch.arange(num_vertices, device=device) + progress = get_progress("Chunking faces for rasterization") + with progress: + for i in progress.track(range(num_faces // num_faces_per_barycentric_chunk)): + s = i * num_faces_per_barycentric_chunk + e = min((i + 1) * num_faces_per_barycentric_chunk, num_faces) + v0 = texture_coordinates[s:e, 0:1, :] # (F, 1, 2) + v1 = texture_coordinates[s:e, 1:2, :] # (F, 1, 2) + v2 = texture_coordinates[s:e, 2:3, :] # (F, 1, 2) + # NOTE: could try clockwise vs counter clockwise + area = get_parallelogram_area(v2, v0, v1) # 2x face area. + w0 = get_parallelogram_area(p, v1, v2) / area # (num_faces_per_barycentric_chunk, N) + w1 = get_parallelogram_area(p, v2, v0) / area + w2 = get_parallelogram_area(p, v0, v1) / area + # get distance from center of triangle + dist_to_center = torch.abs(w0) + torch.abs(w1) + torch.abs(w2) + d_values, d_indices = torch.min(dist_to_center, dim=0, keepdim=True) + d_indices_with_offset = d_indices + s # add offset + condition = d_values < triangle_distances + triangle_distances = torch.where(condition, d_values, triangle_distances) + triangle_indices = torch.where(condition, d_indices_with_offset, triangle_indices) + w0_selected = w0[d_indices[0], arange_list].unsqueeze(0) # (1, N) + w1_selected = w1[d_indices[0], arange_list].unsqueeze(0) # (1, N) + w2_selected = w2[d_indices[0], arange_list].unsqueeze(0) # (1, N) + triangle_w0 = torch.where(condition, w0_selected, triangle_w0) + triangle_w1 = torch.where(condition, w1_selected, triangle_w1) + triangle_w2 = torch.where(condition, w2_selected, triangle_w2) + + nearby_vertices = vertices[faces[triangle_indices[0]]] # (N, 3, 3) + nearby_normals = vertex_normals[faces[triangle_indices[0]]] # (N, 3, 3) + + origins = ( + nearby_vertices[..., 0, :] * triangle_w0[0, :, None] + + nearby_vertices[..., 1, :] * triangle_w1[0, :, None] + + nearby_vertices[..., 2, :] * triangle_w2[0, :, None] + ).float() + directions = -( + nearby_normals[..., 0, :] * triangle_w0[0, :, None] + + nearby_normals[..., 1, :] * triangle_w1[0, :, None] + + nearby_normals[..., 2, :] * triangle_w2[0, :, None] + ).float() + + origins = origins.reshape(uv_coords_shape[0], uv_coords_shape[1], 3) + directions = directions.reshape(uv_coords_shape[0], uv_coords_shape[1], 3) + + # normalize the direction vector to make it a unit vector + directions = torch.nn.functional.normalize(directions, dim=-1) + + return texture_coordinates, origins, directions + + +def export_textured_mesh( + mesh: Mesh, + pipeline: Pipeline, + output_dir: Path, + px_per_uv_triangle: Optional[int] = None, + unwrap_method: Literal["xatlas", "custom"] = "xatlas", + raylen_method: Literal["edge", "none"] = "edge", + num_pixels_per_side=1024, +): + """Textures a mesh using the radiance field from the Pipeline. + The mesh is written to an OBJ file in the output directory, + along with the corresponding material and texture files. + Operations will occur on the same device as the Pipeline. + + Args: + mesh: The mesh to texture. + pipeline: The pipeline to use for texturing. + output_dir: The directory to write the textured mesh to. + px_per_uv_triangle: The number of pixels per side of UV triangle. + unwrap_method: The method to use for unwrapping the mesh. + offset_method: The method to use for computing the ray length to render. + num_pixels_per_side: The number of pixels per side of the texture image. + """ + + # pylint: disable=too-many-statements + + device = pipeline.device + + vertices = mesh.vertices.to(device) + faces = mesh.faces.to(device) + vertex_normals = mesh.normals.to(device) + + summary_log = [] + summary_log.append(f"Unwrapped mesh using {unwrap_method} method.") + summary_log.append(f"Mesh has {len(vertices)} vertices and {len(faces)} faces.") + + if unwrap_method == "xatlas": + CONSOLE.print("Unwrapping mesh with xatlas method... this may take a while.") + texture_coordinates, origins, directions = unwrap_mesh_with_xatlas( + vertices, faces, vertex_normals, num_pixels_per_side=num_pixels_per_side + ) + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Unwrapped mesh with xatlas method") + elif unwrap_method == "custom": + CONSOLE.print("Unwrapping mesh with custom method...") + texture_coordinates, origins, directions = unwrap_mesh_per_uv_triangle( + vertices, faces, vertex_normals, px_per_uv_triangle + ) + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Unwrapped mesh with custom method") + else: + raise ValueError(f"Unwrap method {unwrap_method} not supported.") + + if raylen_method == "edge": + face_vertices = vertices[faces] + # compute the length of the rays we want to render + # we make a reasonable approximation by using the mean length of one edge per face + raylen = 2.0 * torch.mean(torch.norm(face_vertices[:, 1, :] - face_vertices[:, 0, :], dim=-1)).float() + elif raylen_method == "none": + raylen = 0.0 + else: + raise ValueError(f"Ray length method {raylen_method} not supported.") + + summary_log.append(f"Length of rendered rays to compute texture values: {raylen}") + + origins = origins - 0.5 * raylen * directions + pixel_area = torch.ones_like(origins[..., 0:1]) + camera_indices = torch.zeros_like(origins[..., 0:1]) + nears = torch.zeros_like(origins[..., 0:1]) + fars = torch.ones_like(origins[..., 0:1]) * raylen + camera_ray_bundle = RayBundle( + origins=origins, + directions=directions, + pixel_area=pixel_area, + camera_indices=camera_indices, + nears=nears, + fars=fars, + ) + + CONSOLE.print("Creating texture image by rendering with NeRF...") + with torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + + # save the texture image + texture_image = outputs["rgb"].cpu().numpy() + media.write_image(str(output_dir / "material_0.png"), texture_image) + + CONSOLE.print("Writing relevant OBJ information to files...") + # create the .mtl file + lines_mtl = [ + "# Generated with nerfstudio", + "newmtl material_0", + "Ka 1.000 1.000 1.000", + "Kd 1.000 1.000 1.000", + "Ks 0.000 0.000 0.000", + "d 1.0", + "illum 2", + "Ns 1.00000000", + "map_Kd material_0.png", + ] + lines_mtl = [line + "\n" for line in lines_mtl] + file_mtl = open(output_dir / "material_0.mtl", "w", encoding="utf-8") # pylint: disable=consider-using-with + file_mtl.writelines(lines_mtl) + file_mtl.close() + + # create the .obj file + lines_obj = ["# Generated with nerfstudio", "mtllib material_0.mtl", "usemtl material_0"] + lines_obj = [line + "\n" for line in lines_obj] + file_obj = open(output_dir / "mesh.obj", "w", encoding="utf-8") # pylint: disable=consider-using-with + file_obj.writelines(lines_obj) + + # write the geometric vertices + vertices = vertices.cpu().numpy() + progress = get_progress("Writing vertices to file", suffix="lines-per-sec") + with progress: + for i in progress.track(range(len(vertices))): + vertex = vertices[i] + line = f"v {vertex[0]} {vertex[1]} {vertex[2]}\n" + file_obj.write(line) + + # write the texture coordinates + texture_coordinates = texture_coordinates.cpu().numpy() + with progress: + progress = get_progress("Writing texture coordinates to file", suffix="lines-per-sec") + for i in progress.track(range(len(faces))): + for uv in texture_coordinates[i]: + line = f"vt {uv[0]} {1.0 - uv[1]}\n" + file_obj.write(line) + + # write the vertex normals + vertex_normals = vertex_normals.cpu().numpy() + progress = get_progress("Writing vertex normals to file", suffix="lines-per-sec") + with progress: + for i in progress.track(range(len(vertex_normals))): + normal = vertex_normals[i] + line = f"vn {normal[0]} {normal[1]} {normal[2]}\n" + file_obj.write(line) + + # write the faces + faces = faces.cpu().numpy() + progress = get_progress("Writing faces to file", suffix="lines-per-sec") + with progress: + for i in progress.track(range(len(faces))): + face = faces[i] + v1 = face[0] + 1 + v2 = face[1] + 1 + v3 = face[2] + 1 + vt1 = i * 3 + 1 + vt2 = i * 3 + 2 + vt3 = i * 3 + 3 + vn1 = v1 + vn2 = v2 + vn3 = v3 + line = f"f {v1}/{vt1}/{vn1} {v2}/{vt2}/{vn2} {v3}/{vt3}/{vn3}\n" + file_obj.write(line) + + file_obj.close() + + summary_log.append(f"OBJ file saved to {output_dir / 'mesh.obj'}") + summary_log.append(f"MTL file saved to {output_dir / 'material_0.mtl'}") + summary_log.append( + f"Texture image saved to {output_dir / 'material_0.png'} " + f"with resolution {texture_image.shape[1]}x{texture_image.shape[0]} (WxH)" + ) + + CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:") + for summary in summary_log: + CONSOLE.print(summary, justify="center") + CONSOLE.rule() diff --git a/nerfstudio/exporter/tsdf_utils.py b/nerfstudio/exporter/tsdf_utils.py new file mode 100644 index 000000000..049f6354d --- /dev/null +++ b/nerfstudio/exporter/tsdf_utils.py @@ -0,0 +1,351 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +TSDF utils. +""" + +# pylint: disable=no-member + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import numpy as np +import pymeshlab +import torch +import torch.nn.functional as F +from rich.console import Console +from skimage import measure +from torchtyping import TensorType + +from nerfstudio.exporter.exporter_utils import Mesh, render_trajectory +from nerfstudio.pipelines.base_pipeline import Pipeline + +CONSOLE = Console(width=120) + + +@dataclass +class TSDF: + """ + Class for creating TSDFs. + """ + + voxel_coords: TensorType[3, "xdim", "ydim", "zdim"] + """Coordinates of each voxel in the TSDF.""" + values: TensorType["xdim", "ydim", "zdim"] + """TSDF values for each voxel.""" + weights: TensorType["xdim", "ydim", "zdim"] + """TSDF weights for each voxel.""" + colors: TensorType["xdim", "ydim", "zdim", 3] + """TSDF colors for each voxel.""" + voxel_size: TensorType[3] + """Size of each voxel in the TSDF. [x, y, z] size.""" + origin: TensorType[3] + """Origin of the TSDF [xmin, ymin, zmin].""" + truncation_margin: float = 5.0 + """Margin for truncation.""" + + def to(self, device: str): + """Move the tensors to the specified device. + + Args: + device: The device to move the tensors to. E.g., "cuda:0" or "cpu". + """ + self.voxel_coords = self.voxel_coords.to(device) + self.values = self.values.to(device) + self.weights = self.weights.to(device) + self.colors = self.colors.to(device) + self.voxel_size = self.voxel_size.to(device) + self.origin = self.origin.to(device) + return self + + @property + def device(self): + """Returns the device that voxel_coords is on.""" + return self.voxel_coords.device + + @property + def truncation(self): + """Returns the truncation distance.""" + # TODO: clean this up + truncation = self.voxel_size[0] * self.truncation_margin + return truncation + + @staticmethod + def from_aabb(aabb: TensorType[2, 3], volume_dims: TensorType[3]): + """Returns an instance of TSDF from an axis-aligned bounding box and volume dimensions. + + Args: + aabb: The axis-aligned bounding box with shape [[xmin, ymin, zmin], [xmax, ymax, zmax]]. + volume_dims: The volume dimensions with shape [xdim, ydim, zdim]. + """ + + origin = aabb[0] + voxel_size = (aabb[1] - aabb[0]) / volume_dims + + # create the voxel coordinates + xdim = torch.arange(volume_dims[0]) + ydim = torch.arange(volume_dims[1]) + zdim = torch.arange(volume_dims[2]) + grid = torch.stack(torch.meshgrid([xdim, ydim, zdim], indexing="ij"), dim=0) + voxel_coords = origin.view(3, 1, 1, 1) + grid * voxel_size.view(3, 1, 1, 1) + + # initialize the values and weights + values = -torch.ones(volume_dims.tolist()) + weights = torch.zeros(volume_dims.tolist()) + colors = torch.zeros(volume_dims.tolist() + [3]) + + # TODO: move to device + + return TSDF(voxel_coords, values, weights, colors, voxel_size, origin) + + def get_mesh(self) -> Mesh: + """Extracts a mesh using marching cubes.""" + + device = self.values.device + + # run marching cubes on CPU + tsdf_values_np = self.values.clamp(-1, 1).cpu().numpy() + vertices, faces, normals, _ = measure.marching_cubes(tsdf_values_np, level=0, allow_degenerate=False) + + vertices_indices = np.round(vertices).astype(int) + colors = self.colors[vertices_indices[:, 0], vertices_indices[:, 1], vertices_indices[:, 2]] + + # move back to original device + vertices = torch.from_numpy(vertices.copy()).to(device) + faces = torch.from_numpy(faces.copy()).to(device) + normals = torch.from_numpy(normals.copy()).to(device) + + # move vertices back to world space + vertices = self.origin.view(1, 3) + vertices * self.voxel_size.view(1, 3) + + return Mesh(vertices=vertices, faces=faces, normals=normals, colors=colors) + + @classmethod + def export_mesh(cls, mesh: Mesh, filename: str): + """Exports the mesh to a file. + We use pymeshlab to export the mesh as a PLY file. + + Args: + mesh: The mesh to export. + filename: The filename to export the mesh to. + """ + vertex_matrix = mesh.vertices.cpu().numpy().astype("float64") + face_matrix = mesh.faces.cpu().numpy().astype("int32") + v_normals_matrix = mesh.normals.cpu().numpy().astype("float64") + v_color_matrix = mesh.colors.cpu().numpy().astype("float64") + # colors need an alpha channel + v_color_matrix = np.concatenate([v_color_matrix, np.ones((v_color_matrix.shape[0], 1))], axis=-1) + + # create a new Mesh + m = pymeshlab.Mesh( + vertex_matrix=vertex_matrix, + face_matrix=face_matrix, + v_normals_matrix=v_normals_matrix, + v_color_matrix=v_color_matrix, + ) + # create a new MeshSet + ms = pymeshlab.MeshSet() + # add the mesh to the MeshSet + ms.add_mesh(m, "mesh") + # save the current mesh + ms.save_current_mesh(filename) + + def integrate_tsdf( + self, + c2w: TensorType["batch", 4, 4], + K: TensorType["batch", 3, 3], + depth_images: TensorType["batch", 1, "height", "width"], + color_images: Optional[TensorType["batch", 3, "height", "width"]] = None, + mask_images: Optional[TensorType["batch", 1, "height", "width"]] = None, + ): + """Integrates a batch of depth images into the TSDF. + + Args: + c2w: The camera extrinsics. + K: The camera intrinsics. + depth_images: The depth images to integrate. + color_images: The color images to integrate. + mask_images: The mask images to integrate. + """ + + if mask_images is not None: + raise NotImplementedError("Mask images are not supported yet.") + + batch_size = c2w.shape[0] + shape = self.voxel_coords.shape[1:] + + # Project voxel_coords into image space... + + image_size = torch.tensor( + [depth_images.shape[-1], depth_images.shape[-2]], device=self.device + ) # [width, height] + + # make voxel_coords homogeneous + voxel_world_coords = self.voxel_coords.view(3, -1) + voxel_world_coords = torch.cat( + [voxel_world_coords, torch.ones(1, voxel_world_coords.shape[1], device=self.device)], dim=0 + ) + voxel_world_coords = voxel_world_coords.unsqueeze(0) # [1, 4, N] + voxel_world_coords = voxel_world_coords.expand(batch_size, *voxel_world_coords.shape[1:]) # [batch, 4, N] + + voxel_cam_coords = torch.bmm(torch.inverse(c2w), voxel_world_coords) # [batch, 4, N] + + # flip the z axis + voxel_cam_coords[:, 2, :] = -voxel_cam_coords[:, 2, :] + # flip the y axis + voxel_cam_coords[:, 1, :] = -voxel_cam_coords[:, 1, :] + + # we need the distance of the point to the camera, not the z coordinate + voxel_depth = torch.sqrt(torch.sum(voxel_cam_coords[:, :3, :] ** 2, dim=-2, keepdim=True)) # [batch, 1, N] + + voxel_cam_coords_z = voxel_cam_coords[:, 2:3, :] + voxel_cam_points = torch.bmm(K, voxel_cam_coords[:, 0:3, :] / voxel_cam_coords_z) # [batch, 3, N] + voxel_pixel_coords = voxel_cam_points[:, :2, :] # [batch, 2, N] + + # Sample the depth images with grid sample... + + grid = voxel_pixel_coords.permute(0, 2, 1) # [batch, N, 2] + # normalize grid to [-1, 1] + grid = 2.0 * grid / image_size.view(1, 1, 2) - 1.0 # [batch, N, 2] + grid = grid[:, None] # [batch, 1, N, 2] + # depth + sampled_depth = F.grid_sample( + input=depth_images, grid=grid, mode="nearest", padding_mode="zeros", align_corners=False + ) # [batch, N, 1] + sampled_depth = sampled_depth.squeeze(2) # [batch, 1, N] + # colors + if color_images is not None: + sampled_colors = F.grid_sample( + input=color_images, grid=grid, mode="nearest", padding_mode="zeros", align_corners=False + ) # [batch, N, 3] + sampled_colors = sampled_colors.squeeze(2) # [batch, 3, N] + + dist = sampled_depth - voxel_depth # [batch, 1, N] + tsdf_values = torch.clamp(dist / self.truncation, min=-1.0, max=1.0) # [batch, 1, N] + valid_points = (voxel_depth > 0) & (sampled_depth > 0) & (dist > -self.truncation) # [batch, 1, N] + + # Sequentially update the TSDF... + + for i in range(batch_size): + + valid_points_i = valid_points[i] + valid_points_i_shape = valid_points_i.view(*shape) # [xdim, ydim, zdim] + + # the old values + old_tsdf_values_i = self.values[valid_points_i_shape] + old_weights_i = self.weights[valid_points_i_shape] + + # the new values + # TODO: let the new weight be configurable + new_tsdf_values_i = tsdf_values[i][valid_points_i] + new_weights_i = 1.0 + + total_weights = old_weights_i + new_weights_i + + self.values[valid_points_i_shape] = ( + old_tsdf_values_i * old_weights_i + new_tsdf_values_i * new_weights_i + ) / total_weights + self.weights[valid_points_i_shape] = torch.clamp(total_weights, max=1.0) + + if color_images is not None: + old_colors_i = self.colors[valid_points_i_shape] # [M, 3] + new_colors_i = sampled_colors[i][:, valid_points_i.squeeze(0)].permute(1, 0) # [M, 3] + self.colors[valid_points_i_shape] = ( + old_colors_i * old_weights_i[:, None] + new_colors_i * new_weights_i + ) / total_weights[:, None] + + +def export_tsdf_mesh( + pipeline: Pipeline, + output_dir: Path, + downscale_factor: int = 2, + depth_output_name: str = "depth", + rgb_output_name: str = "rgb", + resolution: Union[int, List[int]] = field(default_factory=lambda: [256, 256, 256]), + batch_size: int = 10, + use_bounding_box: bool = True, + bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0), + bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0), +): + """Export a TSDF mesh from a pipeline. + + Args: + pipeline: The pipeline to export the mesh from. + output_dir: The directory to save the mesh to. + downscale_factor: Downscale factor for the images. + depth_output_name: Name of the depth output. + rgb_output_name: Name of the RGB output. + resolution: Resolution of the TSDF volume or [x, y, z] resolutions individually. + batch_size: How many depth images to integrate per batch. + use_bounding_box: Whether to use a bounding box for the TSDF volume. + bounding_box_min: Minimum coordinates of the bounding box. + bounding_box_max: Maximum coordinates of the bounding box. + """ + + device = pipeline.device + + dataparser_outputs = pipeline.datamanager.train_dataset._dataparser_outputs # pylint: disable=protected-access + + # initialize the TSDF volume + if not use_bounding_box: + aabb = dataparser_outputs.scene_box.aabb + else: + aabb = torch.tensor([bounding_box_min, bounding_box_max]) + if isinstance(resolution, int): + volume_dims = torch.tensor([resolution] * 3) + elif isinstance(resolution, List): + volume_dims = torch.tensor(resolution) + else: + raise ValueError("Resolution must be an int or a list.") + tsdf = TSDF.from_aabb(aabb, volume_dims=volume_dims) + # move TSDF to device + tsdf.to(device) + + cameras = dataparser_outputs.cameras + # we turn off distortion when populating the TSDF + color_images, depth_images = render_trajectory( + pipeline, + cameras, + rgb_output_name=rgb_output_name, + depth_output_name=depth_output_name, + rendered_resolution_scaling_factor=1.0 / downscale_factor, + disable_distortion=True, + ) + + # camera extrinsics and intrinsics + c2w: TensorType["N", 3, 4] = cameras.camera_to_worlds.to(device) + # make c2w homogeneous + c2w = torch.cat([c2w, torch.zeros(c2w.shape[0], 1, 4, device=device)], dim=1) + c2w[:, 3, 3] = 1 + K: TensorType["N", 3, 3] = cameras.get_intrinsics_matrices().to(device) + color_images = torch.tensor(np.array(color_images), device=device).permute(0, 3, 1, 2) # shape (N, 3, H, W) + depth_images = torch.tensor(np.array(depth_images), device=device).permute(0, 3, 1, 2) # shape (N, 1, H, W) + + CONSOLE.print("Integrating the TSDF") + for i in range(0, len(c2w), batch_size): + tsdf.integrate_tsdf( + c2w[i : i + batch_size], + K[i : i + batch_size], + depth_images[i : i + batch_size], + color_images=color_images[i : i + batch_size], + ) + + CONSOLE.print("Computing Mesh") + mesh = tsdf.get_mesh() + CONSOLE.print("Saving TSDF Mesh") + tsdf.export_mesh(mesh, filename=str(output_dir / "tsdf_mesh.ply")) diff --git a/nerfstudio/model_components/exporters.py b/nerfstudio/model_components/exporters.py deleted file mode 100644 index 947254500..000000000 --- a/nerfstudio/model_components/exporters.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2022 The Nerfstudio Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Algorithms for exporting data.""" - -from __future__ import annotations - -import sys -from typing import Tuple - -import open3d as o3d -import torch -from rich.console import Console -from rich.progress import ( - BarColumn, - Progress, - TaskProgressColumn, - TextColumn, - TimeRemainingColumn, -) - -from nerfstudio.configs.base_config import Config # pylint: disable=unused-import -from nerfstudio.pipelines.base_pipeline import Pipeline - -CONSOLE = Console(width=120) - - -def generate_point_cloud( - pipeline: Pipeline, - num_points: int = 1000000, - remove_outliers: bool = True, - estimate_normals: bool = False, - rgb_output_name: str = "rgb", - depth_output_name: str = "depth", - use_bounding_box: bool = True, - bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0), - bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0), -) -> o3d.geometry.PointCloud: - """Generate a point cloud from a nerf. - - Args: - pipeline: Pipeline to evaluate with. - num_points: Number of points to generate. May result in less if outlier removal is used. - remove_outliers: Whether to remove outliers. - estimate_normals: Whether to estimate normals. - rgb_output_name: Name of the RGB output. - depth_output_name: Name of the depth output. - use_bounding_box: Whether to use a bounding box to sample points. - bounding_box_min: Minimum of the bounding box. - bounding_box_max: Maximum of the bounding box. - - Returns: - Point cloud. - """ - - progress = Progress( - TextColumn(":cloud: Computing Point Cloud :cloud:"), - BarColumn(), - TaskProgressColumn(show_speed=True), - TimeRemainingColumn(elapsed_when_finished=True, compact=True), - ) - points = [] - rgbs = [] - with progress as progress_bar: - task = progress_bar.add_task("Generating Point Cloud", total=num_points) - while not progress_bar.finished: - with torch.no_grad(): - ray_bundle, _ = pipeline.datamanager.next_train(0) - outputs = pipeline.model(ray_bundle) - if rgb_output_name not in outputs: - CONSOLE.rule("Error", style="red") - CONSOLE.print(f"Could not find {rgb_output_name} in the model outputs", justify="center") - CONSOLE.print(f"Please set --rgb_output_name to one of: {outputs.keys()}", justify="center") - sys.exit(1) - if depth_output_name not in outputs: - CONSOLE.rule("Error", style="red") - CONSOLE.print(f"Could not find {depth_output_name} in the model outputs", justify="center") - CONSOLE.print(f"Please set --depth_output_name to one of: {outputs.keys()}", justify="center") - sys.exit(1) - rgb = outputs[rgb_output_name] - depth = outputs[depth_output_name] - point = ray_bundle.origins + ray_bundle.directions * depth - - if use_bounding_box: - comp_l = torch.tensor(bounding_box_min, device=point.device) - comp_m = torch.tensor(bounding_box_max, device=point.device) - assert torch.all( - comp_l < comp_m - ), f"Bounding box min {bounding_box_min} must be smaller than max {bounding_box_max}" - mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1) - point = point[mask] - rgb = rgb[mask] - - points.append(point) - rgbs.append(rgb) - progress.advance(task, point.shape[0]) - points = torch.cat(points, dim=0) - rgbs = torch.cat(rgbs, dim=0) - - pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(points.float().cpu().numpy()) - pcd.colors = o3d.utility.Vector3dVector(rgbs.float().cpu().numpy()) - - if remove_outliers: - CONSOLE.print("Cleaning Point Cloud") - pcd, _ = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=1.0) - print("\033[A\033[A") - CONSOLE.print("[bold green]:white_check_mark: Cleaning Point Cloud") - - if estimate_normals: - CONSOLE.print("Estimating Point Cloud Normals") - pcd.estimate_normals() - print("\033[A\033[A") - CONSOLE.print("[bold green]:white_check_mark: Estimating Point Cloud Normals") - - return pcd diff --git a/nerfstudio/model_components/scene_colliders.py b/nerfstudio/model_components/scene_colliders.py index 5cf178f99..bf3ccd1b9 100644 --- a/nerfstudio/model_components/scene_colliders.py +++ b/nerfstudio/model_components/scene_colliders.py @@ -26,19 +26,25 @@ from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.scene_box import SceneBox -class SceneBoxCollider(nn.Module): +class SceneCollider(nn.Module): """Module for setting near and far values for rays.""" def __init__(self, **kwargs) -> None: self.kwargs = kwargs super().__init__() - def forward(self, ray_bundle: RayBundle) -> RayBundle: + def set_nears_and_fars(self, ray_bundle) -> RayBundle: """To be implemented.""" raise NotImplementedError + def forward(self, ray_bundle: RayBundle) -> RayBundle: + """Sets the nears and fars if they are not set already.""" + if ray_bundle.nears is not None and ray_bundle.fars is not None: + return ray_bundle + return self.set_nears_and_fars(ray_bundle) -class AABBBoxCollider(SceneBoxCollider): + +class AABBBoxCollider(SceneCollider): """Module for colliding rays with the scene box to compute near and far values. Args: @@ -88,7 +94,7 @@ class AABBBoxCollider(SceneBoxCollider): return nears, fars - def forward(self, ray_bundle: RayBundle) -> RayBundle: + def set_nears_and_fars(self, ray_bundle: RayBundle) -> RayBundle: """Intersects the rays with the scene box and updates the near and far values. Populates nears and fars fields and returns the ray_bundle. @@ -102,7 +108,7 @@ class AABBBoxCollider(SceneBoxCollider): return ray_bundle -class NearFarCollider(SceneBoxCollider): +class NearFarCollider(SceneCollider): """Sets the nears and fars with fixed values. Args: @@ -115,7 +121,7 @@ class NearFarCollider(SceneBoxCollider): self.far_plane = far_plane super().__init__(**kwargs) - def forward(self, ray_bundle: RayBundle) -> RayBundle: + def set_nears_and_fars(self, ray_bundle: RayBundle) -> RayBundle: ones = torch.ones_like(ray_bundle.origins[..., 0:1]) near_plane = self.near_plane if self.training else 0 ray_bundle.nears = ones * near_plane diff --git a/nerfstudio/utils/rich_utils.py b/nerfstudio/utils/rich_utils.py index d4d6282cf..60f114ec4 100644 --- a/nerfstudio/utils/rich_utils.py +++ b/nerfstudio/utils/rich_utils.py @@ -15,9 +15,17 @@ """Additional rich ui components""" from contextlib import nullcontext +from typing import Optional from rich.console import Console -from rich.progress import ProgressColumn +from rich.progress import ( + BarColumn, + Progress, + ProgressColumn, + TaskProgressColumn, + TextColumn, + TimeRemainingColumn, +) from rich.text import Text CONSOLE = Console(width=120) @@ -49,3 +57,12 @@ def status(msg: str, spinner: str = "bouncingBall", verbose: bool = False): if verbose: return nullcontext() return CONSOLE.status(msg, spinner=spinner) + + +def get_progress(description: str, suffix: Optional[str] = None): + """Helper function to return a rich Progress object.""" + progress_list = [TextColumn(description), BarColumn(), TaskProgressColumn(show_speed=True)] + progress_list += [ItersPerSecColumn(suffix=suffix)] if suffix else [] + progress_list += [TimeRemainingColumn(elapsed_when_finished=True, compact=True)] + progress = Progress(*progress_list) + return progress diff --git a/pyproject.toml b/pyproject.toml index 2d22d0684..6feeff079 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,10 +35,12 @@ dependencies = [ "opencv-python==4.6.0.66", "plotly==5.7.0", "protobuf==3.20.0", + "pymeshlab==2022.2.post2", "pyngrok==5.1.0", "python-socketio==5.7.1", "requests", "rich==12.5.1", + "scikit-image==0.19.3", "tensorboard==2.9.0", "torch==1.12.1", "torchmetrics[image]>=0.9.3", @@ -46,7 +48,8 @@ dependencies = [ "torchvision>=0.13.0", "u-msgpack-python>=2.4.1", "nuscenes-devkit>=1.1.1", - "wandb>=0.13.3" + "wandb>=0.13.3", + "xatlas" ] [project.urls] diff --git a/scripts/exporter.py b/scripts/exporter.py index 730d604b9..a739b8814 100644 --- a/scripts/exporter.py +++ b/scripts/exporter.py @@ -4,16 +4,25 @@ Script for exporting NeRF into other formats. from __future__ import annotations -from dataclasses import dataclass +import sys +from dataclasses import dataclass, field from pathlib import Path -from typing import Tuple, Union +from typing import List, Optional, Tuple, Union +import numpy as np import open3d as o3d +import torch import tyro from rich.console import Console -from typing_extensions import Annotated +from typing_extensions import Annotated, Literal -from nerfstudio.model_components.exporters import generate_point_cloud +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.exporter import texture_utils, tsdf_utils +from nerfstudio.exporter.exporter_utils import ( + generate_point_cloud, + get_mesh_from_filename, +) +from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils.eval_utils import eval_setup CONSOLE = Console(width=120) @@ -51,6 +60,8 @@ class ExportPointCloud(Exporter): """Minimum of the bounding box, used if use_bounding_box is True.""" num_rays_per_batch: int = 65536 """Number of rays to evaluate per batch. Decrease if you run out of memory.""" + std_ratio: float = 10.0 + """Threshold based on STD of the average distances across the point cloud to remove outliers.""" def main(self) -> None: """Export point cloud.""" @@ -70,13 +81,16 @@ class ExportPointCloud(Exporter): estimate_normals=self.estimate_normals, rgb_output_name=self.rgb_output_name, depth_output_name=self.depth_output_name, + normal_output_name=None, use_bounding_box=self.use_bounding_box, bounding_box_min=self.bounding_box_min, bounding_box_max=self.bounding_box_max, + std_ratio=self.std_ratio, ) + torch.cuda.empty_cache() CONSOLE.print(f"[bold green]:white_check_mark: Generated {pcd}") - CONSOLE.print("Saving Point Cloud") + CONSOLE.print("Saving Point Cloud...") o3d.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), pcd) print("\033[A\033[A") CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud") @@ -85,25 +99,207 @@ class ExportPointCloud(Exporter): @dataclass class ExportTSDFMesh(Exporter): """ - NOT YET IMPLEMENTED Export a mesh using TSDF processing. """ + downscale_factor: int = 2 + """Downscale the images starting from the resolution used for training.""" + depth_output_name: str = "depth" + """Name of the depth output.""" + rgb_output_name: str = "rgb" + """Name of the RGB output.""" + resolution: Union[int, List[int]] = field(default_factory=lambda: [128, 128, 128]) + """Resolution of the TSDF volume or [x, y, z] resolutions individually.""" + batch_size: int = 10 + """How many depth images to integrate per batch.""" + use_bounding_box: bool = True + """Whether to use a bounding box for the TSDF volume.""" + bounding_box_min: Tuple[float, float, float] = (-1, -1, -1) + """Minimum of the bounding box, used if use_bounding_box is True.""" + bounding_box_max: Tuple[float, float, float] = (1, 1, 1) + """Minimum of the bounding box, used if use_bounding_box is True.""" + texture_method: Literal["tsdf", "nerf"] = "nerf" + """Method to texture the mesh with. Either 'tsdf' or 'nerf'.""" + px_per_uv_triangle: int = 4 + """Number of pixels per UV triangle.""" + unwrap_method: Literal["xatlas", "custom"] = "xatlas" + """The method to use for unwrapping the mesh.""" + num_pixels_per_side: int = 2048 + """If using xatlas for unwrapping, the pixels per side of the texture image.""" + target_num_faces: Optional[int] = 50000 + """Target number of faces for the mesh to texture.""" + def main(self) -> None: """Export mesh""" - raise NotImplementedError("TSDF mesh reconstruction not implemented yet.") + + if not self.output_dir.exists(): + self.output_dir.mkdir(parents=True) + + _, pipeline, _ = eval_setup(self.load_config) + + tsdf_utils.export_tsdf_mesh( + pipeline, + self.output_dir, + self.downscale_factor, + self.depth_output_name, + self.rgb_output_name, + self.resolution, + self.batch_size, + use_bounding_box=self.use_bounding_box, + bounding_box_min=self.bounding_box_min, + bounding_box_max=self.bounding_box_max, + ) + + # possibly + # texture the mesh with NeRF and export to a mesh.obj file + # and a material and texture file + if self.texture_method == "nerf": + # load the mesh from the tsdf export + mesh = get_mesh_from_filename( + str(self.output_dir / "tsdf_mesh.ply"), target_num_faces=self.target_num_faces + ) + CONSOLE.print("Texturing mesh with NeRF") + texture_utils.export_textured_mesh( + mesh, + pipeline, + self.output_dir, + px_per_uv_triangle=self.px_per_uv_triangle if self.unwrap_method == "custom" else None, + unwrap_method=self.unwrap_method, + num_pixels_per_side=self.num_pixels_per_side, + ) @dataclass class ExportPoissonMesh(Exporter): """ - NOT YET IMPLEMENTED Export a mesh using poisson surface reconstruction. """ + num_points: int = 1000000 + """Number of points to generate. May result in less if outlier removal is used.""" + remove_outliers: bool = True + """Remove outliers from the point cloud.""" + depth_output_name: str = "depth" + """Name of the depth output.""" + rgb_output_name: str = "rgb" + """Name of the RGB output.""" + normal_method: Literal["open3d", "model_output"] = "model_output" + """Method to estimate normals with.""" + normal_output_name: str = "normals" + """Name of the normal output.""" + save_point_cloud: bool = False + """Whether to save the point cloud.""" + use_bounding_box: bool = True + """Only query points within the bounding box""" + bounding_box_min: Tuple[float, float, float] = (-1, -1, -1) + """Minimum of the bounding box, used if use_bounding_box is True.""" + bounding_box_max: Tuple[float, float, float] = (1, 1, 1) + """Minimum of the bounding box, used if use_bounding_box is True.""" + num_rays_per_batch: int = 65536 + """Number of rays to evaluate per batch. Decrease if you run out of memory.""" + texture_method: Literal["point_cloud", "nerf"] = "nerf" + """Method to texture the mesh with. Either 'point_cloud' or 'nerf'.""" + px_per_uv_triangle: int = 4 + """Number of pixels per UV triangle.""" + unwrap_method: Literal["xatlas", "custom"] = "xatlas" + """The method to use for unwrapping the mesh.""" + num_pixels_per_side: int = 2048 + """If using xatlas for unwrapping, the pixels per side of the texture image.""" + target_num_faces: Optional[int] = 50000 + """Target number of faces for the mesh to texture.""" + std_ratio: float = 10.0 + """Threshold based on STD of the average distances across the point cloud to remove outliers.""" + + def validate_pipeline(self, pipeline: Pipeline) -> None: + """Check that the pipeline is valid for this exporter.""" + if self.normal_method == "model_output": + CONSOLE.print("Checking that the pipeline has a normal output.") + origins = torch.zeros((1, 3), device=pipeline.device) + directions = torch.ones_like(origins) + pixel_area = torch.ones_like(origins[..., :1]) + camera_indices = torch.zeros_like(origins[..., :1]) + ray_bundle = RayBundle( + origins=origins, directions=directions, pixel_area=pixel_area, camera_indices=camera_indices + ) + outputs = pipeline.model(ray_bundle) + if self.normal_output_name not in outputs: + CONSOLE.print( + f"[bold yellow]Warning: Normal output '{self.normal_output_name}' not found in pipeline outputs." + ) + CONSOLE.print(f"Available outputs: {list(outputs.keys())}") + CONSOLE.print( + "[bold yellow]Warning: Please train a model with normals " + "(e.g., nerfacto with predicted normals turned on)." + ) + CONSOLE.print("[bold yellow]Warning: Or change --normal-method") + CONSOLE.print("[bold yellow]Exiting early.") + sys.exit(1) + def main(self) -> None: """Export mesh""" - raise NotImplementedError("Poisson mesh reconstruction not implemented yet.") + + if not self.output_dir.exists(): + self.output_dir.mkdir(parents=True) + + _, pipeline, _ = eval_setup(self.load_config) + self.validate_pipeline(pipeline) + + # Increase the batchsize to speed up the evaluation. + pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch + + # Whether the normals should be estimated based on the point cloud. + estimate_normals = self.normal_method == "open3d" + + pcd = generate_point_cloud( + pipeline=pipeline, + num_points=self.num_points, + remove_outliers=self.remove_outliers, + estimate_normals=estimate_normals, + rgb_output_name=self.rgb_output_name, + depth_output_name=self.depth_output_name, + normal_output_name=self.normal_output_name if self.normal_method == "model_output" else None, + use_bounding_box=self.use_bounding_box, + bounding_box_min=self.bounding_box_min, + bounding_box_max=self.bounding_box_max, + std_ratio=self.std_ratio, + ) + torch.cuda.empty_cache() + CONSOLE.print(f"[bold green]:white_check_mark: Generated {pcd}") + + if self.save_point_cloud: + CONSOLE.print("Saving Point Cloud...") + o3d.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), pcd) + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud") + + CONSOLE.print("Computing Mesh... this may take a while.") + mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=9) + vertices_to_remove = densities < np.quantile(densities, 0.1) + mesh.remove_vertices_by_mask(vertices_to_remove) + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Computing Mesh") + + CONSOLE.print("Saving Mesh...") + o3d.io.write_triangle_mesh(str(self.output_dir / "poisson_mesh.ply"), mesh) + print("\033[A\033[A") + CONSOLE.print("[bold green]:white_check_mark: Saving Mesh") + + # This will texture the mesh with NeRF and export to a mesh.obj file + # and a material and texture file + if self.texture_method == "nerf": + # load the mesh from the poisson reconstruction + mesh = get_mesh_from_filename( + str(self.output_dir / "poisson_mesh.ply"), target_num_faces=self.target_num_faces + ) + CONSOLE.print("Texturing mesh with NeRF") + texture_utils.export_textured_mesh( + mesh, + pipeline, + self.output_dir, + px_per_uv_triangle=self.px_per_uv_triangle if self.unwrap_method == "custom" else None, + unwrap_method=self.unwrap_method, + num_pixels_per_side=self.num_pixels_per_side, + ) @dataclass diff --git a/scripts/texture.py b/scripts/texture.py new file mode 100644 index 000000000..ec6545b87 --- /dev/null +++ b/scripts/texture.py @@ -0,0 +1,78 @@ +""" +Script to texture an existing mesh file. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import tyro +from rich.console import Console +from typing_extensions import Literal + +from nerfstudio.exporter import texture_utils +from nerfstudio.exporter.exporter_utils import get_mesh_from_filename +from nerfstudio.utils.eval_utils import eval_setup + +CONSOLE = Console(width=120) + + +@dataclass +class TextureMesh: + """ + Export a textured mesh with color computed from the NeRF. + """ + + load_config: Path + """Path to the config YAML file.""" + output_dir: Path + """Path to the output directory.""" + input_mesh_filename: Path + """Mesh filename to texture.""" + px_per_uv_triangle: int = 4 + """Number of pixels per UV square.""" + unwrap_method: Literal["xatlas", "custom"] = "xatlas" + """The method to use for unwrapping the mesh.""" + num_pixels_per_side: int = 2048 + """If using xatlas for unwrapping, the pixels per side of the texture image.""" + target_num_faces: Optional[int] = 50000 + """Target number of faces for the mesh to texture.""" + + def main(self) -> None: + """Export textured mesh""" + # pylint: disable=too-many-statements + + if not self.output_dir.exists(): + self.output_dir.mkdir(parents=True) + + # load the Mesh + mesh = get_mesh_from_filename(str(self.input_mesh_filename), target_num_faces=self.target_num_faces) + + # load the Pipeline + _, pipeline, _ = eval_setup(self.load_config, test_mode="inference") + + # texture the mesh with NeRF and export to a mesh.obj file + # and a material and texture file + texture_utils.export_textured_mesh( + mesh, + pipeline, + self.px_per_uv_triangle, + self.output_dir, + unwrap_method=self.unwrap_method, + num_pixels_per_side=self.num_pixels_per_side, + ) + + +def entrypoint(): + """Entrypoint for use with pyproject scripts.""" + tyro.extras.set_accent_color("bright_yellow") + tyro.cli(tyro.conf.FlagConversionOff[TextureMesh]).main() + + +if __name__ == "__main__": + entrypoint() + +# For sphinx docs +get_parser_fn = lambda: tyro.extras.get_parser(TextureMesh) # noqa