Mesh export (#809)

* start of mesh PR

* point cloud export mvp

* start of mesh PR

* point cloud export mvp

* Save out mesh. Poor quality atm

* reorganized mesh algorithms and specified CLI

* CLI reorg

* tsdf algorithm in progress

* debugging

* tsdf functional but camera distortion params not in

* playing with distortion params

* TSDF works but texture looks bad

* added trimesh

* moved to exporter

* removed files

* saving progress

* save texture image

* seems to work w/o barycentric

* barycentric start

* gap between triangles

* blocking out diagonal

* new barcentric and scaling

* meshing working. need to remove extra rows

* fixed triangle packing

* gitignore changes

* removed package

* fix Literal import

* comments

* update .mtl file

* poisson mesh with model output normals & resolved some PR comments on git

* starting docs. breaking up big function. todo: add xatlas

* xatlas and nvdiffrast working

* triangle_id

* progress on expensive texture coordinates method

* pytorch meshing works

* removing nvdiffrast

* remove trimesh

* cleanup

* few changes

* pred normals by default and add minimum additions to viewer

* fix cli in viewer

* add xatlas as dependency

* make point cloud more dense

* reverted viewer changes

* turn off normals by default

* fix docs

* finished todos on github

* remove xatlas

* xatlas

* black

Co-authored-by: Matt Tancik <tancik@berkeley.edu>
This commit is contained in:
Ethan WeberandMatt Tancik authored and GitHub committed 2022-12-02 10:49:19 -08:00
1 parent d9395f2576
commit 1052fccec2
15 files changed
+1526 -166

No files matched your search

+205 -9
View File
@@ -4,16 +4,25 @@ Script for exporting NeRF into other formats.
from __future__ import annotations
from dataclasses import dataclass
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Tuple, Union
from typing import List, Optional, Tuple, Union
import numpy as np
import open3d as o3d
import torch
import tyro
from rich.console import Console
from typing_extensions import Annotated
from typing_extensions import Annotated, Literal
from nerfstudio.model_components.exporters import generate_point_cloud
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.exporter import texture_utils, tsdf_utils
from nerfstudio.exporter.exporter_utils import (
generate_point_cloud,
get_mesh_from_filename,
)
from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils.eval_utils import eval_setup
CONSOLE = Console(width=120)
@@ -51,6 +60,8 @@ class ExportPointCloud(Exporter):
"""Minimum of the bounding box, used if use_bounding_box is True."""
num_rays_per_batch: int = 65536
"""Number of rays to evaluate per batch. Decrease if you run out of memory."""
std_ratio: float = 10.0
"""Threshold based on STD of the average distances across the point cloud to remove outliers."""
def main(self) -> None:
"""Export point cloud."""
@@ -70,13 +81,16 @@ class ExportPointCloud(Exporter):
estimate_normals=self.estimate_normals,
rgb_output_name=self.rgb_output_name,
depth_output_name=self.depth_output_name,
normal_output_name=None,
use_bounding_box=self.use_bounding_box,
bounding_box_min=self.bounding_box_min,
bounding_box_max=self.bounding_box_max,
std_ratio=self.std_ratio,
)
torch.cuda.empty_cache()
CONSOLE.print(f"[bold green]:white_check_mark: Generated {pcd}")
CONSOLE.print("Saving Point Cloud")
CONSOLE.print("Saving Point Cloud...")
o3d.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), pcd)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud")
@@ -85,25 +99,207 @@ class ExportPointCloud(Exporter):
@dataclass
class ExportTSDFMesh(Exporter):
"""
NOT YET IMPLEMENTED
Export a mesh using TSDF processing.
"""
downscale_factor: int = 2
"""Downscale the images starting from the resolution used for training."""
depth_output_name: str = "depth"
"""Name of the depth output."""
rgb_output_name: str = "rgb"
"""Name of the RGB output."""
resolution: Union[int, List[int]] = field(default_factory=lambda: [128, 128, 128])
"""Resolution of the TSDF volume or [x, y, z] resolutions individually."""
batch_size: int = 10
"""How many depth images to integrate per batch."""
use_bounding_box: bool = True
"""Whether to use a bounding box for the TSDF volume."""
bounding_box_min: Tuple[float, float, float] = (-1, -1, -1)
"""Minimum of the bounding box, used if use_bounding_box is True."""
bounding_box_max: Tuple[float, float, float] = (1, 1, 1)
"""Minimum of the bounding box, used if use_bounding_box is True."""
texture_method: Literal["tsdf", "nerf"] = "nerf"
"""Method to texture the mesh with. Either 'tsdf' or 'nerf'."""
px_per_uv_triangle: int = 4
"""Number of pixels per UV triangle."""
unwrap_method: Literal["xatlas", "custom"] = "xatlas"
"""The method to use for unwrapping the mesh."""
num_pixels_per_side: int = 2048
"""If using xatlas for unwrapping, the pixels per side of the texture image."""
target_num_faces: Optional[int] = 50000
"""Target number of faces for the mesh to texture."""
def main(self) -> None:
"""Export mesh"""
raise NotImplementedError("TSDF mesh reconstruction not implemented yet.")
if not self.output_dir.exists():
self.output_dir.mkdir(parents=True)
_, pipeline, _ = eval_setup(self.load_config)
tsdf_utils.export_tsdf_mesh(
pipeline,
self.output_dir,
self.downscale_factor,
self.depth_output_name,
self.rgb_output_name,
self.resolution,
self.batch_size,
use_bounding_box=self.use_bounding_box,
bounding_box_min=self.bounding_box_min,
bounding_box_max=self.bounding_box_max,
)
# possibly
# texture the mesh with NeRF and export to a mesh.obj file
# and a material and texture file
if self.texture_method == "nerf":
# load the mesh from the tsdf export
mesh = get_mesh_from_filename(
str(self.output_dir / "tsdf_mesh.ply"), target_num_faces=self.target_num_faces
)
CONSOLE.print("Texturing mesh with NeRF")
texture_utils.export_textured_mesh(
mesh,
pipeline,
self.output_dir,
px_per_uv_triangle=self.px_per_uv_triangle if self.unwrap_method == "custom" else None,
unwrap_method=self.unwrap_method,
num_pixels_per_side=self.num_pixels_per_side,
)
@dataclass
class ExportPoissonMesh(Exporter):
"""
NOT YET IMPLEMENTED
Export a mesh using poisson surface reconstruction.
"""
num_points: int = 1000000
"""Number of points to generate. May result in less if outlier removal is used."""
remove_outliers: bool = True
"""Remove outliers from the point cloud."""
depth_output_name: str = "depth"
"""Name of the depth output."""
rgb_output_name: str = "rgb"
"""Name of the RGB output."""
normal_method: Literal["open3d", "model_output"] = "model_output"
"""Method to estimate normals with."""
normal_output_name: str = "normals"
"""Name of the normal output."""
save_point_cloud: bool = False
"""Whether to save the point cloud."""
use_bounding_box: bool = True
"""Only query points within the bounding box"""
bounding_box_min: Tuple[float, float, float] = (-1, -1, -1)
"""Minimum of the bounding box, used if use_bounding_box is True."""
bounding_box_max: Tuple[float, float, float] = (1, 1, 1)
"""Minimum of the bounding box, used if use_bounding_box is True."""
num_rays_per_batch: int = 65536
"""Number of rays to evaluate per batch. Decrease if you run out of memory."""
texture_method: Literal["point_cloud", "nerf"] = "nerf"
"""Method to texture the mesh with. Either 'point_cloud' or 'nerf'."""
px_per_uv_triangle: int = 4
"""Number of pixels per UV triangle."""
unwrap_method: Literal["xatlas", "custom"] = "xatlas"
"""The method to use for unwrapping the mesh."""
num_pixels_per_side: int = 2048
"""If using xatlas for unwrapping, the pixels per side of the texture image."""
target_num_faces: Optional[int] = 50000
"""Target number of faces for the mesh to texture."""
std_ratio: float = 10.0
"""Threshold based on STD of the average distances across the point cloud to remove outliers."""
def validate_pipeline(self, pipeline: Pipeline) -> None:
"""Check that the pipeline is valid for this exporter."""
if self.normal_method == "model_output":
CONSOLE.print("Checking that the pipeline has a normal output.")
origins = torch.zeros((1, 3), device=pipeline.device)
directions = torch.ones_like(origins)
pixel_area = torch.ones_like(origins[..., :1])
camera_indices = torch.zeros_like(origins[..., :1])
ray_bundle = RayBundle(
origins=origins, directions=directions, pixel_area=pixel_area, camera_indices=camera_indices
)
outputs = pipeline.model(ray_bundle)
if self.normal_output_name not in outputs:
CONSOLE.print(
f"[bold yellow]Warning: Normal output '{self.normal_output_name}' not found in pipeline outputs."
)
CONSOLE.print(f"Available outputs: {list(outputs.keys())}")
CONSOLE.print(
"[bold yellow]Warning: Please train a model with normals "
"(e.g., nerfacto with predicted normals turned on)."
)
CONSOLE.print("[bold yellow]Warning: Or change --normal-method")
CONSOLE.print("[bold yellow]Exiting early.")
sys.exit(1)
def main(self) -> None:
"""Export mesh"""
raise NotImplementedError("Poisson mesh reconstruction not implemented yet.")
if not self.output_dir.exists():
self.output_dir.mkdir(parents=True)
_, pipeline, _ = eval_setup(self.load_config)
self.validate_pipeline(pipeline)
# Increase the batchsize to speed up the evaluation.
pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch
# Whether the normals should be estimated based on the point cloud.
estimate_normals = self.normal_method == "open3d"
pcd = generate_point_cloud(
pipeline=pipeline,
num_points=self.num_points,
remove_outliers=self.remove_outliers,
estimate_normals=estimate_normals,
rgb_output_name=self.rgb_output_name,
depth_output_name=self.depth_output_name,
normal_output_name=self.normal_output_name if self.normal_method == "model_output" else None,
use_bounding_box=self.use_bounding_box,
bounding_box_min=self.bounding_box_min,
bounding_box_max=self.bounding_box_max,
std_ratio=self.std_ratio,
)
torch.cuda.empty_cache()
CONSOLE.print(f"[bold green]:white_check_mark: Generated {pcd}")
if self.save_point_cloud:
CONSOLE.print("Saving Point Cloud...")
o3d.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), pcd)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud")
CONSOLE.print("Computing Mesh... this may take a while.")
mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=9)
vertices_to_remove = densities < np.quantile(densities, 0.1)
mesh.remove_vertices_by_mask(vertices_to_remove)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Computing Mesh")
CONSOLE.print("Saving Mesh...")
o3d.io.write_triangle_mesh(str(self.output_dir / "poisson_mesh.ply"), mesh)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Saving Mesh")
# This will texture the mesh with NeRF and export to a mesh.obj file
# and a material and texture file
if self.texture_method == "nerf":
# load the mesh from the poisson reconstruction
mesh = get_mesh_from_filename(
str(self.output_dir / "poisson_mesh.ply"), target_num_faces=self.target_num_faces
)
CONSOLE.print("Texturing mesh with NeRF")
texture_utils.export_textured_mesh(
mesh,
pipeline,
self.output_dir,
px_per_uv_triangle=self.px_per_uv_triangle if self.unwrap_method == "custom" else None,
unwrap_method=self.unwrap_method,
num_pixels_per_side=self.num_pixels_per_side,
)
@dataclass
+78
View File
@@ -0,0 +1,78 @@
"""
Script to texture an existing mesh file.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
import tyro
from rich.console import Console
from typing_extensions import Literal
from nerfstudio.exporter import texture_utils
from nerfstudio.exporter.exporter_utils import get_mesh_from_filename
from nerfstudio.utils.eval_utils import eval_setup
CONSOLE = Console(width=120)
@dataclass
class TextureMesh:
"""
Export a textured mesh with color computed from the NeRF.
"""
load_config: Path
"""Path to the config YAML file."""
output_dir: Path
"""Path to the output directory."""
input_mesh_filename: Path
"""Mesh filename to texture."""
px_per_uv_triangle: int = 4
"""Number of pixels per UV square."""
unwrap_method: Literal["xatlas", "custom"] = "xatlas"
"""The method to use for unwrapping the mesh."""
num_pixels_per_side: int = 2048
"""If using xatlas for unwrapping, the pixels per side of the texture image."""
target_num_faces: Optional[int] = 50000
"""Target number of faces for the mesh to texture."""
def main(self) -> None:
"""Export textured mesh"""
# pylint: disable=too-many-statements
if not self.output_dir.exists():
self.output_dir.mkdir(parents=True)
# load the Mesh
mesh = get_mesh_from_filename(str(self.input_mesh_filename), target_num_faces=self.target_num_faces)
# load the Pipeline
_, pipeline, _ = eval_setup(self.load_config, test_mode="inference")
# texture the mesh with NeRF and export to a mesh.obj file
# and a material and texture file
texture_utils.export_textured_mesh(
mesh,
pipeline,
self.px_per_uv_triangle,
self.output_dir,
unwrap_method=self.unwrap_method,
num_pixels_per_side=self.num_pixels_per_side,
)
def entrypoint():
"""Entrypoint for use with pyproject scripts."""
tyro.extras.set_accent_color("bright_yellow")
tyro.cli(tyro.conf.FlagConversionOff[TextureMesh]).main()
if __name__ == "__main__":
entrypoint()
# For sphinx docs
get_parser_fn = lambda: tyro.extras.get_parser(TextureMesh) # noqa