add sdf mesh export and texturing (#1620)

* add sdf mesh export and texturing

* update bug not allowing use of no monopriors

* resolve comments

* update sdf name to be more generic

* fix silly naming bug
This commit is contained in:
Pablo Vela authored and GitHub committed 2023-04-11 13:34:45 -05:00
1 parent 2c6a328780
commit e7ec208b6a
5 files changed
+323 -10

No files matched your search

@@ -51,7 +51,7 @@ class SDFStudioDataParserConfig(DataParserConfig):
scene_scale: float = 2.0
"""
Sets the bounding cube to have edge length of this size.
The longest dimension of the Friends axis-aligned bbox will be scaled to this value.
The longest dimension of the axis-aligned bbox will be scaled to this value.
"""
skip_every_for_val_split: int = 1
"""sub sampling validation images"""
@@ -141,8 +141,8 @@ class SDFStudio(DataParser):
# TODO supports downsample
# cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor)
assert meta["has_mono_prior"] == self.config.include_mono_prior, f"no mono prior in {self.config.data}"
if self.config.include_mono_prior:
assert meta["has_mono_prior"], f"no mono prior in {self.config.data}"
dataparser_outputs = DataparserOutputs(
image_filenames=image_filenames,
+253
View File
@@ -0,0 +1,253 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module implements the Marching Cubes algorithm for extracting
isosurfaces
"""
from typing import Callable, List, Tuple, Union
import numpy as np
import torch
import trimesh
from skimage import measure
from torchtyping import TensorType
avg_pool_3d = torch.nn.AvgPool3d(2, stride=2)
upsample = torch.nn.Upsample(scale_factor=2, mode="nearest")
def create_point_pyramid(
points: TensorType[3, "height", "width", "depth"]
) -> List[TensorType[3, "height", "width", "depth"]]:
"""
Create a point pyramid for multi-resolution evaluation.
Args:
points: A torch tensor containing 3D points.
Returns:
A list of torch tensors representing points at different resolutions.
"""
points_pyramid = [points]
for _ in range(3):
points = avg_pool_3d(points[None])[0]
points_pyramid.append(points)
points_pyramid = points_pyramid[::-1]
return points_pyramid
def evaluate_sdf(sdf: Callable, points: TensorType["batch", 3]) -> TensorType["batch"]:
"""
Evaluate a signed distance function (SDF) for a batch of points.
Args:
sdf: A callable function that takes a tensor of size (N, 3) containing
3D points and returns a tensor of size (N,) with the SDF values.
points: A torch tensor containing 3D points.
Returns:
A torch tensor with the SDF values evaluated at the given points.
"""
z = []
for _, pnts in enumerate(torch.split(points, 100000, dim=0)):
z.append(sdf(pnts))
z = torch.cat(z, axis=0)
return z
def evaluate_multiresolution_sdf(
evaluate: Callable,
points_pyramid: List[TensorType[3, "height", "width", "depth"]],
coarse_mask: Union[TensorType[1, 1, "height", "width", "depth"], None],
x_max: float,
x_min: float,
crop_n: int,
) -> TensorType["batch"]:
"""
Evaluate SDF values using a multi-resolution approach with a given point pyramid.
Args:
evaluate: A callable function to evaluate SDF values at given points.
points_pyramid: A list of torch tensors representing points at different resolutions.
coarse_mask: A binary mask tensor indicating regions of the bounding box where the SDF
is expected to have a zero-crossing.
x_max: The maximum x-coordinate of the bounding box.
x_min: The minimum x-coordinate of the bounding box.
crop_n: The resolution of the grid used to sample the SDF.
Returns:
A torch tensor with the SDF values evaluated at the given points.
"""
mask = None
threshold = 2 * (x_max - x_min) / crop_n * 8
for pid, pts in enumerate(points_pyramid):
coarse_n = pts.shape[-1]
pts = pts.reshape(3, -1).permute(1, 0).contiguous()
if mask is None:
# Only evaluate SDF
if coarse_mask is not None:
pts_sdf = torch.ones_like(pts[:, 1])
valid_mask = torch.nn.functional.grid_sample(coarse_mask, pts[None, None, None])[0, 0, 0, 0] > 0
if valid_mask.any():
pts_sdf[valid_mask] = evaluate(pts[valid_mask].contiguous())
else:
pts_sdf = evaluate(pts)
else:
mask = mask.reshape(-1)
pts_to_eval = pts[mask]
if pts_to_eval.shape[0] > 0:
pts_sdf_eval = evaluate(pts_to_eval.contiguous())
pts_sdf[mask] = pts_sdf_eval
if pid < 3:
# Update mask
mask = torch.abs(pts_sdf) < threshold
mask = mask.reshape(coarse_n, coarse_n, coarse_n)[None, None]
mask = upsample(mask.float()).bool()
pts_sdf = pts_sdf.reshape(coarse_n, coarse_n, coarse_n)[None, None]
pts_sdf = upsample(pts_sdf)
pts_sdf = pts_sdf.reshape(-1)
threshold /= 2.0
return pts_sdf
@torch.no_grad()
def generate_mesh_with_multires_marching_cubes(
geometry_callable_field: Callable,
resolution: int = 512,
bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0),
bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0),
isosurface_threshold: float = 0.0,
coarse_mask: Union[None, TensorType["height", "width", "depth"]] = None,
) -> trimesh.Trimesh:
"""
Computes the isosurface of a signed distance function (SDF) defined by the
callable `sdf` in a given bounding box with a specified resolution. The SDF
is sampled at a set of points within a regular grid, and the marching cubes
algorithm is used to generate a mesh that approximates the isosurface at a
specified isovalue `level`.
Args:
sdf: A callable function that takes as input a tensor of size
(N, 3) containing 3D points, and returns a tensor of size (N,) containing
the signed distance function evaluated at those points.
output_path: The output directory where the resulting mesh will be saved.
resolution: The resolution of the grid used to sample the SDF.
bounding_box_min: The minimum coordinates of the bounding box in which the SDF
will be evaluated.
bounding_box_max: The maximum coordinates of the bounding box in which the SDF
will be evaluated.
isosurface_threshold: The isovalue at which to approximate the isosurface.
coarse_mask: A binary mask tensor of size ("height", "width", "depth") that indicates the regions
of the bounding box where the SDF is expected to have a zero-crossing. If
provided, the algorithm first evaluates the SDF at the coarse voxels where
the mask is True, and then refines the evaluation within these voxels using
a multi-scale approach. If None, evaluates the SDF at all points in the
bounding box.
Returns:
A torch tensor with the SDF values evaluated at the given points.
"""
# Check if resolution is divisible by 512
assert (
resolution % 512 == 0
), f"""resolution must be divisible by 512, got {resolution}.
This is important because the algorithm uses a multi-resolution approach
to evaluate the SDF where the mimimum resolution is 512."""
# Prepare coarse mask if provided
if coarse_mask is not None:
coarse_mask = coarse_mask.permute(2, 1, 0)[None, None].cuda().float()
# Initialize variables
crop_n = 512
N = resolution // crop_n
grid_min = bounding_box_min
grid_max = bounding_box_max
xs = np.linspace(grid_min[0], grid_max[0], N + 1)
ys = np.linspace(grid_min[1], grid_max[1], N + 1)
zs = np.linspace(grid_min[2], grid_max[2], N + 1)
# Initialize meshes list
meshes = []
# Iterate over the grid
for i in range(N):
for j in range(N):
for k in range(N):
# Calculate grid cell boundaries
x_min, x_max = xs[i], xs[i + 1]
y_min, y_max = ys[j], ys[j + 1]
z_min, z_max = zs[k], zs[k + 1]
# Create point grid
x = np.linspace(x_min, x_max, crop_n)
y = np.linspace(y_min, y_max, crop_n)
z = np.linspace(z_min, z_max, crop_n)
xx, yy, zz = np.meshgrid(x, y, z, indexing="ij")
points = torch.tensor(np.vstack([xx.ravel(), yy.ravel(), zz.ravel()]).T, dtype=torch.float).cuda()
# Function to evaluate SDF for a batch of points
def evaluate(points: torch.Tensor) -> torch.Tensor:
return evaluate_sdf(geometry_callable_field, points)
# Construct point pyramids
points = points.reshape(crop_n, crop_n, crop_n, 3).permute(3, 0, 1, 2)
if coarse_mask is not None:
points_tmp = points.permute(1, 2, 3, 0)[None].cuda()
current_mask = torch.nn.functional.grid_sample(coarse_mask, points_tmp)
current_mask = (current_mask > 0.0).cpu().numpy()[0, 0]
else:
current_mask = None
# Create point pyramid for multi-resolution evaluation
points_pyramid = create_point_pyramid(points)
# Evaluate SDF using multi-resolution approach and mask
pts_sdf = evaluate_multiresolution_sdf(evaluate, points_pyramid, coarse_mask, x_max, x_min, crop_n)
z = pts_sdf.detach().cpu().numpy()
# Skip if no surface found
if current_mask is not None:
valid_z = z.reshape(crop_n, crop_n, crop_n)[current_mask]
if valid_z.shape[0] <= 0 or (
np.min(valid_z) > isosurface_threshold or np.max(valid_z) < isosurface_threshold
):
continue
if not (np.min(z) > isosurface_threshold or np.max(z) < isosurface_threshold):
z = z.astype(np.float32)
verts, faces, normals, _ = measure.marching_cubes(
volume=z.reshape(crop_n, crop_n, crop_n),
level=isosurface_threshold,
spacing=(
(x_max - x_min) / (crop_n - 1),
(y_max - y_min) / (crop_n - 1),
(z_max - z_min) / (crop_n - 1),
),
mask=current_mask,
)
verts = verts + np.array([x_min, y_min, z_min])
meshcrop = trimesh.Trimesh(verts, faces, normals)
meshes.append(meshcrop)
combined_mesh: trimesh.Trimesh = trimesh.util.concatenate(meshes)
return combined_mesh
+2
View File
@@ -401,6 +401,7 @@ def export_textured_mesh(
camera_indices = torch.zeros_like(origins[..., 0:1])
nears = torch.zeros_like(origins[..., 0:1])
fars = torch.ones_like(origins[..., 0:1]) * raylen
directions_norm = torch.ones_like(origins[..., 0:1]) # for surface model
camera_ray_bundle = RayBundle(
origins=origins,
directions=directions,
@@ -408,6 +409,7 @@ def export_textured_mesh(
camera_indices=camera_indices,
nears=nears,
fars=fars,
metadata={"directions_norm": directions_norm},
)
CONSOLE.print("Creating texture image by rendering with NeRF...")