Depth data loading for Polycam LiDAR data (#1253)

* Add depth data loading for polycam datasets

* misc: missing arguments in process_data_utils.downscale_images

* Move processing to polycam_utils

* delete unused imports

* process_data_utils: patch for frame%05d issue

* accidentally deleted neighbor flag from ffmpeg

* wrong ffmpeg command

* upscale directly

* redundant comment

* remove unused imports

* black formatting
This commit is contained in:
ManuConcepBrito authored and GitHub committed 2023-01-24 14:45:13 -08:00
1 parent bc65639edf
commit 828d6bc893
3 files changed
+209 -28

No files matched your search

+131 -1
View File
@@ -17,10 +17,11 @@
import json
import sys
from pathlib import Path
from typing import List
from typing import List, Tuple
from rich.console import Console
from nerfstudio.process_data import process_data_utils
from nerfstudio.process_data.process_data_utils import CAMERA_MODELS
from nerfstudio.utils import io
@@ -29,6 +30,7 @@ CONSOLE = Console(width=120)
def polycam_to_json(
image_filenames: List[Path],
depth_filenames: List[Path],
cameras_dir: Path,
output_dir: Path,
min_blur_score: float = 0.0,
@@ -38,6 +40,7 @@ def polycam_to_json(
Args:
image_filenames: List of paths to the original images.
depth_filenames: List of paths to the original depth maps.
cameras_dir: Path to the polycam cameras directory.
output_dir: Path to the output directory.
min_blur_score: Minimum blur score to use an image. Images below this value will be skipped.
@@ -46,6 +49,7 @@ def polycam_to_json(
Returns:
Summary of the conversion.
"""
use_depth = len(image_filenames) == len(depth_filenames)
data = {}
data["camera_model"] = CAMERA_MODELS["perspective"].value
# Needs to be a string for camera_utils.auto_orient_and_center_poses
@@ -67,6 +71,8 @@ def polycam_to_json(
frame["w"] = frame_json["width"] - crop_border_pixels * 2
frame["h"] = frame_json["height"] - crop_border_pixels * 2
frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}"
if use_depth:
frame["depth_map_path"] = f"./depth/frame_{i+1:05d}{depth_filenames[i].suffix}"
# Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions.
frame["transform_matrix"] = [
[frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]],
@@ -90,3 +96,127 @@ def polycam_to_json(
sys.exit(1)
return summary
def process_images(
polycam_image_dir: Path,
image_dir: Path,
crop_border_pixels: int = 15,
max_dataset_size: int = 600,
num_downscales: int = 3,
verbose: bool = True,
) -> Tuple[List[str], List[Path]]:
"""
Process RGB images only
Args:
polycam_image_dir: Path to the directory containing RGB Images
image_dir: Output directory for processed images
crop_border_pixels: Number of pixels to crop from each border of the image. Useful as borders may be
black due to undistortion.
max_dataset_size: Max number of images to train on. If the dataset has more, images will be sampled
approximately evenly. If -1, use all images.
num_downscales: Number of times to downscale the images. Downscales by 2 each time. For example a value of 3
will downscale the images by 2x, 4x, and 8x.
verbose: If True, print extra logging.
Returns:
summary_log: Summary of the processing.
polycam_image_filenames: List of processed images paths
"""
summary_log = []
polycam_image_filenames, num_orig_images = process_data_utils.get_image_filenames(
polycam_image_dir, max_dataset_size
)
# Copy images to output directory
copied_image_paths = process_data_utils.copy_images_list(
polycam_image_filenames,
image_dir=image_dir,
crop_border_pixels=crop_border_pixels,
verbose=verbose,
)
num_frames = len(copied_image_paths)
copied_image_paths = [Path("images/" + copied_image_path.name) for copied_image_path in copied_image_paths]
if max_dataset_size > 0 and num_frames != num_orig_images:
summary_log.append(f"Started with {num_frames} images out of {num_orig_images} total")
summary_log.append(
"To change the size of the dataset add the argument --max_dataset_size to larger than the "
f"current value ({max_dataset_size}), or -1 to use all images."
)
else:
summary_log.append(f"Started with {num_frames} images")
# Downscale images
summary_log.append(process_data_utils.downscale_images(image_dir, num_downscales, verbose=verbose))
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")
sys.exit(1)
return summary_log, polycam_image_filenames
def process_depth_maps(
polycam_depth_dir: Path,
depth_dir: Path,
num_processed_images: int,
crop_border_pixels: int = 15,
max_dataset_size: int = 600,
num_downscales: int = 3,
verbose: bool = True,
) -> Tuple[List[str], List[Path]]:
"""
Process Depth maps from polycam only
Args:
polycam_depth_dir: Path to the directory containing depth maps
depth_dir: Output directory for processed depth maps
num_processed_images: Number of RGB processed that must match the number of depth maps
crop_border_pixels: Number of pixels to crop from each border of the image. Useful as borders may be
black due to undistortion.
max_dataset_size: Max number of images to train on. If the dataset has more, images will be sampled
approximately evenly. If -1, use all images.
num_downscales: Number of times to downscale the images. Downscales by 2 each time. For example a value of 3
will downscale the images by 2x, 4x, and 8x.
verbose: If True, print extra logging.
Returns:
summary_log: Summary of the processing.
polycam_depth_maps_filenames: List of processed depth maps paths
"""
summary_log = []
polycam_depth_maps_filenames, num_orig_depth_maps = process_data_utils.get_image_filenames(
polycam_depth_dir, max_dataset_size
)
# Copy depth images to output directory
copied_depth_maps_paths = process_data_utils.copy_and_upscale_polycam_depth_maps_list(
polycam_depth_maps_filenames, depth_dir=depth_dir, crop_border_pixels=crop_border_pixels, verbose=verbose
)
num_processed_depth_maps = len(copied_depth_maps_paths)
# assert same number of images as depth maps
if num_processed_images != num_processed_depth_maps:
raise ValueError(
f"Expected same amount of depth maps as images. "
f"Instead got {num_processed_images} images and {num_processed_depth_maps} depth maps"
)
if crop_border_pixels > 0 and num_processed_depth_maps != num_orig_depth_maps:
summary_log.append(f"Started with {num_processed_depth_maps} images out of {num_orig_depth_maps} total")
summary_log.append(
"To change the size of the dataset add the argument --max_dataset_size to larger than the "
f"current value ({crop_border_pixels}), or -1 to use all images."
)
else:
summary_log.append(f"Started with {num_processed_depth_maps} images")
# Downscale depth maps
summary_log.append(
process_data_utils.downscale_images(depth_dir, num_downscales, folder_name="depth", verbose=verbose)
)
return summary_log, polycam_depth_maps_filenames