mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 21:03:46 +08:00
Refactor process data (#976)
This commit is contained in:
1 parent
1f85a31120
commit
532f8067fd
10 files changed
+900
-737
No files matched your search
+35
-719
@@ -1,717 +1,31 @@
|
||||
#!/usr/bin/env python
|
||||
# pylint: disable=too-many-lines
|
||||
"""Processes a video or image sequence to a nerfstudio compatible dataset."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import sys
|
||||
import zipfile
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union
|
||||
from typing import Union
|
||||
|
||||
import appdirs
|
||||
import numpy as np
|
||||
import requests
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
from rich.progress import track
|
||||
from scipy.spatial.transform import Rotation
|
||||
from typing_extensions import Annotated, Literal
|
||||
|
||||
from nerfstudio.utils import colmap_utils, install_checks, io
|
||||
from nerfstudio.process_data import (
|
||||
colmap_utils,
|
||||
insta360_utils,
|
||||
polycam_utils,
|
||||
process_data_utils,
|
||||
record3d_utils,
|
||||
)
|
||||
from nerfstudio.process_data.process_data_utils import CAMERA_MODELS
|
||||
from nerfstudio.utils import install_checks
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
|
||||
class CameraModel(Enum):
|
||||
"""Enum for camera types."""
|
||||
|
||||
OPENCV = "OPENCV"
|
||||
OPENCV_FISHEYE = "OPENCV_FISHEYE"
|
||||
|
||||
|
||||
CAMERA_MODELS = {
|
||||
"perspective": CameraModel.OPENCV,
|
||||
"fisheye": CameraModel.OPENCV_FISHEYE,
|
||||
}
|
||||
|
||||
|
||||
def status(msg: str, spinner: str = "bouncingBall", verbose: bool = False):
|
||||
"""A context manager that does nothing is verbose is True. Otherwise it hides logs under a message.
|
||||
|
||||
Args:
|
||||
msg: The message to log.
|
||||
spinner: The spinner to use.
|
||||
verbose: If True, print all logs, else hide them.
|
||||
"""
|
||||
if verbose:
|
||||
return nullcontext()
|
||||
return CONSOLE.status(msg, spinner=spinner)
|
||||
|
||||
|
||||
def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float:
|
||||
"""Returns the version of COLMAP.
|
||||
This code assumes that colmap returns a version string of the form
|
||||
"COLMAP 3.8 ..." which may not be true for all versions of COLMAP.
|
||||
|
||||
Args:
|
||||
default_version: Default version to return if COLMAP version can't be determined.
|
||||
Returns:
|
||||
The version of COLMAP.
|
||||
"""
|
||||
output = run_command(colmap_cmd, verbose=False)
|
||||
assert output is not None
|
||||
for line in output.split("\n"):
|
||||
if line.startswith("COLMAP"):
|
||||
return float(line.split(" ")[1])
|
||||
CONSOLE.print(f"[bold red]Could not find COLMAP version. Using default {default_version}")
|
||||
return default_version
|
||||
|
||||
|
||||
def get_vocab_tree() -> Path:
|
||||
"""Return path to vocab tree. Downloads vocab tree if it doesn't exist.
|
||||
|
||||
Returns:
|
||||
The path to the vocab tree.
|
||||
"""
|
||||
vocab_tree_filename = Path(appdirs.user_data_dir("nerfstudio")) / "vocab_tree.fbow"
|
||||
|
||||
if not vocab_tree_filename.exists():
|
||||
r = requests.get("https://demuc.de/colmap/vocab_tree_flickr100K_words32K.bin", stream=True)
|
||||
vocab_tree_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(vocab_tree_filename, "wb") as f:
|
||||
total_length = r.headers.get("content-length")
|
||||
assert total_length is not None
|
||||
for chunk in track(
|
||||
r.iter_content(chunk_size=1024),
|
||||
total=int(total_length) / 1024 + 1,
|
||||
description="Downloading vocab tree...",
|
||||
):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
return vocab_tree_filename
|
||||
|
||||
|
||||
def run_command(cmd: str, verbose=False) -> Optional[str]:
|
||||
"""Runs a command and returns the output.
|
||||
|
||||
Args:
|
||||
cmd: Command to run.
|
||||
verbose: If True, logs the output of the command.
|
||||
Returns:
|
||||
The output of the command if return_output is True, otherwise None.
|
||||
"""
|
||||
out = subprocess.run(cmd, capture_output=not verbose, shell=True, check=False)
|
||||
if out.returncode != 0:
|
||||
CONSOLE.rule("[bold red] :skull: :skull: :skull: ERROR :skull: :skull: :skull: ", style="red")
|
||||
CONSOLE.print(f"[bold red]Error running command: {cmd}")
|
||||
CONSOLE.rule(style="red")
|
||||
CONSOLE.print(out.stderr.decode("utf-8"))
|
||||
sys.exit(1)
|
||||
if out.stdout is not None:
|
||||
return out.stdout.decode("utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def get_num_frames_in_video(video: Path) -> int:
|
||||
"""Returns the number of frames in a video.
|
||||
|
||||
Args:
|
||||
video: Path to a video.
|
||||
|
||||
Returns:
|
||||
The number of frames in a video.
|
||||
"""
|
||||
cmd = f"ffprobe -v error -select_streams v:0 -count_packets \
|
||||
-show_entries stream=nb_read_packets -of csv=p=0 {video}"
|
||||
output = run_command(cmd)
|
||||
assert output is not None
|
||||
output = output.strip(" ,\t\n\r")
|
||||
return int(output)
|
||||
|
||||
|
||||
def convert_video_to_images(
|
||||
video_path: Path, image_dir: Path, num_frames_target: int, verbose: bool = False
|
||||
) -> Tuple[List[str], int]:
|
||||
"""Converts a video into a sequence of images.
|
||||
|
||||
Args:
|
||||
video_path: Path to the video.
|
||||
output_dir: Path to the output directory.
|
||||
num_frames_target: Number of frames to extract.
|
||||
verbose: If True, logs the output of the command.
|
||||
Returns:
|
||||
A tuple containing summary of the conversion and the number of extracted frames.
|
||||
"""
|
||||
|
||||
with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose):
|
||||
# delete existing images in folder
|
||||
for img in image_dir.glob("*.png"):
|
||||
if verbose:
|
||||
CONSOLE.log(f"Deleting {img}")
|
||||
img.unlink()
|
||||
|
||||
num_frames = get_num_frames_in_video(video_path)
|
||||
if num_frames == 0:
|
||||
CONSOLE.print(f"[bold red]Error: Video has no frames: {video_path}")
|
||||
sys.exit(1)
|
||||
print("Number of frames in video:", num_frames)
|
||||
|
||||
out_filename = image_dir / "frame_%05d.png"
|
||||
ffmpeg_cmd = f"ffmpeg -i {video_path}"
|
||||
spacing = num_frames // num_frames_target
|
||||
|
||||
if spacing > 1:
|
||||
ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB -r 1"
|
||||
else:
|
||||
CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.")
|
||||
|
||||
ffmpeg_cmd += f" {out_filename}"
|
||||
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
num_final_frames = len(list(image_dir.glob("*.png")))
|
||||
summary_log = []
|
||||
summary_log.append(f"Starting with {num_frames} video frames")
|
||||
summary_log.append(f"We extracted {num_final_frames} images")
|
||||
CONSOLE.log("[bold green]:tada: Done converting video to images.")
|
||||
|
||||
return summary_log, num_final_frames
|
||||
|
||||
|
||||
def convert_insta360_to_images(
|
||||
video_front: Path,
|
||||
video_back: Path,
|
||||
image_dir: Path,
|
||||
num_frames_target: int,
|
||||
crop_percentage: float = 0.7,
|
||||
verbose: bool = False,
|
||||
) -> Tuple[List[str], int]:
|
||||
"""Converts a video into a sequence of images.
|
||||
|
||||
Args:
|
||||
video_front: Path to the front video.
|
||||
video_back: Path to the back video.
|
||||
output_dir: Path to the output directory.
|
||||
num_frames_target: Number of frames to extract.
|
||||
verbose: If True, logs the output of the command.
|
||||
Returns:
|
||||
A tuple containing summary of the conversion and the number of extracted frames.
|
||||
"""
|
||||
|
||||
with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose):
|
||||
# delete existing images in folder
|
||||
for img in image_dir.glob("*.png"):
|
||||
if verbose:
|
||||
CONSOLE.log(f"Deleting {img}")
|
||||
img.unlink()
|
||||
|
||||
num_frames_front = get_num_frames_in_video(video_front)
|
||||
num_frames_back = get_num_frames_in_video(video_back)
|
||||
if num_frames_front == 0:
|
||||
CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}")
|
||||
sys.exit(1)
|
||||
if num_frames_back == 0:
|
||||
CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}")
|
||||
sys.exit(1)
|
||||
|
||||
spacing = num_frames_front // (num_frames_target // 2)
|
||||
vf_cmds = []
|
||||
if spacing > 1:
|
||||
vf_cmds = [f"thumbnail={spacing}", "setpts=N/TB"]
|
||||
else:
|
||||
CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.")
|
||||
|
||||
vf_cmds.append(f"crop=iw*{crop_percentage}:ih*{crop_percentage}")
|
||||
|
||||
front_vf_cmds = vf_cmds + ["transpose=2"]
|
||||
back_vf_cmds = vf_cmds + ["transpose=1"]
|
||||
|
||||
front_ffmpeg_cmd = f"ffmpeg -i {video_front} -vf {','.join(front_vf_cmds)} -r 1 {image_dir / 'frame_%05d.png'}"
|
||||
back_ffmpeg_cmd = (
|
||||
f"ffmpeg -i {video_back} -vf {','.join(back_vf_cmds)} -r 1 {image_dir / 'back_frame_%05d.png'}"
|
||||
)
|
||||
|
||||
run_command(front_ffmpeg_cmd, verbose=verbose)
|
||||
run_command(back_ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
num_extracted_front_frames = len(list(image_dir.glob("frame*.png")))
|
||||
for i, img in enumerate(image_dir.glob("back_frame_*.png")):
|
||||
img.rename(image_dir / f"frame_{i+1+num_extracted_front_frames:05d}.png")
|
||||
|
||||
num_final_frames = len(list(image_dir.glob("*.png")))
|
||||
summary_log = []
|
||||
summary_log.append(f"Starting with {num_frames_front + num_frames_back} video frames")
|
||||
summary_log.append(f"We extracted {num_final_frames} images")
|
||||
CONSOLE.log("[bold green]:tada: Done converting insta360 to images.")
|
||||
|
||||
return summary_log, num_final_frames
|
||||
|
||||
|
||||
def copy_images_list(
|
||||
image_paths: List[Path], image_dir: Path, crop_border_pixels: Optional[int] = None, verbose: bool = False
|
||||
) -> List[Path]:
|
||||
"""Copy all images in a list of Paths. Useful for filtering from a directory.
|
||||
Args:
|
||||
image_paths: List of Paths of images to copy to a new directory.
|
||||
image_dir: Path to the output directory.
|
||||
crop_border_pixels: If not None, crops each edge by the specified number of pixels.
|
||||
verbose: If True, print extra logging.
|
||||
Returns:
|
||||
A list of the copied image Paths.
|
||||
"""
|
||||
|
||||
# Remove original directory only if we provide a proper image folder path
|
||||
if image_dir.is_dir() and len(image_paths):
|
||||
shutil.rmtree(image_dir, ignore_errors=True)
|
||||
image_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
copied_image_paths = []
|
||||
|
||||
# Images should be 1-indexed for the rest of the pipeline.
|
||||
for idx, image_path in enumerate(image_paths):
|
||||
if verbose:
|
||||
CONSOLE.log(f"Copying image {idx + 1} of {len(image_paths)}...")
|
||||
copied_image_path = image_dir / f"frame_{idx + 1:05d}{image_path.suffix}"
|
||||
shutil.copy(image_path, copied_image_path)
|
||||
copied_image_paths.append(copied_image_path)
|
||||
|
||||
if crop_border_pixels is not None:
|
||||
file_type = image_paths[0].suffix
|
||||
filename = f"frame_%05d{file_type}"
|
||||
crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}"
|
||||
ffmpeg_cmd = f"ffmpeg -y -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}"
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
num_frames = len(image_paths)
|
||||
|
||||
if num_frames == 0:
|
||||
CONSOLE.log("[bold red]:skull: No usable images in the data folder.")
|
||||
else:
|
||||
CONSOLE.log("[bold green]:tada: Done copying images.")
|
||||
|
||||
return copied_image_paths
|
||||
|
||||
|
||||
def copy_images(data: Path, image_dir: Path, verbose) -> int:
|
||||
"""Copy images from a directory to a new directory.
|
||||
|
||||
Args:
|
||||
data: Path to the directory of images.
|
||||
image_dir: Path to the output directory.
|
||||
verbose: If True, print extra logging.
|
||||
Returns:
|
||||
The number of images copied.
|
||||
"""
|
||||
with status(msg="[bold yellow]Copying images...", spinner="bouncingBall", verbose=verbose):
|
||||
allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"]
|
||||
image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts])
|
||||
|
||||
num_frames = len(copy_images_list(image_paths, image_dir, verbose))
|
||||
|
||||
return num_frames
|
||||
|
||||
|
||||
def downscale_images(image_dir: Path, num_downscales: int, verbose: bool = False) -> str:
|
||||
"""Downscales the images in the directory. Uses FFMPEG.
|
||||
|
||||
Assumes images are named frame_00001.png, frame_00002.png, etc.
|
||||
|
||||
Args:
|
||||
image_dir: Path to the directory containing the images.
|
||||
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
|
||||
verbose: If True, logs the output of the command.
|
||||
|
||||
Returns:
|
||||
Summary of downscaling.
|
||||
"""
|
||||
|
||||
if num_downscales == 0:
|
||||
return "No downscaling performed."
|
||||
|
||||
with status(msg="[bold yellow]Downscaling images...", spinner="growVertical", verbose=verbose):
|
||||
downscale_factors = [2**i for i in range(num_downscales + 1)[1:]]
|
||||
for downscale_factor in downscale_factors:
|
||||
assert downscale_factor > 1
|
||||
assert isinstance(downscale_factor, int)
|
||||
downscale_dir = image_dir.parent / f"images_{downscale_factor}"
|
||||
downscale_dir.mkdir(parents=True, exist_ok=True)
|
||||
file_type = image_dir.glob("frame_*").__next__().suffix
|
||||
filename = f"frame_%05d{file_type}"
|
||||
ffmpeg_cmd = [
|
||||
f"ffmpeg -i {image_dir / filename} ",
|
||||
f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ",
|
||||
f"{downscale_dir / filename}",
|
||||
]
|
||||
ffmpeg_cmd = " ".join(ffmpeg_cmd)
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
CONSOLE.log("[bold green]:tada: Done downscaling images.")
|
||||
downscale_text = [f"[bold blue]{2**(i+1)}x[/bold blue]" for i in range(num_downscales)]
|
||||
downscale_text = ", ".join(downscale_text[:-1]) + " and " + downscale_text[-1]
|
||||
return f"We downsampled the images by {downscale_text}"
|
||||
|
||||
|
||||
def run_colmap(
|
||||
image_dir: Path,
|
||||
colmap_dir: Path,
|
||||
camera_model: CameraModel,
|
||||
gpu: bool = True,
|
||||
verbose: bool = False,
|
||||
matching_method: Literal["vocab_tree", "exhaustive", "sequential"] = "vocab_tree",
|
||||
colmap_cmd: str = "colmap",
|
||||
) -> None:
|
||||
"""Runs COLMAP on the images.
|
||||
|
||||
Args:
|
||||
image_dir: Path to the directory containing the images.
|
||||
colmap_dir: Path to the output directory.
|
||||
camera_model: Camera model to use.
|
||||
gpu: If True, use GPU.
|
||||
verbose: If True, logs the output of the command.
|
||||
"""
|
||||
|
||||
colmap_version = get_colmap_version(colmap_cmd)
|
||||
|
||||
colmap_database_path = colmap_dir / "database.db"
|
||||
if colmap_database_path.exists():
|
||||
# Can't use missing_ok argument because of Python 3.7 compatibility.
|
||||
colmap_database_path.unlink()
|
||||
|
||||
# Feature extraction
|
||||
feature_extractor_cmd = [
|
||||
f"{colmap_cmd} feature_extractor",
|
||||
f"--database_path {colmap_dir / 'database.db'}",
|
||||
f"--image_path {image_dir}",
|
||||
"--ImageReader.single_camera 1",
|
||||
f"--ImageReader.camera_model {camera_model.value}",
|
||||
f"--SiftExtraction.use_gpu {int(gpu)}",
|
||||
]
|
||||
feature_extractor_cmd = " ".join(feature_extractor_cmd)
|
||||
with status(msg="[bold yellow]Running COLMAP feature extractor...", spinner="moon", verbose=verbose):
|
||||
run_command(feature_extractor_cmd, verbose=verbose)
|
||||
|
||||
CONSOLE.log("[bold green]:tada: Done extracting COLMAP features.")
|
||||
|
||||
# Feature matching
|
||||
feature_matcher_cmd = [
|
||||
f"{colmap_cmd} {matching_method}_matcher",
|
||||
f"--database_path {colmap_dir / 'database.db'}",
|
||||
f"--SiftMatching.use_gpu {int(gpu)}",
|
||||
]
|
||||
if matching_method == "vocab_tree":
|
||||
vocab_tree_filename = get_vocab_tree()
|
||||
feature_matcher_cmd.append(f"--VocabTreeMatching.vocab_tree_path {vocab_tree_filename}")
|
||||
feature_matcher_cmd = " ".join(feature_matcher_cmd)
|
||||
with status(msg="[bold yellow]Running COLMAP feature matcher...", spinner="runner", verbose=verbose):
|
||||
run_command(feature_matcher_cmd, verbose=verbose)
|
||||
CONSOLE.log("[bold green]:tada: Done matching COLMAP features.")
|
||||
|
||||
# Bundle adjustment
|
||||
sparse_dir = colmap_dir / "sparse"
|
||||
sparse_dir.mkdir(parents=True, exist_ok=True)
|
||||
mapper_cmd = [
|
||||
f"{colmap_cmd} mapper",
|
||||
f"--database_path {colmap_dir / 'database.db'}",
|
||||
f"--image_path {image_dir}",
|
||||
f"--output_path {sparse_dir}",
|
||||
]
|
||||
if colmap_version >= 3.7:
|
||||
mapper_cmd.append("--Mapper.ba_global_function_tolerance 1e-6")
|
||||
|
||||
mapper_cmd = " ".join(mapper_cmd)
|
||||
|
||||
with status(
|
||||
msg="[bold yellow]Running COLMAP bundle adjustment... (This may take a while)",
|
||||
spinner="circle",
|
||||
verbose=verbose,
|
||||
):
|
||||
run_command(mapper_cmd, verbose=verbose)
|
||||
CONSOLE.log("[bold green]:tada: Done COLMAP bundle adjustment.")
|
||||
with status(msg="[bold yellow]Refine intrinsics...", spinner="dqpb", verbose=verbose):
|
||||
bundle_adjuster_cmd = [
|
||||
f"{colmap_cmd} bundle_adjuster",
|
||||
f"--input_path {sparse_dir}/0",
|
||||
f"--output_path {sparse_dir}/0",
|
||||
"--BundleAdjustment.refine_principal_point 1",
|
||||
]
|
||||
run_command(" ".join(bundle_adjuster_cmd), verbose=verbose)
|
||||
CONSOLE.log("[bold green]:tada: Done refining intrinsics.")
|
||||
|
||||
|
||||
def colmap_to_json(cameras_path: Path, images_path: Path, output_dir: Path, camera_model: CameraModel) -> int:
|
||||
"""Converts COLMAP's cameras.bin and images.bin to a JSON file.
|
||||
|
||||
Args:
|
||||
cameras_path: Path to the cameras.bin file.
|
||||
images_path: Path to the images.bin file.
|
||||
output_dir: Path to the output directory.
|
||||
camera_model: Camera model used.
|
||||
|
||||
Returns:
|
||||
The number of registered images.
|
||||
"""
|
||||
|
||||
cameras = colmap_utils.read_cameras_binary(cameras_path)
|
||||
images = colmap_utils.read_images_binary(images_path)
|
||||
|
||||
# Only supports one camera
|
||||
camera_params = cameras[1].params
|
||||
|
||||
frames = []
|
||||
for _, im_data in images.items():
|
||||
rotation = colmap_utils.qvec2rotmat(im_data.qvec)
|
||||
translation = im_data.tvec.reshape(3, 1)
|
||||
w2c = np.concatenate([rotation, translation], 1)
|
||||
w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0)
|
||||
c2w = np.linalg.inv(w2c)
|
||||
# Convert from COLMAP's camera coordinate system to ours
|
||||
c2w[0:3, 1:3] *= -1
|
||||
c2w = c2w[np.array([1, 0, 2, 3]), :]
|
||||
c2w[2, :] *= -1
|
||||
|
||||
name = Path(f"./images/{im_data.name}")
|
||||
|
||||
frame = {
|
||||
"file_path": name.as_posix(),
|
||||
"transform_matrix": c2w.tolist(),
|
||||
}
|
||||
frames.append(frame)
|
||||
|
||||
out = {
|
||||
"fl_x": float(camera_params[0]),
|
||||
"fl_y": float(camera_params[1]),
|
||||
"cx": float(camera_params[2]),
|
||||
"cy": float(camera_params[3]),
|
||||
"w": cameras[1].width,
|
||||
"h": cameras[1].height,
|
||||
"camera_model": camera_model.value,
|
||||
}
|
||||
|
||||
if camera_model == CameraModel.OPENCV:
|
||||
out.update(
|
||||
{
|
||||
"k1": float(camera_params[4]),
|
||||
"k2": float(camera_params[5]),
|
||||
"p1": float(camera_params[6]),
|
||||
"p2": float(camera_params[7]),
|
||||
}
|
||||
)
|
||||
if camera_model == CameraModel.OPENCV_FISHEYE:
|
||||
out.update(
|
||||
{
|
||||
"k1": float(camera_params[4]),
|
||||
"k2": float(camera_params[5]),
|
||||
"k3": float(camera_params[6]),
|
||||
"k4": float(camera_params[7]),
|
||||
}
|
||||
)
|
||||
|
||||
out["frames"] = frames
|
||||
|
||||
with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
|
||||
json.dump(out, f, indent=4)
|
||||
|
||||
return len(frames)
|
||||
|
||||
|
||||
def record3d_to_json(images_paths: List[Path], metadata_path: Path, output_dir: Path, indices: np.ndarray) -> int:
|
||||
"""Converts Record3D's metadata and image paths to a JSON file.
|
||||
|
||||
Args:
|
||||
images_paths: list if image paths.
|
||||
metadata_path: Path to the Record3D metadata JSON file.
|
||||
output_dir: Path to the output directory.
|
||||
indices: Indices to sample the metadata_path. Should be the same length as images_paths.
|
||||
|
||||
Returns:
|
||||
The number of registered images.
|
||||
"""
|
||||
|
||||
assert len(images_paths) == len(indices)
|
||||
|
||||
metadata_dict = io.load_from_json(metadata_path)
|
||||
|
||||
poses_data = np.array(metadata_dict["poses"]) # (N, 3, 4)
|
||||
camera_to_worlds = np.concatenate(
|
||||
[Rotation.from_quat(poses_data[:, :4]).as_matrix(), poses_data[:, 4:, None]],
|
||||
axis=-1,
|
||||
).astype(np.float32)
|
||||
camera_to_worlds = camera_to_worlds[indices]
|
||||
|
||||
homogeneous_coord = np.zeros_like(camera_to_worlds[..., :1, :])
|
||||
homogeneous_coord[..., :, 3] = 1
|
||||
camera_to_worlds = np.concatenate([camera_to_worlds, homogeneous_coord], -2)
|
||||
|
||||
frames = []
|
||||
for i, im_path in enumerate(images_paths):
|
||||
c2w = camera_to_worlds[i]
|
||||
frame = {
|
||||
"file_path": im_path.as_posix(),
|
||||
"transform_matrix": c2w.tolist(),
|
||||
}
|
||||
frames.append(frame)
|
||||
|
||||
# Camera intrinsics
|
||||
K = np.array(metadata_dict["K"]).reshape((3, 3)).T
|
||||
focal_length = K[0, 0]
|
||||
|
||||
H = metadata_dict["h"]
|
||||
W = metadata_dict["w"]
|
||||
|
||||
# TODO(akristoffersen): The metadata dict comes with principle points,
|
||||
# but caused errors in image coord indexing. Should update once that is fixed.
|
||||
cx, cy = W / 2, H / 2
|
||||
|
||||
out = {
|
||||
"fl_x": focal_length,
|
||||
"fl_y": focal_length,
|
||||
"cx": cx,
|
||||
"cy": cy,
|
||||
"w": W,
|
||||
"h": H,
|
||||
"camera_model": CAMERA_MODELS["perspective"].name,
|
||||
}
|
||||
|
||||
out["frames"] = frames
|
||||
|
||||
with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
|
||||
json.dump(out, f, indent=4)
|
||||
|
||||
return len(frames)
|
||||
|
||||
|
||||
def get_insta360_filenames(data: Path) -> Tuple[Path, Path]:
|
||||
"""Returns the filenames of the Insta360 videos from a single video file.
|
||||
|
||||
Example input name: VID_20220212_070353_00_003.insv
|
||||
|
||||
Args:
|
||||
data: Path to a Insta360 file.
|
||||
|
||||
Returns:
|
||||
The filenames of the Insta360 videios.
|
||||
"""
|
||||
if data.suffix != ".insv":
|
||||
raise ValueError("The input file must be an .insv file.")
|
||||
file_parts = data.stem.split("_")
|
||||
|
||||
stem_back = f"VID_{file_parts[1]}_{file_parts[2]}_00_{file_parts[4]}.insv"
|
||||
stem_front = f"VID_{file_parts[1]}_{file_parts[2]}_10_{file_parts[4]}.insv"
|
||||
|
||||
filename_back = data.parent / stem_back
|
||||
filename_front = data.parent / stem_front
|
||||
|
||||
if not filename_back.exists():
|
||||
raise FileNotFoundError(f"Could not find {filename_back}")
|
||||
if not filename_front.exists():
|
||||
raise FileNotFoundError(f"Could not find {filename_front}")
|
||||
|
||||
return filename_back, filename_front
|
||||
|
||||
|
||||
def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str:
|
||||
"""Returns a summary of the matching results.
|
||||
|
||||
Args:
|
||||
num_intial_frames: The number of initial frames.
|
||||
num_matched_frames: The number of matched frames.
|
||||
|
||||
Returns:
|
||||
A summary of the matching results.
|
||||
"""
|
||||
match_ratio = num_matched_frames / num_intial_frames
|
||||
if match_ratio == 1:
|
||||
return "[bold green]COLAMP found poses for all images, CONGRATS!"
|
||||
if match_ratio < 0.4:
|
||||
result = f"[bold red]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%"
|
||||
result += " of the images. This is low.\nThis can be caused by a variety of reasons,"
|
||||
result += " such poor scene coverage, blurry images, or large exposure changes."
|
||||
return result
|
||||
if match_ratio < 0.8:
|
||||
result = f"[bold yellow]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%"
|
||||
result += " of the images.\nThis isn't great, but may be ok."
|
||||
result += "\nMissing poses can be caused by a variety of reasons, such poor scene coverage, blurry images,"
|
||||
result += " or large exposure changes."
|
||||
return result
|
||||
return f"[bold green]COLMAP found poses for {num_matched_frames / num_intial_frames * 100:.2f}% of the images."
|
||||
|
||||
|
||||
def polycam_to_json(
|
||||
image_filenames: List[Path],
|
||||
cameras_dir: Path,
|
||||
output_dir: Path,
|
||||
min_blur_score: float = 0.0,
|
||||
crop_border_pixels: int = 0,
|
||||
) -> List[str]:
|
||||
"""Convert Polycam data into a nerfstudio dataset.
|
||||
|
||||
Args:
|
||||
image_filenames: List of paths to the original images.
|
||||
cameras_dir: Path to the polycam cameras directory.
|
||||
output_dir: Path to the output directory.
|
||||
min_blur_score: Minimum blur score to use an image. Images below this value will be skipped.
|
||||
crop_border_pixels: Number of pixels to crop from each border of the image.
|
||||
|
||||
Returns:
|
||||
Summary of the conversion.
|
||||
"""
|
||||
|
||||
frame_json = io.load_from_json(cameras_dir / f"{image_filenames[0].stem}.json")
|
||||
|
||||
# TODO Add per-frame intrinsics
|
||||
|
||||
data = {}
|
||||
data["fl_x"] = frame_json["fx"]
|
||||
data["fl_y"] = frame_json["fy"]
|
||||
data["cx"] = frame_json["cx"] - crop_border_pixels
|
||||
data["cy"] = frame_json["cy"] - crop_border_pixels
|
||||
data["w"] = frame_json["width"] - crop_border_pixels * 2
|
||||
data["h"] = frame_json["height"] - crop_border_pixels * 2
|
||||
data["camera_model"] = CAMERA_MODELS["perspective"].value
|
||||
# Needs to be a string for camera_utils.auto_orient_and_center_poses
|
||||
data["orientation_override"] = "none"
|
||||
|
||||
frames = []
|
||||
skipped_frames = 0
|
||||
for i, image_filename in enumerate(image_filenames):
|
||||
json_filename = cameras_dir / f"{image_filename.stem}.json"
|
||||
frame_json = io.load_from_json(json_filename)
|
||||
if "blur_score" in frame_json and frame_json["blur_score"] < min_blur_score:
|
||||
skipped_frames += 1
|
||||
continue
|
||||
frame = {}
|
||||
frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}"
|
||||
# Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions.
|
||||
frame["transform_matrix"] = [
|
||||
[frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]],
|
||||
[frame_json["t_00"], frame_json["t_01"], frame_json["t_02"], frame_json["t_03"]],
|
||||
[frame_json["t_10"], frame_json["t_11"], frame_json["t_12"], frame_json["t_13"]],
|
||||
[0.0, 0.0, 0.0, 1.0],
|
||||
]
|
||||
frames.append(frame)
|
||||
data["frames"] = frames
|
||||
|
||||
with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=4)
|
||||
|
||||
summary = []
|
||||
if skipped_frames > 0:
|
||||
summary.append(f"Skipped {skipped_frames} frames due to low blur score.")
|
||||
summary.append(f"Final dataset is {len(image_filenames) - skipped_frames} frames.")
|
||||
|
||||
if len(image_filenames) - skipped_frames == 0:
|
||||
CONSOLE.print("[bold red]No images remain after filtering, exiting")
|
||||
sys.exit(1)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessImages:
|
||||
"""Process images into a nerfstudio dataset.
|
||||
@@ -755,18 +69,18 @@ class ProcessImages:
|
||||
summary_log = []
|
||||
|
||||
# Copy images to output directory
|
||||
num_frames = copy_images(self.data, image_dir=image_dir, verbose=self.verbose)
|
||||
num_frames = process_data_utils.copy_images(self.data, image_dir=image_dir, verbose=self.verbose)
|
||||
summary_log.append(f"Starting with {num_frames} images")
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
# Run COLMAP
|
||||
colmap_dir = self.output_dir / "colmap"
|
||||
if not self.skip_colmap:
|
||||
colmap_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_colmap(
|
||||
colmap_utils.run_colmap(
|
||||
image_dir=image_dir,
|
||||
colmap_dir=colmap_dir,
|
||||
camera_model=CAMERA_MODELS[self.camera_type],
|
||||
@@ -779,14 +93,14 @@ class ProcessImages:
|
||||
# Save transforms.json
|
||||
if (colmap_dir / "sparse" / "0" / "cameras.bin").exists():
|
||||
with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"):
|
||||
num_matched_frames = colmap_to_json(
|
||||
num_matched_frames = colmap_utils.colmap_to_json(
|
||||
cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin",
|
||||
images_path=colmap_dir / "sparse" / "0" / "images.bin",
|
||||
output_dir=self.output_dir,
|
||||
camera_model=CAMERA_MODELS[self.camera_type],
|
||||
)
|
||||
summary_log.append(f"Colmap matched {num_matched_frames} images")
|
||||
summary_log.append(get_matching_summary(num_frames, num_matched_frames))
|
||||
summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames))
|
||||
else:
|
||||
CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json")
|
||||
|
||||
@@ -841,19 +155,19 @@ class ProcessVideo:
|
||||
image_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Convert video to images
|
||||
summary_log, num_extracted_frames = convert_video_to_images(
|
||||
summary_log, num_extracted_frames = process_data_utils.convert_video_to_images(
|
||||
self.data, image_dir=image_dir, num_frames_target=self.num_frames_target, verbose=self.verbose
|
||||
)
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
# Run Colmap
|
||||
colmap_dir = self.output_dir / "colmap"
|
||||
if not self.skip_colmap:
|
||||
colmap_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_colmap(
|
||||
colmap_utils.run_colmap(
|
||||
image_dir=image_dir,
|
||||
colmap_dir=colmap_dir,
|
||||
camera_model=CAMERA_MODELS[self.camera_type],
|
||||
@@ -866,14 +180,14 @@ class ProcessVideo:
|
||||
# Save transforms.json
|
||||
if (colmap_dir / "sparse" / "0" / "cameras.bin").exists():
|
||||
with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"):
|
||||
num_matched_frames = colmap_to_json(
|
||||
num_matched_frames = colmap_utils.colmap_to_json(
|
||||
cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin",
|
||||
images_path=colmap_dir / "sparse" / "0" / "images.bin",
|
||||
output_dir=self.output_dir,
|
||||
camera_model=CAMERA_MODELS[self.camera_type],
|
||||
)
|
||||
summary_log.append(f"Colmap matched {num_matched_frames} images")
|
||||
summary_log.append(get_matching_summary(num_extracted_frames, num_matched_frames))
|
||||
summary_log.append(colmap_utils.get_matching_summary(num_extracted_frames, num_matched_frames))
|
||||
else:
|
||||
CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json")
|
||||
|
||||
@@ -929,10 +243,10 @@ class ProcessInsta360:
|
||||
image_dir = self.output_dir / "images"
|
||||
image_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
filename_back, filename_front = get_insta360_filenames(self.data)
|
||||
filename_back, filename_front = insta360_utils.get_insta360_filenames(self.data)
|
||||
|
||||
# Convert video to images
|
||||
summary_log, num_extracted_frames = convert_insta360_to_images(
|
||||
summary_log, num_extracted_frames = insta360_utils.convert_insta360_to_images(
|
||||
video_front=filename_front,
|
||||
video_back=filename_back,
|
||||
image_dir=image_dir,
|
||||
@@ -941,14 +255,14 @@ class ProcessInsta360:
|
||||
)
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
# Run Colmap
|
||||
colmap_dir = self.output_dir / "colmap"
|
||||
if not self.skip_colmap:
|
||||
colmap_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_colmap(
|
||||
colmap_utils.run_colmap(
|
||||
image_dir=image_dir,
|
||||
colmap_dir=colmap_dir,
|
||||
camera_model=CAMERA_MODELS["fisheye"],
|
||||
@@ -961,14 +275,14 @@ class ProcessInsta360:
|
||||
# Save transforms.json
|
||||
if (colmap_dir / "sparse" / "0" / "cameras.bin").exists():
|
||||
with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"):
|
||||
num_matched_frames = colmap_to_json(
|
||||
num_matched_frames = colmap_utils.colmap_to_json(
|
||||
cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin",
|
||||
images_path=colmap_dir / "sparse" / "0" / "images.bin",
|
||||
output_dir=self.output_dir,
|
||||
camera_model=CAMERA_MODELS["fisheye"],
|
||||
)
|
||||
summary_log.append(f"Colmap matched {num_matched_frames} images")
|
||||
summary_log.append(get_matching_summary(num_extracted_frames, num_matched_frames))
|
||||
summary_log.append(colmap_utils.get_matching_summary(num_extracted_frames, num_matched_frames))
|
||||
else:
|
||||
CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json")
|
||||
|
||||
@@ -1030,7 +344,9 @@ class ProcessRecord3D:
|
||||
|
||||
record3d_image_filenames = list(np.array(record3d_image_filenames)[idx])
|
||||
# Copy images to output directory
|
||||
copied_image_paths = copy_images_list(record3d_image_filenames, image_dir=image_dir, verbose=self.verbose)
|
||||
copied_image_paths = process_data_utils.copy_images_list(
|
||||
record3d_image_filenames, image_dir=image_dir, verbose=self.verbose
|
||||
)
|
||||
num_frames = len(copied_image_paths)
|
||||
|
||||
copied_image_paths = [Path("images/" + copied_image_path.name) for copied_image_path in copied_image_paths]
|
||||
@@ -1042,10 +358,10 @@ class ProcessRecord3D:
|
||||
)
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
metadata_path = self.data / "metadata.json"
|
||||
record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx)
|
||||
record3d_utils.record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx)
|
||||
CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:")
|
||||
|
||||
for summary in summary_log:
|
||||
@@ -1128,7 +444,7 @@ class ProcessPolycam:
|
||||
|
||||
polycam_image_filenames = list(np.array(polycam_image_filenames)[idx])
|
||||
# Copy images to output directory
|
||||
copied_image_paths = copy_images_list(
|
||||
copied_image_paths = process_data_utils.copy_images_list(
|
||||
polycam_image_filenames,
|
||||
image_dir=image_dir,
|
||||
crop_border_pixels=self.crop_border_pixels,
|
||||
@@ -1148,14 +464,14 @@ class ProcessPolycam:
|
||||
summary_log.append(f"Started with {num_frames} images")
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
# Save json
|
||||
if num_frames == 0:
|
||||
CONSOLE.print("[bold red]No images found, exiting")
|
||||
sys.exit(1)
|
||||
summary_log.extend(
|
||||
polycam_to_json(
|
||||
polycam_utils.polycam_to_json(
|
||||
image_filenames=polycam_image_filenames,
|
||||
cameras_dir=polycam_cameras_dir,
|
||||
output_dir=self.output_dir,
|
||||
|
||||
Reference in new issue
Block a user